From 6f24badfa940fadbc5cf66d8266a221211ff1adc Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 21 Apr 2026 16:52:28 -0600 Subject: [PATCH 01/69] [hipFile] Create a hipFile shim layer compatible with cufile-python This shim layer matches the cufile-python API that is currently in use by LMCache. This allows hipFile to maintain its own API while not requiring additional code changes in LMCache. Signed-off-by: Riley Dixon --- lmcache/v1/memory_management.py | 11 +- lmcache/v1/storage_backend/gds_backend.py | 6 +- lmcache/v1/storage_backend/hipfile_shim.py | 159 +++++++++++++++++++++ 3 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 lmcache/v1/storage_backend/hipfile_shim.py diff --git a/lmcache/v1/memory_management.py b/lmcache/v1/memory_management.py index 93d5736afd2..0bf50ef2034 100644 --- a/lmcache/v1/memory_management.py +++ b/lmcache/v1/memory_management.py @@ -2411,9 +2411,8 @@ def __init__(self, size: int, device=None): # HACK: hipfile import is placed here to avoid import errors on # hardware without GPUDirect Storage / hipFile support. # Third Party - from hipfile.bindings import hipFileBufDeregister, hipFileBufRegister + from hipfile import Buffer - self.hipFileBufDeregister = hipFileBufDeregister if device is None: if torch.cuda.is_available(): # TODO: On ROCm, PyTorch still uses the CUDA API internally @@ -2423,10 +2422,14 @@ def __init__(self, size: int, device=None): super().__init__(size, device, align_bytes=4096) self.base_pointer = self.tensor.data_ptr() - hipFileBufRegister(ctypes.c_void_p(self.base_pointer), size, flags=0) + self.hipfile_buffer = Buffer(self.base_pointer, size, flags=0) + self.hipfile_buffer.register() def __del__(self): - self.hipFileBufDeregister(ctypes.c_void_p(self.base_pointer)) + try: + self.hipfile_buffer.deregister() + except Exception: + pass def __str__(self): return "HipFileMemoryAllocator" diff --git a/lmcache/v1/storage_backend/gds_backend.py b/lmcache/v1/storage_backend/gds_backend.py index ebb5f8a2e5a..f74c4069999 100644 --- a/lmcache/v1/storage_backend/gds_backend.py +++ b/lmcache/v1/storage_backend/gds_backend.py @@ -287,11 +287,11 @@ def __init__( elif self.gds_backend == "hipfile": # HACK: hipfile import may be buggy on some hardware # (e.g., without GPUDirect), so it's temporarily put here. - # Third Party - import hipfile + # First Party + from lmcache.v1.storage_backend import hipfile_shim self.cudart = None - self.gds_module = hipfile + self.gds_module = hipfile_shim self._gds_driver = self.gds_module.CuFileDriver() else: raise ValueError(f"Unsupported gds_backend '{self.gds_backend}'") diff --git a/lmcache/v1/storage_backend/hipfile_shim.py b/lmcache/v1/storage_backend/hipfile_shim.py new file mode 100644 index 00000000000..508eeaca8c8 --- /dev/null +++ b/lmcache/v1/storage_backend/hipfile_shim.py @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Shim that exposes hipfile behind compatible names for cufile-python. + +This module wraps ``hipfile.Driver``, ``hipfile.FileHandle``, and +``hipfile.Buffer`` so that callers written against the cufile-python API +(``CuFileDriver`` / ``CuFile``) can use hipfile without any code changes. +""" + +# Standard +import ctypes +import os + +# Third Party +from hipfile import Buffer, Driver, FileHandle + + +def _os_flags(mode: str) -> int: + """Translate a Python-style mode string to ``os.open`` flags.""" + flags_map = { + "r": os.O_RDONLY, + "r+": os.O_RDWR, + "w": os.O_CREAT | os.O_WRONLY | os.O_TRUNC, + "w+": os.O_CREAT | os.O_RDWR | os.O_TRUNC, + "a": os.O_CREAT | os.O_WRONLY | os.O_APPEND, + "a+": os.O_CREAT | os.O_RDWR | os.O_APPEND, + } + return flags_map[mode] + + +def _singleton(cls): + """Decorator that turns *cls* into a singleton.""" + _instances: dict = {} + + def wrapper(*args, **kwargs): + if cls not in _instances: + _instances[cls] = cls(*args, **kwargs) + return _instances[cls] + + return wrapper + + +@_singleton +class CuFileDriver: + """Singleton wrapper around :class:`hipfile.Driver`. + + Matches the cufile-python ``CuFileDriver`` interface: the driver is opened + on first instantiation and closed when the singleton is destroyed. + """ + + def __init__(self) -> None: + self._driver = Driver() + self._driver.open() + + def __del__(self) -> None: + self._driver.close() + + +class CuFile: + """Wrapper around :class:`hipfile.FileHandle` with cufile-python semantics. + + Args: + path: Filesystem path to open. + mode: Python-style mode string (``"r"``, ``"r+"``, ``"w"``, etc.). + use_direct_io: If ``True``, ``O_DIRECT`` is ORed into the flags. + """ + + def __init__( + self, + path: str, + mode: str = "r", + use_direct_io: bool = False, + ) -> None: + flags = _os_flags(mode) + if use_direct_io: + flags |= os.O_DIRECT + self._file_handle = FileHandle(path, flags) + + @property + def is_open(self) -> bool: + """Return ``True`` if the underlying handle has been opened.""" + return self._file_handle.handle is not None + + def open(self) -> None: + """Open the file and register the hipfile handle.""" + if self.is_open: + return + self._file_handle.open() + + def close(self) -> None: + """Deregister the handle and close the file.""" + if not self.is_open: + return + self._file_handle.close() + + def __enter__(self) -> "CuFile": + self.open() + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.close() + + def __del__(self) -> None: + if self.is_open: + self.close() + + def read( + self, + dest: ctypes.c_void_p, + size: int, + file_offset: int = 0, + dev_offset: int = 0, + ) -> int: + """Read from the file into a GPU buffer. + + Args: + dest: Device pointer (``ctypes.c_void_p``) to read into. + size: Number of bytes to read. + file_offset: Byte offset in the file. + dev_offset: Byte offset in the device buffer. + + Returns: + Number of bytes actually read. + + Raises: + IOError: If the file is not open. + """ + if not self.is_open: + raise IOError("File is not open.") + buf = Buffer.from_ctypes_void_p(dest, size, flags=0) + return self._file_handle.read(buf, size, file_offset, dev_offset) + + def write( + self, + src: ctypes.c_void_p, + size: int, + file_offset: int = 0, + dev_offset: int = 0, + ) -> int: + """Write from a GPU buffer into the file. + + Args: + src: Device pointer (``ctypes.c_void_p``) to write from. + size: Number of bytes to write. + file_offset: Byte offset in the file. + dev_offset: Byte offset in the device buffer. + + Returns: + Number of bytes actually written. + + Raises: + IOError: If the file is not open. + """ + if not self.is_open: + raise IOError("File is not open.") + buf = Buffer.from_ctypes_void_p(src, size, flags=0) + return self._file_handle.write(buf, size, file_offset, dev_offset) + + +__all__ = ["CuFile", "CuFileDriver"] From 15f8afede411dd988234d07f518b5b84c94c13b9 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 21 Apr 2026 17:24:28 -0600 Subject: [PATCH 02/69] [Docs] Update hipFile repo URL Signed-off-by: Riley Dixon --- docs/source/kv_cache/storage_backends/gds.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/kv_cache/storage_backends/gds.rst b/docs/source/kv_cache/storage_backends/gds.rst index c71c332d455..2c1116296bb 100644 --- a/docs/source/kv_cache/storage_backends/gds.rst +++ b/docs/source/kv_cache/storage_backends/gds.rst @@ -123,7 +123,7 @@ Using AMD hipFile hipFile is alpha software and has been tested on limited hardware. For full installation details, see the - `hipFile install guide `__. + `hipFile install guide `__. **Prerequisites:** From 26acb0ebe29fdb1549cc21f5bd64ae802fde4feb Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Thu, 23 Apr 2026 15:34:56 -0600 Subject: [PATCH 03/69] [requirements] Add hipfile to rocm_core requirements. Signed-off-by: Riley Dixon --- requirements/rocm_core.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/rocm_core.txt b/requirements/rocm_core.txt index 21e9e66294a..0e552affc53 100644 --- a/requirements/rocm_core.txt +++ b/requirements/rocm_core.txt @@ -7,3 +7,4 @@ # per the "LMCache on ROCm / Without vLLM docker base image" docs. cupy-rocm-7-0 +hipfile From cf1eaf38e68a92d5c01b454659453d9f3d8205b5 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Fri, 1 May 2026 16:18:08 -0600 Subject: [PATCH 04/69] [hipFile/CI] Add explicit requirements install step for ROCm dependencies `uv pip install -e . --no-build-isolation` will not download and install missing build dependencies. hipFile uses `scikit_build_core` as the build backend. Since hipFile is currently under active development, the hipFile Python bindings are not tied to a particular release and are only published in an sdist format. As such, the hipFile bindings need to be built at install time. Signed-off-by: Riley Dixon --- .buildkite/pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 9ed51852969..6fe481522ce 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -25,6 +25,7 @@ steps: export CXX=hipcc export BUILD_WITH_HIP=1 uv pip install torch torchvision --index-url https://download.pytorch.org/whl/rocm7.0 + uv pip install -r requirements/rocm_core.txt fi uv pip install -r requirements/common.txt From d74c43a829354722391446419909a9a96fb743c7 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Mon, 4 May 2026 16:14:19 -0600 Subject: [PATCH 05/69] [hipFile] Add missing typing information. Signed-off-by: Riley Dixon --- lmcache/v1/storage_backend/hipfile_shim.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/lmcache/v1/storage_backend/hipfile_shim.py b/lmcache/v1/storage_backend/hipfile_shim.py index 508eeaca8c8..f5550faa031 100644 --- a/lmcache/v1/storage_backend/hipfile_shim.py +++ b/lmcache/v1/storage_backend/hipfile_shim.py @@ -9,6 +9,9 @@ # Standard import ctypes import os +from collections.abc import Callable +from types import TracebackType +from typing import Any # Third Party from hipfile import Buffer, Driver, FileHandle @@ -16,7 +19,7 @@ def _os_flags(mode: str) -> int: """Translate a Python-style mode string to ``os.open`` flags.""" - flags_map = { + flags_map: dict[str, int] = { "r": os.O_RDONLY, "r+": os.O_RDWR, "w": os.O_CREAT | os.O_WRONLY | os.O_TRUNC, @@ -27,11 +30,11 @@ def _os_flags(mode: str) -> int: return flags_map[mode] -def _singleton(cls): +def _singleton(cls: type) -> Callable: """Decorator that turns *cls* into a singleton.""" - _instances: dict = {} + _instances: dict[type, object] = {} - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: if cls not in _instances: _instances[cls] = cls(*args, **kwargs) return _instances[cls] @@ -96,7 +99,12 @@ def __enter__(self) -> "CuFile": self.open() return self - def __exit__(self, exc_type, exc_val, exc_tb) -> None: + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: self.close() def __del__(self) -> None: From bb6a8dc8476bd3c2a00d23f4e613d310c437950d Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Tue, 5 May 2026 11:29:01 -0600 Subject: [PATCH 06/69] [hipFile/CI] Revert adding hipFile as a hard dependency The hipFile Python bindings require that the hipFile C library & header is available on the system. Unfortunately at this time the hipFile C library is still in early access and does not have a package published on the ROCm package repo. This presents a challenge for how to fetch & install the hipFile C library in the LMCache CI. For now, lets leave hipFile as a soft-dependency but I will follow-up and add it back once I solve this early-release distribution problem. Signed-off-by: Riley Dixon --- requirements/rocm_core.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/rocm_core.txt b/requirements/rocm_core.txt index 0e552affc53..21e9e66294a 100644 --- a/requirements/rocm_core.txt +++ b/requirements/rocm_core.txt @@ -7,4 +7,3 @@ # per the "LMCache on ROCm / Without vLLM docker base image" docs. cupy-rocm-7-0 -hipfile From b741629b447435a6a7edc36c6b4fbbcabd68fd03 Mon Sep 17 00:00:00 2001 From: Riley Dixon Date: Wed, 6 May 2026 15:51:37 -0600 Subject: [PATCH 07/69] [hipFile] Sort imports Signed-off-by: Riley Dixon --- lmcache/v1/storage_backend/hipfile_shim.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lmcache/v1/storage_backend/hipfile_shim.py b/lmcache/v1/storage_backend/hipfile_shim.py index f5550faa031..dbc992d8205 100644 --- a/lmcache/v1/storage_backend/hipfile_shim.py +++ b/lmcache/v1/storage_backend/hipfile_shim.py @@ -7,11 +7,11 @@ """ # Standard -import ctypes -import os from collections.abc import Callable from types import TracebackType from typing import Any +import ctypes +import os # Third Party from hipfile import Buffer, Driver, FileHandle From 63ba53995a74d707955f5e7127447e506651a1a0 Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Thu, 14 May 2026 15:00:54 -0700 Subject: [PATCH 08/69] [CLI] Fix slim install imports and smoke-test wheel in CI (#3279) Signed-off-by: deng451e <838677410@qq.com> --- .github/workflows/build_cli_artifacts.yml | 28 ++++++++++++++ lmcache/cli/commands/bench/__init__.py | 47 +++++++++++++++++------ lmcache/cli/commands/server.py | 10 ++--- requirements/cli.txt | 1 + 4 files changed, 70 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_cli_artifacts.yml b/.github/workflows/build_cli_artifacts.yml index a43ba698579..a1e522ec47a 100644 --- a/.github/workflows/build_cli_artifacts.yml +++ b/.github/workflows/build_cli_artifacts.yml @@ -5,6 +5,15 @@ name: Build CLI Artifacts on: workflow_call: + pull_request: + branches: + - dev + - "release-**" + paths: + - 'lmcache/cli/**.py' + - 'pyproject_cli.toml' + - 'requirements/cli.txt' + - '.github/workflows/build_cli_artifacts.yml' env: LC_ALL: en_US.UTF-8 @@ -13,6 +22,11 @@ defaults: run: shell: bash +# Cancel stale runs per ref; PR and dev-push refs differ, so workflow_call is unaffected. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: contents: read @@ -64,9 +78,23 @@ jobs: - name: Smoke-test CLI wheel (no torch) run: | pip install dist/lmcache_cli-*.whl + + # CLI wheel must be pure-Python. + if unzip -l dist/lmcache_cli-*.whl | grep -E '\.(so|pyd)$'; then + echo "::error::CLI wheel contains compiled extensions; should be pure-Python" + exit 1 + fi + python -c "import lmcache" lmcache --help + # Slim-supported subcommands only (server is full-only). + for cmd in mock kvcache describe ping query bench tool trace; do + echo "::group::lmcache $cmd --help" + lmcache "$cmd" --help + echo "::endgroup::" + done + - name: Upload CLI release artifacts to GHA uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: diff --git a/lmcache/cli/commands/bench/__init__.py b/lmcache/cli/commands/bench/__init__.py index 920ab4f670b..79fb1429b15 100644 --- a/lmcache/cli/commands/bench/__init__.py +++ b/lmcache/cli/commands/bench/__init__.py @@ -26,9 +26,17 @@ StatsCollector, ) from lmcache.cli.commands.bench.engine_bench.workloads import create_workload -from lmcache.cli.commands.test_cache import TestCacheCommand from lmcache.logging import init_logger +# Gated for slim install — TestCacheCommand pulls torch/MP runtime. +_TEST_CACHE_IMPORT_ERROR: ImportError | None = None +try: + # First Party + from lmcache.cli.commands.test_cache import TestCacheCommand +except ImportError as _exc: + _TEST_CACHE_IMPORT_ERROR = _exc + TestCacheCommand = None # type: ignore[assignment,misc] + logger = init_logger(__name__) @@ -37,10 +45,10 @@ class BenchCommand(BaseCommand): def __init__(self) -> None: super().__init__() - # Stateless delegate for the ``kvcache`` sub-target. - # Cached once to avoid redundant instantiation across - # ``help()``, ``add_arguments()`` and ``execute()``. - self._kvcache_delegate = TestCacheCommand() + # None on slim install; _register_kvcache registers a stub instead. + self._kvcache_delegate = ( + TestCacheCommand() if TestCacheCommand is not None else None + ) def name(self) -> str: return "bench" @@ -329,13 +337,20 @@ def _register_kvcache( self, subparsers: argparse._SubParsersAction, ) -> None: - """Register ``lmcache bench kvcache`` subcommand. - - Arguments and execution are delegated to - :class:`~lmcache.cli.commands.test_cache.TestCacheCommand` so - the per-chunk store/retrieve/checksum logic lives in a single - place. + """Register ``lmcache bench kvcache``. Delegates to + :class:`TestCacheCommand`, or registers a stub on slim install. """ + if self._kvcache_delegate is None: + subparsers.add_parser( + "kvcache", + help="(requires full lmcache install)", + description=( + "End-to-end sanity test for the LMCache MP cache server. " + "Requires the full `lmcache` package; not available in " + "the `lmcache-cli` install." + ), + ).set_defaults(func=self.execute) + return parser = subparsers.add_parser( "kvcache", help=self._kvcache_delegate.help(), @@ -350,6 +365,16 @@ def _register_kvcache( def _bench_kvcache(self, args: argparse.Namespace) -> None: """Dispatch ``lmcache bench kvcache`` to ``TestCacheCommand``.""" + if self._kvcache_delegate is None: + print( + "ERROR: `lmcache bench kvcache` needs the full LMCache " + "package, but only the `lmcache-cli` shell is installed.\n" + " Install the full package with `pip install lmcache` " + "and try again.\n" + f" Original import error: {_TEST_CACHE_IMPORT_ERROR}", + file=sys.stderr, + ) + sys.exit(1) self._kvcache_delegate.execute(args) def execute(self, args: argparse.Namespace) -> None: diff --git a/lmcache/cli/commands/server.py b/lmcache/cli/commands/server.py index e719b65ee6c..e51ba7c415d 100644 --- a/lmcache/cli/commands/server.py +++ b/lmcache/cli/commands/server.py @@ -47,6 +47,11 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: add_http_frontend_args, add_mp_server_args, ) + + add_mp_server_args(parser) + add_storage_manager_args(parser) + add_http_frontend_args(parser) + add_observability_args(parser) except ImportError as e: print( f"Failed to import server dependencies: {e}. " @@ -54,11 +59,6 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: ) return - add_mp_server_args(parser) - add_storage_manager_args(parser) - add_http_frontend_args(parser) - add_observability_args(parser) - def execute(self, args: argparse.Namespace) -> None: """Parse CLI arguments into config objects and launch the HTTP server. diff --git a/requirements/cli.txt b/requirements/cli.txt index f2e322cfe02..23762b0791b 100644 --- a/requirements/cli.txt +++ b/requirements/cli.txt @@ -1,2 +1,3 @@ matplotlib openai +sortedcontainers From 80813a14c35c5716798ac311b90483681f7942fb Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Thu, 14 May 2026 15:04:45 -0700 Subject: [PATCH 09/69] [CI] swap default to cu13 wheels and cu12.9 secondary (#3275) Signed-off-by: deng451e <838677410@qq.com> --- ...rtifacts.yml => build_cu129_artifacts.yml} | 32 +++---- .github/workflows/nightly_build.yml | 83 ++++++++-------- .github/workflows/pr_full_build.yml | 10 +- .github/workflows/publish.yml | 95 ++++++++++--------- docker/Dockerfile | 40 +++++--- docker/Dockerfile.standalone | 7 +- docs/source/getting_started/installation.rst | 59 ++++++++---- pyproject.toml | 14 +-- requirements/cuda.txt | 6 +- requirements/cuda12_core.txt | 6 ++ requirements/cuda13_core.txt | 6 ++ requirements/cuda_core.txt | 9 -- setup.py | 13 ++- 13 files changed, 216 insertions(+), 164 deletions(-) rename .github/workflows/{build_cu130_artifacts.yml => build_cu129_artifacts.yml} (78%) create mode 100644 requirements/cuda12_core.txt create mode 100644 requirements/cuda13_core.txt delete mode 100644 requirements/cuda_core.txt diff --git a/.github/workflows/build_cu130_artifacts.yml b/.github/workflows/build_cu129_artifacts.yml similarity index 78% rename from .github/workflows/build_cu130_artifacts.yml rename to .github/workflows/build_cu129_artifacts.yml index b704ce27812..8c6caf3d668 100644 --- a/.github/workflows/build_cu130_artifacts.yml +++ b/.github/workflows/build_cu129_artifacts.yml @@ -1,10 +1,10 @@ -name: Build cu130 Artifacts +name: Build cu129 Artifacts -# Reusable workflow that builds the cu130 CUDA 13.0 wheel via cibuildwheel -# against the pytorch manylinux cu13 base image. -# Uploaded as the `release-cu130-artifacts` GHA artifact. -# The cu13 wheel carries no local version suffix and is published to a -# dedicated v{tag}-cu13 GitHub Release (not PyPI). +# Reusable workflow that builds the cu129 CUDA 12.9 wheel via cibuildwheel +# against the pytorch manylinux cu12.9 base image. +# Uploaded as the `release-cu129-artifacts` GHA artifact. +# The cu12.9 wheel carries no local version suffix and is published to a +# dedicated v{tag}-cu12 GitHub Release (not PyPI). on: workflow_call: @@ -20,8 +20,8 @@ permissions: contents: read jobs: - build-cu130-artifacts: - name: Build cu130 artifacts + build-cu129-artifacts: + name: Build cu129 artifacts runs-on: ubuntu-latest steps: - name: "Harden Runner" @@ -84,15 +84,15 @@ jobs: run: | rm -rf dist/ - - name: Get base version for cu13 wheel + - name: Get base version for cu12.9 wheel run: | BASE=$(git tag --list 'v[0-9]*' | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1 | sed 's/^v//' || echo "0.0.0.dev") - echo "CU130_VERSION=${BASE}" >> "$GITHUB_ENV" + echo "CU129_VERSION=${BASE}" >> "$GITHUB_ENV" - - name: Build cu130 CUDA wheels with cibuildwheel + - name: Build cu12.9 CUDA wheels with cibuildwheel env: - CIBW_MANYLINUX_X86_64_IMAGE: docker.io/pytorch/manylinux2_28-builder:cuda13.0 - CIBW_ENVIRONMENT: 'TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0;10.0;12.0" ENABLE_CXX11_ABI=1 SETUPTOOLS_SCM_PRETEND_VERSION=${{ env.CU130_VERSION }} PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cu130' + CIBW_MANYLINUX_X86_64_IMAGE: docker.io/pytorch/manylinux2_28-builder:cuda12.9 + CIBW_ENVIRONMENT: 'TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0;8.6;8.9;9.0;10.0" ENABLE_CXX11_ABI=1 LMCACHE_CUDA_MAJOR=12 SETUPTOOLS_SCM_PRETEND_VERSION=${{ env.CU129_VERSION }} PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cu129' CIBW_REPAIR_WHEEL_COMMAND_LINUX: >- auditwheel repair --plat manylinux_2_28_x86_64 @@ -102,14 +102,14 @@ jobs: --exclude libtorch_cpu.so --exclude libc10.so --exclude libc10_cuda.so - --exclude libcudart.so.13 + --exclude libcudart.so.12 -w {dest_dir} {wheel} CIBW_TEST_COMMAND: "python -c 'import lmcache.c_ops'" run: | python -m cibuildwheel --output-dir dist - - name: Upload cu130 release artifacts to GHA + - name: Upload cu12.9 release artifacts to GHA uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: - name: release-cu130-artifacts + name: release-cu129-artifacts path: dist/ diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index 6e8b5704fdd..a2b5178b424 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -66,36 +66,35 @@ jobs: pip install cibuildwheel - - name: Build cu12.9 nightly wheel + - name: Build cu13.0 nightly wheel env: - CIBW_ENVIRONMENT: 'TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0;8.6;8.9;9.0;10.0;12.0" ENABLE_CXX11_ABI=1' CIBW_TEST_COMMAND: "python -c 'import lmcache.c_ops'" run: | - python -m cibuildwheel --output-dir dist/cu129 + python -m cibuildwheel --output-dir dist/cu130 - - name: Publish cu12.9 wheels to rolling nightly release + - name: Publish cu13.0 wheels to rolling nightly release env: GITHUB_TOKEN: ${{ github.token }} run: | gh release delete nightly --yes --cleanup-tag --repo ${{ github.repository }} 2>/dev/null || true gh release create nightly \ --repo ${{ github.repository }} \ - --title "Nightly $(date +'%Y-%m-%d') · CUDA 12.9" \ - --notes "Nightly CUDA 12.9 wheels built from \`dev\` on $(date +'%Y-%m-%d'). + --title "Nightly $(date +'%Y-%m-%d') · CUDA 13.0" \ + --notes "Nightly CUDA 13.0 wheels built from \`dev\` on $(date +'%Y-%m-%d'). \`\`\` uv pip install lmcache --pre \\ - --extra-index-url https://download.pytorch.org/whl/cu129 \\ + --extra-index-url https://download.pytorch.org/whl/cu130 \\ --find-links https://github.com/${{ github.repository }}/releases/expanded_assets/nightly \\ --index-strategy unsafe-best-match \`\`\`" \ --prerelease \ - dist/cu129/*.whl + dist/cu130/*.whl - - name: Build cu13.0 nightly wheel + - name: Build cu12.9 nightly wheel env: - CIBW_MANYLINUX_X86_64_IMAGE: docker.io/pytorch/manylinux2_28-builder:cuda13.0 - CIBW_ENVIRONMENT: 'TORCH_CUDA_ARCH_LIST="8.0;8.6;8.9;9.0;10.0;12.0" ENABLE_CXX11_ABI=1 PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cu130' + CIBW_MANYLINUX_X86_64_IMAGE: docker.io/pytorch/manylinux2_28-builder:cuda12.9 + CIBW_ENVIRONMENT: 'TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0;8.6;8.9;9.0;10.0" ENABLE_CXX11_ABI=1 LMCACHE_CUDA_MAJOR=12 PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cu129' CIBW_REPAIR_WHEEL_COMMAND_LINUX: >- auditwheel repair --plat manylinux_2_28_x86_64 @@ -105,30 +104,30 @@ jobs: --exclude libtorch_cpu.so --exclude libc10.so --exclude libc10_cuda.so - --exclude libcudart.so.13 + --exclude libcudart.so.12 -w {dest_dir} {wheel} CIBW_TEST_COMMAND: "python -c 'import lmcache.c_ops'" run: | - python -m cibuildwheel --output-dir dist/cu130 + python -m cibuildwheel --output-dir dist/cu129 - - name: Publish cu13.0 wheels to rolling nightly-cu13 release + - name: Publish cu12.9 wheels to rolling nightly-cu129 release env: GITHUB_TOKEN: ${{ github.token }} run: | - gh release delete nightly-cu13 --yes --cleanup-tag --repo ${{ github.repository }} 2>/dev/null || true - gh release create nightly-cu13 \ + gh release delete nightly-cu129 --yes --cleanup-tag --repo ${{ github.repository }} 2>/dev/null || true + gh release create nightly-cu129 \ --repo ${{ github.repository }} \ - --title "Nightly $(date +'%Y-%m-%d') · CUDA 13.0" \ - --notes "Nightly CUDA 13.0 wheels built from \`dev\` on $(date +'%Y-%m-%d'). + --title "Nightly $(date +'%Y-%m-%d') · CUDA 12.9" \ + --notes "Nightly CUDA 12.9 wheels built from \`dev\` on $(date +'%Y-%m-%d'). \`\`\` uv pip install lmcache --pre \\ - --extra-index-url https://download.pytorch.org/whl/cu130 \\ - --find-links https://github.com/${{ github.repository }}/releases/expanded_assets/nightly-cu13 \\ + --extra-index-url https://download.pytorch.org/whl/cu129 \\ + --find-links https://github.com/${{ github.repository }}/releases/expanded_assets/nightly-cu129 \\ --index-strategy unsafe-best-match \`\`\`" \ --prerelease \ - dist/cu130/*.whl + dist/cu129/*.whl nightly-build: name: Build image @@ -191,12 +190,14 @@ jobs: run: | echo "NOW=$(date +'%Y-%m-%d')" >> "$GITHUB_ENV" - - name: Build lmcache/vllm-openai container image + - name: Build lmcache/vllm-openai container image (cu13 nightly) run: | docker system prune -f --all docker builder prune -af docker build \ - --build-arg CUDA_VERSION=12.9 --build-arg UBUNTU_VERSION=24.04 \ + --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.09-cuda13.0-devel-ubuntu24.04 \ + --build-arg CUDA_VERSION=13.0 \ + --build-arg torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' \ --target image-build \ --tag lmcache/vllm-openai:latest-nightly --tag lmcache/vllm-openai:nightly-${{ env.NOW }} \ --file docker/Dockerfile . @@ -206,12 +207,14 @@ jobs: docker push lmcache/vllm-openai:latest-nightly docker push lmcache/vllm-openai:nightly-${{ env.NOW }} - - name: Build lmcache/standalone container image + - name: Build lmcache/standalone container image (cu13 nightly) run: | docker system prune -f --all docker builder prune -af docker build \ - --build-arg CUDA_VERSION=12.9 --build-arg UBUNTU_VERSION=24.04 \ + --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.09-cuda13.0-devel-ubuntu24.04 \ + --build-arg CUDA_VERSION=13.0 \ + --build-arg torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' \ --target lmcache-final \ --tag lmcache/standalone:nightly --tag lmcache/standalone:nightly-${{ env.NOW }} \ --file docker/Dockerfile.standalone . @@ -221,36 +224,34 @@ jobs: docker push lmcache/standalone:nightly docker push lmcache/standalone:nightly-${{ env.NOW }} - - name: Build lmcache/vllm-openai container image (cu13 nightly) + - name: Build lmcache/vllm-openai container image (cu12.9 nightly) run: | docker system prune -f --all docker builder prune -af docker build \ - --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.09-cuda13.0-devel-ubuntu24.04 \ - --build-arg CUDA_VERSION=13.0 \ - --build-arg torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' \ + --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.05-cuda12.9-devel-ubuntu24.04 \ + --build-arg CUDA_VERSION=12.9 --build-arg UBUNTU_VERSION=24.04 \ --target image-build \ - --tag lmcache/vllm-openai:latest-nightly-cu13 --tag lmcache/vllm-openai:nightly-${{ env.NOW }}-cu13 \ + --tag lmcache/vllm-openai:latest-nightly-cu129 --tag lmcache/vllm-openai:nightly-${{ env.NOW }}-cu129 \ --file docker/Dockerfile . - - name: Push lmcache/vllm-openai cu13 nightly image to DockerHub + - name: Push lmcache/vllm-openai cu12.9 nightly image to DockerHub run: | - docker push lmcache/vllm-openai:latest-nightly-cu13 - docker push lmcache/vllm-openai:nightly-${{ env.NOW }}-cu13 + docker push lmcache/vllm-openai:latest-nightly-cu129 + docker push lmcache/vllm-openai:nightly-${{ env.NOW }}-cu129 - - name: Build lmcache/standalone container image (cu13 nightly) + - name: Build lmcache/standalone container image (cu12.9 nightly) run: | docker system prune -f --all docker builder prune -af docker build \ - --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.09-cuda13.0-devel-ubuntu24.04 \ - --build-arg CUDA_VERSION=13.0 \ - --build-arg torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' \ + --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.05-cuda12.9-devel-ubuntu24.04 \ + --build-arg CUDA_VERSION=12.9 --build-arg UBUNTU_VERSION=24.04 \ --target lmcache-final \ - --tag lmcache/standalone:nightly-cu13 --tag lmcache/standalone:nightly-${{ env.NOW }}-cu13 \ + --tag lmcache/standalone:nightly-cu129 --tag lmcache/standalone:nightly-${{ env.NOW }}-cu129 \ --file docker/Dockerfile.standalone . - - name: Push lmcache/standalone cu13 nightly image to DockerHub + - name: Push lmcache/standalone cu12.9 nightly image to DockerHub run: | - docker push lmcache/standalone:nightly-cu13 - docker push lmcache/standalone:nightly-${{ env.NOW }}-cu13 + docker push lmcache/standalone:nightly-cu129 + docker push lmcache/standalone:nightly-${{ env.NOW }}-cu129 diff --git a/.github/workflows/pr_full_build.yml b/.github/workflows/pr_full_build.yml index f229d130d7f..cdba6434ba7 100644 --- a/.github/workflows/pr_full_build.yml +++ b/.github/workflows/pr_full_build.yml @@ -1,6 +1,6 @@ name: PR Full Build -# Build all release artifacts (sdist + CUDA wheel, CLI wheel, cu130 wheel) +# Build all release artifacts (sdist + CUDA wheel, CLI wheel, cu129 wheel) # on PRs carrying the `full` label. The `full` label is added automatically # by automerge-labeler.yml when auto-merge is enabled on a code-change PR, # so this acts as a pre-merge sanity check that wheels still build. @@ -38,10 +38,10 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'full') uses: ./.github/workflows/build_cli_artifacts.yml - build-cu130: - name: Build cu130 + build-cu129: + name: Build cu129 if: contains(github.event.pull_request.labels.*.name, 'full') - uses: ./.github/workflows/build_cu130_artifacts.yml + uses: ./.github/workflows/build_cu129_artifacts.yml # Fork PRs won't expose DOCKERHUB_TOKEN even with `inherit`; the login - # step in build_cu130_artifacts.yml guards on the token being non-empty. + # step in build_cu129_artifacts.yml guards on the token being non-empty. secrets: inherit diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 57c354e5a4e..f890f213f5a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -56,11 +56,11 @@ jobs: - '.github/workflows/publish.yml' - '.github/workflows/build_main_artifacts.yml' - '.github/workflows/build_cli_artifacts.yml' - - '.github/workflows/build_cu130_artifacts.yml' + - '.github/workflows/build_cu129_artifacts.yml' - '!operator/**' # Build each artifact group in its own reusable workflow so that a - # failure in one (e.g. cu130) does not block publishing of the others. + # failure in one (e.g. cu129) does not block publishing of the others. build-main: needs: changes @@ -74,12 +74,12 @@ jobs: name: Build CLI uses: ./.github/workflows/build_cli_artifacts.yml - build-cu130: + build-cu129: needs: changes if: needs.changes.outputs.lmcache == 'true' - name: Build cu130 - uses: ./.github/workflows/build_cu130_artifacts.yml - # DockerHub creds needed for the cu130 manylinux image pull; + name: Build cu129 + uses: ./.github/workflows/build_cu129_artifacts.yml + # DockerHub creds needed for the cu12.9 manylinux image pull; # reusable workflows do not auto-inherit secrets. secrets: inherit @@ -273,9 +273,9 @@ jobs: with: verbose: true - # Publish cu130 wheel to GitHub Release (not PyPI) when a release is published - publish-cu130-github-release: - name: Publish cu130 wheel to GitHub Release + # Publish cu12.9 wheel to GitHub Release (not PyPI) when a release is published + publish-cu129-github-release: + name: Publish cu129 wheel to GitHub Release if: | !failure() && !cancelled() && needs.changes.outputs.lmcache == 'true' && @@ -283,7 +283,7 @@ jobs: permissions: contents: write runs-on: ubuntu-latest - needs: [changes, build-cu130, test, code-quality] + needs: [changes, build-cu129, test, code-quality] steps: - name: Harden Runner @@ -295,27 +295,27 @@ jobs: api.github.com:443 uploads.github.com:443 - - name: Fetch cu130 release artifacts + - name: Fetch cu12.9 release artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: - name: release-cu130-artifacts + name: release-cu129-artifacts path: dist - - name: Publish cu13.0 wheel to dedicated GitHub Release + - name: Publish cu12.9 wheel to dedicated GitHub Release env: GITHUB_TOKEN: ${{ github.token }} run: | - gh release create "${{ github.ref_name }}-cu13" \ + gh release create "${{ github.ref_name }}-cu129" \ --repo ${{ github.repository }} \ --latest=false \ - --title "Release ${{ github.ref_name }} · CUDA 13.0" \ - --notes "CUDA 13.0 wheel for LMCache ${{ github.ref_name }}. + --title "Release ${{ github.ref_name }} · CUDA 12.9" \ + --notes "CUDA 12.9 wheel for LMCache ${{ github.ref_name }}. \`\`\` VERSION=${{ github.ref_name }} uv pip install lmcache==${VERSION#v} \\ - --extra-index-url https://download.pytorch.org/whl/cu130 \\ - --find-links https://github.com/${{ github.repository }}/releases/expanded_assets/${VERSION}-cu13 \\ + --extra-index-url https://download.pytorch.org/whl/cu129 \\ + --find-links https://github.com/${{ github.repository }}/releases/expanded_assets/${VERSION}-cu129 \\ --index-strategy unsafe-best-match \`\`\`" \ dist/*.whl @@ -330,7 +330,7 @@ jobs: github.repository_owner == 'LMCache' && github.event.action == 'published' runs-on: ubuntu-latest - needs: [publish-pypi, publish-cu130-github-release] + needs: [publish-pypi, publish-cu129-github-release] steps: - name: "Harden Runner" @@ -381,11 +381,12 @@ jobs: run: | echo "LATEST_TAG=$(git tag --list 'v[0-9]*' | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1)" >> "$GITHUB_ENV" - - name: Build lmcache/vllm-openai container image (cu12.9) + - name: Build lmcache/vllm-openai container image (cu13) run: | docker build \ - --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.05-cuda12.9-devel-ubuntu24.04 \ - --build-arg CUDA_VERSION=12.9 \ + --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.09-cuda13.0-devel-ubuntu24.04 \ + --build-arg CUDA_VERSION=13.0 \ + --build-arg torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' \ --target image-release \ --tag lmcache/vllm-openai:latest --tag lmcache/vllm-openai:${{ env.LATEST_TAG }} \ --file docker/Dockerfile . @@ -416,61 +417,63 @@ jobs: docker system prune -f --all docker builder prune -af - - name: Build lmcache/standalone container image (cu12.9) + - name: Build lmcache/standalone container image (cu13) run: | docker build \ - --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.05-cuda12.9-devel-ubuntu24.04 \ - --build-arg CUDA_VERSION=12.9 \ + --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.09-cuda13.0-devel-ubuntu24.04 \ + --build-arg CUDA_VERSION=13.0 \ + --build-arg torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' \ --target lmcache-final \ --tag lmcache/standalone:latest --tag lmcache/standalone:${{ env.LATEST_TAG }} \ + --tag lmcache/standalone:latest-cu130 --tag lmcache/standalone:${{ env.LATEST_TAG }}-cu130 \ --file docker/Dockerfile.standalone . - name: Push lmcache/standalone container image to DockerHub run: | docker push lmcache/standalone:latest docker push lmcache/standalone:${{ env.LATEST_TAG }} + docker push lmcache/standalone:latest-cu130 + docker push lmcache/standalone:${{ env.LATEST_TAG }}-cu130 - name: Clean up Docker layers and cache run: | docker system prune -f --all docker builder prune -af - - name: Build lmcache/vllm-openai container image (cu13) - if: needs.publish-cu130-github-release.result == 'success' + - name: Build lmcache/vllm-openai container image (cu12.9) + if: needs.publish-cu129-github-release.result == 'success' run: | docker build \ - --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.09-cuda13.0-devel-ubuntu24.04 \ - --build-arg CUDA_VERSION=13.0 \ - --build-arg torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' \ + --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.05-cuda12.9-devel-ubuntu24.04 \ + --build-arg CUDA_VERSION=12.9 \ --build-arg LMCACHE_VERSION=${{ env.LATEST_TAG }} \ - --target image-release-cu13 \ - --tag lmcache/vllm-openai:latest-cu13 --tag lmcache/vllm-openai:${{ env.LATEST_TAG }}-cu13 \ + --target image-release-cu129 \ + --tag lmcache/vllm-openai:latest-cu129 --tag lmcache/vllm-openai:${{ env.LATEST_TAG }}-cu129 \ --file docker/Dockerfile . - - name: Push lmcache/vllm-openai cu13 container image to DockerHub - if: needs.publish-cu130-github-release.result == 'success' + - name: Push lmcache/vllm-openai cu129 container image to DockerHub + if: needs.publish-cu129-github-release.result == 'success' run: | - docker push lmcache/vllm-openai:latest-cu13 - docker push lmcache/vllm-openai:${{ env.LATEST_TAG }}-cu13 + docker push lmcache/vllm-openai:latest-cu129 + docker push lmcache/vllm-openai:${{ env.LATEST_TAG }}-cu129 - name: Clean up Docker layers and cache run: | docker system prune -f --all docker builder prune -af - - name: Build lmcache/standalone container image (cu13) - if: needs.publish-cu130-github-release.result == 'success' + - name: Build lmcache/standalone container image (cu12.9) + if: needs.publish-cu129-github-release.result == 'success' run: | docker build \ - --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.09-cuda13.0-devel-ubuntu24.04 \ - --build-arg CUDA_VERSION=13.0 \ - --build-arg torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' \ + --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.05-cuda12.9-devel-ubuntu24.04 \ + --build-arg CUDA_VERSION=12.9 \ --target lmcache-final \ - --tag lmcache/standalone:latest-cu13 --tag lmcache/standalone:${{ env.LATEST_TAG }}-cu13 \ + --tag lmcache/standalone:latest-cu129 --tag lmcache/standalone:${{ env.LATEST_TAG }}-cu129 \ --file docker/Dockerfile.standalone . - - name: Push lmcache/standalone cu13 container image to DockerHub - if: needs.publish-cu130-github-release.result == 'success' + - name: Push lmcache/standalone cu129 container image to DockerHub + if: needs.publish-cu129-github-release.result == 'success' run: | - docker push lmcache/standalone:latest-cu13 - docker push lmcache/standalone:${{ env.LATEST_TAG }}-cu13 + docker push lmcache/standalone:latest-cu129 + docker push lmcache/standalone:${{ env.LATEST_TAG }}-cu129 diff --git a/docker/Dockerfile b/docker/Dockerfile index 6fc05041fab..3540d2ac8c6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,9 +6,9 @@ # docs/source/getting_started/installation.rst # docs/production/docker_deployment.rst -ARG CUDA_VERSION=12.9 +ARG CUDA_VERSION=13.0 ARG UBUNTU_VERSION=24.04 -ARG NGC_VERSION=25.05 +ARG NGC_VERSION=25.09 ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:${NGC_VERSION}-cuda${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} #################### BASE BUILD IMAGE #################### @@ -40,7 +40,7 @@ RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ WORKDIR /workspace # CUDA arch list used by torch -ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' +ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} #################### vLLM IMAGE & LMCache (Build) ########################## @@ -79,6 +79,7 @@ RUN --mount=type=cache,target=/root/.cache/ccache,id=ccache \ --mount=type=cache,target=/root/.cache/uv,id=uv-cache,sharing=locked \ . /opt/venv/bin/activate && \ CUDA_TAG=cu$(echo ${CUDA_VERSION} | tr -d '.') && \ + export LMCACHE_CUDA_MAJOR=$(echo ${CUDA_VERSION} | cut -d. -f1) && \ if [ "$VLLM_VERSION" = "nightly" ]; then \ uv pip install --prerelease=allow \ 'vllm[runai,tensorizer,flashinfer]' \ @@ -96,27 +97,36 @@ RUN --mount=type=cache,target=/root/.cache/ccache,id=ccache \ WORKDIR /workspace ENTRYPOINT ["/opt/venv/bin/vllm", "serve"] -#################### vLLM IMAGE & LMCache (Release, cu12) ####################### -# Integrate vLLM and LMCache stable releases, and expose vLLM OpenAI API +#################### vLLM IMAGE & LMCache (Release, cu13) ####################### +# Integrate vLLM and LMCache stable releases, and expose vLLM OpenAI API. +# The default lmcache wheel on PyPI is built against cu13; the cu13 torch +# index is hinted explicitly so vLLM also resolves to its cu13 build. FROM base AS image-release -# Install LMCache and vLLM stable releases. -# It is imperative that LMCache uses the same torch version as the -# vLLM stable release. +ARG CUDA_VERSION + RUN . /opt/venv/bin/activate && \ - uv pip install --prerelease=allow vllm[runai,tensorizer,flashinfer] && \ - uv pip install lmcache --verbose + CUDA_TAG=cu$(echo ${CUDA_VERSION} | tr -d '.') && \ + uv pip install --prerelease=allow \ + vllm[runai,tensorizer,flashinfer] \ + --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG} \ + --index-strategy unsafe-best-match && \ + uv pip install lmcache \ + --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG} \ + --index-strategy unsafe-best-match --verbose WORKDIR /workspace ENTRYPOINT ["/opt/venv/bin/vllm", "serve"] -#################### vLLM IMAGE & LMCache (Release, cu13) ####################### -# Installs stable vLLM with cu13 index and lmcache from the v{tag}-cu13 GitHub Release. +#################### vLLM IMAGE & LMCache (Release, cu129) ###################### +# Installs stable vLLM with cu129 index and lmcache from the v{tag}-cu129 GitHub Release. -FROM base AS image-release-cu13 +FROM base AS image-release-cu129 -ARG CUDA_VERSION +# Override the global CUDA_VERSION default (13.0) for this cu12.9 stage so local +# builds that omit --build-arg CUDA_VERSION still resolve the correct torch index. +ARG CUDA_VERSION=12.9 ARG LMCACHE_VERSION RUN . /opt/venv/bin/activate && \ @@ -128,7 +138,7 @@ RUN . /opt/venv/bin/activate && \ --index-strategy unsafe-best-match && \ uv pip install lmcache==${VER} \ --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG} \ - --find-links https://github.com/LMCache/LMCache/releases/expanded_assets/v${VER}-cu13 \ + --find-links https://github.com/LMCache/LMCache/releases/expanded_assets/v${VER}-cu129 \ --index-strategy unsafe-best-match --verbose WORKDIR /workspace diff --git a/docker/Dockerfile.standalone b/docker/Dockerfile.standalone index 8d1cb5f1413..5abbcb29963 100644 --- a/docker/Dockerfile.standalone +++ b/docker/Dockerfile.standalone @@ -1,9 +1,9 @@ # The LMCache Standalone Dockerfile is used to build a LMCache image # without vLLM integration. -ARG CUDA_VERSION=12.9 +ARG CUDA_VERSION=13.0 ARG UBUNTU_VERSION=24.04 -ARG NGC_VERSION=25.05 +ARG NGC_VERSION=25.09 ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:${NGC_VERSION}-cuda${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} #################### BASE BUILD IMAGE #################### @@ -42,7 +42,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install ray nvidia-ml-py # CUDA arch list used by torch -ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' +ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} #################### LMCache Build ########################## @@ -80,6 +80,7 @@ WORKDIR /workspace/LMCache RUN --mount=type=cache,target=/root/.cache/ccache,id=ccache \ --mount=type=cache,target=/root/.cache/uv,id=uv-cache,sharing=locked \ . /opt/venv/bin/activate && \ + export LMCACHE_CUDA_MAJOR=$(echo ${CUDA_VERSION} | cut -d. -f1) && \ python3 -c 'import torch; print("TORCH=", torch.__version__)' && \ python3 setup.py bdist_wheel --dist-dir=dist_lmcache diff --git a/docs/source/getting_started/installation.rst b/docs/source/getting_started/installation.rst index 23e21758a34..f90a256b7e1 100644 --- a/docs/source/getting_started/installation.rst +++ b/docs/source/getting_started/installation.rst @@ -18,7 +18,7 @@ Install LMCache .. tab-set:: - .. tab-item:: CUDA 12.9 + .. tab-item:: CUDA 13.0 .. code-block:: bash @@ -31,9 +31,9 @@ Install LMCache You're all set! You can now start using LMCache. For hands-on guides and more usage examples, see the :ref:`quickstart_examples` section. - .. tab-item:: CUDA 13.0 + .. tab-item:: CUDA 12.9 - The CUDA 13.0 wheel is published to a dedicated + The CUDA 12.9 wheel is published to a dedicated `GitHub Release `__ rather than PyPI. .. code-block:: bash @@ -42,13 +42,13 @@ Install LMCache source .venv/bin/activate VERSION=0.4.3 # replace with target release uv pip install lmcache==${VERSION} \ - --extra-index-url https://download.pytorch.org/whl/cu130 \ - --find-links https://github.com/LMCache/LMCache/releases/expanded_assets/v${VERSION}-cu13 \ + --extra-index-url https://download.pytorch.org/whl/cu129 \ + --find-links https://github.com/LMCache/LMCache/releases/expanded_assets/v${VERSION}-cu129 \ --index-strategy unsafe-best-match .. note:: - ``--extra-index-url https://download.pytorch.org/whl/cu130`` ensures the CUDA 13.0 + ``--extra-index-url https://download.pytorch.org/whl/cu129`` ensures the CUDA 12.9 build of PyTorch is resolved. Without it, pip may select a mismatched CUDA variant. .. tab-item:: Nightly @@ -59,26 +59,26 @@ Install LMCache .. tab-set:: - .. tab-item:: CUDA 12.9 + .. tab-item:: CUDA 13.0 .. code-block:: bash uv venv --python 3.12 source .venv/bin/activate uv pip install lmcache --pre \ - --extra-index-url https://download.pytorch.org/whl/cu129 \ + --extra-index-url https://download.pytorch.org/whl/cu130 \ --find-links https://github.com/LMCache/LMCache/releases/expanded_assets/nightly \ --index-strategy unsafe-best-match - .. tab-item:: CUDA 13.0 + .. tab-item:: CUDA 12.9 .. code-block:: bash uv venv --python 3.12 source .venv/bin/activate uv pip install lmcache --pre \ - --extra-index-url https://download.pytorch.org/whl/cu130 \ - --find-links https://github.com/LMCache/LMCache/releases/expanded_assets/nightly-cu13 \ + --extra-index-url https://download.pytorch.org/whl/cu129 \ + --find-links https://github.com/LMCache/LMCache/releases/expanded_assets/nightly-cu129 \ --index-strategy unsafe-best-match .. tab-item:: From Source @@ -88,7 +88,7 @@ Install LMCache .. tab-set:: - .. tab-item:: CUDA + .. tab-item:: CUDA 13.0 .. code-block:: bash @@ -99,9 +99,30 @@ Install LMCache source .venv/bin/activate uv pip install -r requirements/build.txt - uv pip install vllm # pulls in required torch version + uv pip install vllm # pulls in required torch version (cu13) uv pip install -e . --no-build-isolation + .. tab-item:: CUDA 12.9 + + .. code-block:: bash + + git clone https://github.com/LMCache/LMCache.git + cd LMCache + + uv venv --python 3.12 + source .venv/bin/activate + + uv pip install -r requirements/build.txt + # Pin vLLM (and torch) to the cu12.9 wheel index so the local + # CUDA 12 toolchain matches what the extensions are built against. + uv pip install vllm \ + --extra-index-url https://download.pytorch.org/whl/cu129 \ + --index-strategy unsafe-best-match + # LMCACHE_CUDA_MAJOR=12 makes setup.py pick cupy-cuda12x / nixl-cu12 + # for install_requires instead of the cu13 defaults. + LMCACHE_CUDA_MAJOR=12 \ + uv pip install -e . --no-build-isolation + .. tab-item:: ROCm .. code-block:: bash @@ -133,33 +154,33 @@ Install LMCache .. tab-set:: - .. tab-item:: CUDA 12.9 + .. tab-item:: CUDA 13.0 .. code-block:: bash docker pull lmcache/vllm-openai - .. tab-item:: CUDA 13.0 + .. tab-item:: CUDA 12.9 .. code-block:: bash - docker pull lmcache/vllm-openai:latest-cu13 + docker pull lmcache/vllm-openai:latest-cu129 .. tab-item:: Nightly .. tab-set:: - .. tab-item:: CUDA 12.9 + .. tab-item:: CUDA 13.0 .. code-block:: bash docker pull lmcache/vllm-openai:latest-nightly - .. tab-item:: CUDA 13.0 + .. tab-item:: CUDA 12.9 .. code-block:: bash - docker pull lmcache/vllm-openai:latest-nightly-cu13 + docker pull lmcache/vllm-openai:latest-nightly-cu129 .. tab-item:: ROCm diff --git a/pyproject.toml b/pyproject.toml index 3cc2752356e..df343505fbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ issues = "https://github.com/LMCache/LMCache" version_file = "lmcache/_version.py" # do not include +gREV local version, required for Test PyPI upload local_scheme = "no-local-version" -# only treat vX.Y.Z-style tags as version anchors; ignores nightly/nightly-cu13 tags +# only treat vX.Y.Z-style tags as version anchors; ignores nightly/nightly-cu129 tags tag_regex = "^v?(?P\\d+\\.\\d+\\.\\d+.*)$" git_describe_command = ["git", "describe", "--tags", "--long", "--match", "v[0-9]*.[0-9]*", "--exclude", "v*-*"] @@ -159,18 +159,18 @@ build = "cp3*-manylinux_x86_64" # see https://developer.nvidia.com/cuda-gpus for compute capabilities # "CUDA-Enabled Datacenter Products" -# 7.0: V100 # 7.5: T4 # 8.0: A100, A30 # 8.6: A40, A10, A16, A2 # 8.9: L4, L40, L40S # 9.0: GH200, H200, H100 # 10.0: GB200, B200 -environment = {TORCH_CUDA_ARCH_LIST = "7.0;7.5;8.0;8.6;8.9;9.0;10.0", ENABLE_CXX11_ABI = "1"} +# 12.0: RTX 50-series (PTX) +# CUDA 13 dropped support for compute capability 7.0 (Volta). +environment = {TORCH_CUDA_ARCH_LIST = "7.5;8.0;8.6;8.9;9.0;10.0;12.0", ENABLE_CXX11_ABI = "1", PIP_EXTRA_INDEX_URL = "https://download.pytorch.org/whl/cu130"} -# Use the PyTorch manylinux image version '2_28' that contains CUDA 12.9 -# and torch 2.7 supports (https://pypi.org/project/torch/2.7.0/#files) -manylinux-x86_64-image = "docker.io/pytorch/manylinux2_28-builder:cuda12.9" +# Use the PyTorch manylinux image version '2_28' that contains CUDA 13.0 +manylinux-x86_64-image = "docker.io/pytorch/manylinux2_28-builder:cuda13.0" [tool.cibuildwheel.linux] repair-wheel-command = """ @@ -182,6 +182,6 @@ auditwheel repair \ --exclude libtorch_cpu.so \ --exclude libc10.so \ --exclude libc10_cuda.so \ - --exclude libcudart.so.12 \ + --exclude libcudart.so.13 \ -w {dest_dir} {wheel} """ diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 2adb8031e7f..8b6cf3ee72f 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -1,7 +1,9 @@ # Common project dependencies -r common.txt -# Vendor-specific runtime deps (cupy, nixl) baked into install_requires --r cuda_core.txt +# Vendor-specific runtime deps (cupy, nixl) baked into install_requires. +# Defaults to the cu13 list; the cu12.9 build path overrides via setup.py +# (LMCACHE_CUDA_MAJOR=12) and does not consume cuda.txt directly. +-r cuda13_core.txt # Dependencies for NVIDIA GPUs ray >= 2.9 diff --git a/requirements/cuda12_core.txt b/requirements/cuda12_core.txt new file mode 100644 index 00000000000..3d4047bb6dc --- /dev/null +++ b/requirements/cuda12_core.txt @@ -0,0 +1,6 @@ +# Vendor-specific runtime deps for CUDA 12.9 wheels. Selected by setup.py when +# LMCACHE_CUDA_MAJOR=12. The cu13 default lives in cuda13_core.txt. + +cupy-cuda12x +# nixl on PyPI is a meta-package that pulls nixl-cu12 (CUDA-only). +nixl; python_version < "3.13" diff --git a/requirements/cuda13_core.txt b/requirements/cuda13_core.txt new file mode 100644 index 00000000000..526149981ae --- /dev/null +++ b/requirements/cuda13_core.txt @@ -0,0 +1,6 @@ +# Vendor-specific runtime deps for CUDA 13 wheels. Selected by setup.py when +# LMCACHE_CUDA_MAJOR is unset or "13" (the default, matching the PyPI build). +# The cu12 analogue lives in cuda12_core.txt. + +cupy-cuda13x +nixl-cu13; python_version < "3.13" diff --git a/requirements/cuda_core.txt b/requirements/cuda_core.txt deleted file mode 100644 index d73178bcb08..00000000000 --- a/requirements/cuda_core.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Vendor-specific runtime deps baked into install_requires by setup.py -# when building for CUDA (i.e. BUILD_WITH_HIP is unset). -# Kept separate from cuda.txt so `pip install -e .` stays lightweight -# (no ray/xformers/torchvision) while Docker's `pip install -r cuda.txt` -# still pulls these through the -r chain. - -cupy-cuda12x -# nixl on PyPI is a meta-package that pulls nixl-cu12 (CUDA-only). -nixl; python_version < "3.13" diff --git a/setup.py b/setup.py index d70c4aa22e3..6ad52dafb21 100644 --- a/setup.py +++ b/setup.py @@ -309,7 +309,18 @@ def source_dist_extension() -> tuple[list, dict]: ext_modules, cmdclass = get_extension() install_requires = _read_requirements(ROOT_DIR / "requirements" / "common.txt") - core_file = "rocm_core.txt" if BUILD_WITH_HIP else "cuda_core.txt" + if BUILD_WITH_HIP: + core_file = "rocm_core.txt" + else: + # CUDA major selects between cu12 and cu13 vendor pins (cupy, nixl). + # Defaults to cu13 (the PyPI build); cu12.9 wheel builds set + # LMCACHE_CUDA_MAJOR=12 to pull cu12 wheels of those deps. + cuda_major = os.environ.get("LMCACHE_CUDA_MAJOR", "13") + if cuda_major not in ("12", "13"): + raise ValueError( + f"LMCACHE_CUDA_MAJOR must be '12' or '13', got '{cuda_major}'" + ) + core_file = f"cuda{cuda_major}_core.txt" install_requires += _read_requirements(ROOT_DIR / "requirements" / core_file) setup( From 8cfac668f84edaa72e58a35232aea428feea84da Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Thu, 14 May 2026 16:26:42 -0700 Subject: [PATCH 10/69] Revert "[CLI][CB] propagate CLI kvcache clear to CB fingerprint tabe" (#3276) Signed-off-by: deng451e <838677410@qq.com> --- lmcache/v1/multiprocess/blend_server_v2.py | 33 ----------------- tests/v1/multiprocess/test_blend_server_v2.py | 35 ------------------- 2 files changed, 68 deletions(-) diff --git a/lmcache/v1/multiprocess/blend_server_v2.py b/lmcache/v1/multiprocess/blend_server_v2.py index 957562272e7..b5bc6efc380 100644 --- a/lmcache/v1/multiprocess/blend_server_v2.py +++ b/lmcache/v1/multiprocess/blend_server_v2.py @@ -334,21 +334,6 @@ def has_chunk(self, token_hash: bytes) -> bool: with self._lock: return token_hash in self._token_hash_to_compact_id - def clear(self) -> int: - """Drop every registered chunk and reset compact-ID allocation. - - Returns: - Number of live (non-evicted) chunks removed. - """ - with self._lock: - num_live = len(self._token_hash_to_compact_id) - self._table_id.fill(-1) - self._compact_id_to_slot.fill(-1) - self._chunk_token_hash = [] - self._token_hash_to_start = {} - self._token_hash_to_compact_id = {} - return num_live - def _unique_token_coverage(results: list[CBMatchResult]) -> int: """Return the number of unique query tokens covered by a set of CBMatchResults. @@ -426,24 +411,6 @@ def cb_register_kv_cache( gpu_context.num_layers, ) - def clear(self) -> None: - """Clear the CB fingerprint table, then storage. - - Matcher is cleared first so any concurrent CB store that completes - between the two steps leaves a stale matcher entry (self-healed by - the lazy-eviction path in cb_lookup_pre_computed) rather than an - orphaned chunk in storage. - """ - num_dropped = self._token_range_matcher.clear() - super().clear() - if num_dropped: - self._event_bus.publish( - Event( - event_type=EventType.CB_CHUNKS_EVICTED, - metadata={"num_chunks": num_dropped}, - ) - ) - def cb_unregister_kv_cache(self, instance_id: int) -> None: """ Unregister the KV cache buffer for the given instance_id diff --git a/tests/v1/multiprocess/test_blend_server_v2.py b/tests/v1/multiprocess/test_blend_server_v2.py index 6ceb7bfa4cf..bdc91e68e17 100644 --- a/tests/v1/multiprocess/test_blend_server_v2.py +++ b/tests/v1/multiprocess/test_blend_server_v2.py @@ -348,41 +348,6 @@ def test_has_chunk_false_after_eviction(self): matcher.remove_chunks([token_hash]) assert matcher.has_chunk(token_hash) is False - def test_clear_returns_live_chunk_count_and_drops_matches(self): - """clear() drops every registered chunk and reports the live count.""" - matcher = BlendTokenRangeMatcher(chunk_size=4) - hash_a = ObjectKey.IntHash2Bytes(7001) - hash_b = ObjectKey.IntHash2Bytes(7002) - matcher.on_new_token_hashes([1, 2, 3, 4, 5, 6, 7, 8], [hash_a, hash_b]) - - # Evict one before clear() so the count reflects only live entries. - matcher.remove_chunks([hash_a]) - assert matcher.clear() == 1 - - assert matcher.match_sub_sequence([1, 2, 3, 4, 5, 6, 7, 8]) == [] - assert matcher.has_chunk(hash_a) is False - assert matcher.has_chunk(hash_b) is False - - def test_clear_then_register_works(self): - """After clear(), new registrations behave like a fresh matcher.""" - matcher = BlendTokenRangeMatcher(chunk_size=4) - hash_a = ObjectKey.IntHash2Bytes(7101) - hash_b = ObjectKey.IntHash2Bytes(7102) - - matcher.on_new_token_hashes([1, 2, 3, 4], [hash_a]) - matcher.clear() - - matcher.on_new_token_hashes([10, 20, 30, 40], [hash_b]) - assert matcher.match_sub_sequence([1, 2, 3, 4]) == [] - results = matcher.match_sub_sequence([10, 20, 30, 40]) - assert len(results) == 1 - assert results[0].hash == hash_b - - def test_clear_on_empty_matcher_returns_zero(self): - """clear() on a never-populated matcher is a no-op returning 0.""" - matcher = BlendTokenRangeMatcher(chunk_size=4) - assert matcher.clear() == 0 - class TestUniqueTokenCoverage: """Unit tests for _unique_token_coverage — the interval-merge helper that From 168bbb976a297cad788dc7b45c70c99d25a0baef Mon Sep 17 00:00:00 2001 From: Jason Goldschmidt <92800864+jgoldsch12@users.noreply.github.com> Date: Thu, 14 May 2026 19:33:17 -0400 Subject: [PATCH 11/69] fix: use unique device_id per OBJ register/deregister cycle (#2989) When nixl_async_put is enabled, the async PUT cleanup thread (deregister_memory) and the sync GET worker thread (register_memory + prepXfer) both use device_id = 0, 1, 2, ... for every call. NIXL's OBJ backend tracks registrations in a flat unordered_map keyed by devId with no reference counting. The PUT's deregister erases devId=0, which deletes the GET's registration, causing postXfer to fail with NIXL_ERR_INVALID_PARAM and crashing the EngineCore. Fix: replace the per-call sequential device_id (idx) with a monotonically increasing counter protected by a threading.Lock. Each register/deregister cycle now gets globally unique IDs, so concurrent operations never collide in devIdToObjKey_. Fixes https://github.com/LMCache/LMCache/issues/2983 Signed-off-by: Jason Goldschmidt --- .buildkite/pipeline.yml | 1 + .../storage_backend/nixl_storage_backend.py | 29 ++- tests/v1/test_device_id_race.py | 178 ++++++++++++++++++ 3 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 tests/v1/test_device_id_race.py diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2551f6aea2b..6d024a7d721 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -39,6 +39,7 @@ steps: --ignore=tests/disagg --ignore=tests/v1/test_pos_kernels.py \ --ignore=tests/v1/test_nixl_storage.py \ --ignore=tests/v1/test_nixl_batched_contains.py \ + --ignore=tests/v1/test_device_id_race.py \ --ignore=tests/skipped \ --ignore=tests/v1/storage_backend/test_eic.py diff --git a/lmcache/v1/storage_backend/nixl_storage_backend.py b/lmcache/v1/storage_backend/nixl_storage_backend.py index c4d1230519a..0583985cb89 100644 --- a/lmcache/v1/storage_backend/nixl_storage_backend.py +++ b/lmcache/v1/storage_backend/nixl_storage_backend.py @@ -1095,6 +1095,13 @@ def __init__( self.meta_fmt: Optional[MemoryFormat] = None self.init_chunk_meta(metadata) + # Monotonically increasing counter for OBJ device_id values. + # Each register/deregister cycle must use globally unique IDs to + # avoid a race where an async PUT deregister erases a concurrent + # GET registration in NIXL's devIdToObjKey_ map. + self._device_id_counter = 0 + self._device_id_lock = threading.Lock() + self.agent = NixlDynamicStorageAgent( self.memory_allocator, nixl_config.buffer_device, @@ -1109,6 +1116,22 @@ def set_presence_cache(self, cache: PresenceCache) -> None: if self.enable_presence_cache: self.key_presence_cache = cache + def _alloc_device_ids(self, count: int) -> list[int]: + """Allocate ``count`` globally unique OBJ device_id values. + + The NIXL OBJ backend indexes registrations by device_id in a flat + unordered_map with no reference counting. If two concurrent + operations (e.g. async PUT cleanup + sync GET) use the same + device_id sequence (0, 1, 2, ...), the PUT deregister can erase + the GET entry and cause ``prepXfer``/``postXfer`` to fail with + NIXL_ERR_INVALID_PARAM. Using a monotonic counter ensures every + register/deregister cycle gets its own ID range. + """ + with self._device_id_lock: + start = self._device_id_counter + self._device_id_counter += count + return list(range(start, start + count)) + def _cache_contains(self, chunk_hash: int) -> bool: if not self.enable_presence_cache or self.key_presence_cache is None: return False @@ -1210,9 +1233,10 @@ def storage_to_mem( storage_indices.append(idx) if self.agent.mem_type == "OBJ": + device_ids = self._alloc_device_ids(len(keys)) for idx in range(len(keys)): object_key = self._format_object_key(keys[idx]) - descs.append(NixlDesc(device_id=idx, meta_info=object_key)) + descs.append(NixlDesc(device_id=device_ids[idx], meta_info=object_key)) else: # Already validated in validate_nixl_backend raise ValueError(f"unexpected mem_type: {self.agent.mem_type}") @@ -1305,9 +1329,10 @@ async def mem_to_storage( descs = [] if self.agent.mem_type == "OBJ": + device_ids = self._alloc_device_ids(len(keys)) for idx in range(len(keys)): object_key = self._format_object_key(keys[idx]) - descs.append(NixlDesc(device_id=idx, meta_info=object_key)) + descs.append(NixlDesc(device_id=device_ids[idx], meta_info=object_key)) else: # Already validated in validate_nixl_backend raise ValueError(f"unexpected mem_type: {self.agent.mem_type}") diff --git a/tests/v1/test_device_id_race.py b/tests/v1/test_device_id_race.py new file mode 100644 index 00000000000..c6def16be8f --- /dev/null +++ b/tests/v1/test_device_id_race.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for NixlDynamicStorageBackend._alloc_device_ids. + +These tests verify the fix for the NIXL OBJ devIdToObjKey_ race condition +(https://github.com/LMCache/LMCache/issues/2983). When nixl_async_put is +enabled, async PUT cleanup and sync GET run concurrently. Both previously +used device_id = 0, 1, 2, ... for every call, causing the PUT's +deregister to erase the GET registration in NIXL's flat map. + +The fix uses a monotonically increasing counter so each register/deregister +cycle gets globally unique device_ids. These tests validate correctness +and thread safety of that counter without requiring NIXL or CUDA hardware. +""" + +# Standard +import threading + +# Third Party +import pytest + + +# --------------------------------------------------------------------------- +# Helper: build a minimal NixlDynamicStorageBackend with only the fields +# needed by _alloc_device_ids, bypassing __init__'s heavy dependencies. +# --------------------------------------------------------------------------- +def _make_stub_backend(): + """Return a NixlDynamicStorageBackend instance with __init__ bypassed.""" + # Import inside function so the test file can be collected even when + # nixl is not installed (the importorskip below handles the skip). + # First Party + from lmcache.v1.storage_backend.nixl_storage_backend import ( + NixlDynamicStorageBackend, + ) + + obj = object.__new__(NixlDynamicStorageBackend) + obj._device_id_counter = 0 + obj._device_id_lock = threading.Lock() + return obj + + +# Skip the entire module if nixl is not importable (mirrors existing tests) +pytest.importorskip("nixl", reason="nixl package is required for nixl tests") + + +# ---- Basic correctness ---------------------------------------------------- + + +class TestAllocDeviceIds: + """Tests for _alloc_device_ids correctness.""" + + def test_single_alloc_returns_sequential_range(self): + backend = _make_stub_backend() + ids = backend._alloc_device_ids(5) + assert ids == [0, 1, 2, 3, 4] + + def test_successive_allocs_are_non_overlapping(self): + backend = _make_stub_backend() + first = backend._alloc_device_ids(3) + second = backend._alloc_device_ids(4) + third = backend._alloc_device_ids(2) + assert first == [0, 1, 2] + assert second == [3, 4, 5, 6] + assert third == [7, 8] + + def test_alloc_zero_returns_empty(self): + backend = _make_stub_backend() + ids = backend._alloc_device_ids(0) + assert ids == [] + # Counter should not advance + assert backend._device_id_counter == 0 + + def test_alloc_one_returns_single_element(self): + backend = _make_stub_backend() + ids = backend._alloc_device_ids(1) + assert ids == [0] + assert backend._device_id_counter == 1 + + def test_counter_advances_correctly(self): + backend = _make_stub_backend() + backend._alloc_device_ids(10) + assert backend._device_id_counter == 10 + backend._alloc_device_ids(5) + assert backend._device_id_counter == 15 + + def test_all_ids_globally_unique_across_many_allocs(self): + backend = _make_stub_backend() + all_ids = [] + for n in [1, 3, 7, 2, 10, 5]: + all_ids.extend(backend._alloc_device_ids(n)) + assert len(all_ids) == len(set(all_ids)), "duplicate device_ids found" + assert all_ids == list(range(28)) + + +# ---- Thread safety --------------------------------------------------------- + + +class TestAllocDeviceIdsThreadSafety: + """Verify no duplicate IDs under concurrent access.""" + + def test_concurrent_allocs_produce_unique_ids(self): + """Simulate the race: multiple threads calling _alloc_device_ids + concurrently, as happens when async PUT cleanup and sync GET + overlap.""" + backend = _make_stub_backend() + num_threads = 16 + allocs_per_thread = 200 + batch_size = 5 # typical: one device_id per key in a batch + results: list[list[int]] = [[] for _ in range(num_threads)] + + barrier = threading.Barrier(num_threads) + + def worker(thread_idx): + barrier.wait() # maximize contention + for _ in range(allocs_per_thread): + ids = backend._alloc_device_ids(batch_size) + results[thread_idx].extend(ids) + + threads = [ + threading.Thread(target=worker, args=(i,)) for i in range(num_threads) + ] + for t in threads: + t.start() + for t in threads: + t.join() + + all_ids = [] + for r in results: + all_ids.extend(r) + + expected_total = num_threads * allocs_per_thread * batch_size + assert len(all_ids) == expected_total + assert len(set(all_ids)) == expected_total, ( + f"found {expected_total - len(set(all_ids))} duplicate device_ids " + f"under concurrent access" + ) + + def test_interleaved_put_get_simulation(self): + """Simulate interleaved PUT (async cleanup) and GET (sync worker) + operations to verify the fix prevents the exact race from #44.""" + backend = _make_stub_backend() + put_ids: list[list[int]] = [] + get_ids: list[list[int]] = [] + barrier = threading.Barrier(2) + + def put_worker(): + """Simulates async PUT: register → transfer → deregister.""" + barrier.wait() + for _ in range(100): + ids = backend._alloc_device_ids(3) + put_ids.append(ids) + + def get_worker(): + """Simulates sync GET: register → prepXfer → postXfer → deregister.""" + barrier.wait() + for _ in range(100): + ids = backend._alloc_device_ids(3) + get_ids.append(ids) + + t_put = threading.Thread(target=put_worker) + t_get = threading.Thread(target=get_worker) + t_put.start() + t_get.start() + t_put.join() + t_get.join() + + # Flatten and check uniqueness + all_put = [id_ for batch in put_ids for id_ in batch] + all_get = [id_ for batch in get_ids for id_ in batch] + + # No overlap between PUT and GET id ranges + overlap = set(all_put) & set(all_get) + assert len(overlap) == 0, ( + f"PUT and GET device_ids overlap: {sorted(list(overlap))[:10]}..." + ) + + # All IDs globally unique + combined = all_put + all_get + assert len(combined) == len(set(combined)) From 478a53cea6b865d4e4774729e31c563aca77b3d7 Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Fri, 15 May 2026 08:15:35 +0800 Subject: [PATCH 12/69] fix(hpu): implement device-specific initialize_kvcaches_ptr for HPU connector (#3287) * fix(hpu): implement device-specific initialize_kvcaches_ptr for HPU connector Signed-off-by: Tony Lin --- lmcache/v1/gpu_connector/hpu_connector.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lmcache/v1/gpu_connector/hpu_connector.py b/lmcache/v1/gpu_connector/hpu_connector.py index 124b1c6f8dd..03bb25dbc27 100644 --- a/lmcache/v1/gpu_connector/hpu_connector.py +++ b/lmcache/v1/gpu_connector/hpu_connector.py @@ -237,6 +237,11 @@ def get_shape(self, num_tokens: int) -> torch.Size: kv_size = 1 if self.use_mla else 2 return torch.Size([kv_size, self.num_layers, num_tokens, self.hidden_dim_size]) + def initialize_kvcaches_ptr(self, **kwargs) -> None: + """Initialize the kvcaches pointers if not already initialized.""" + if "kvcaches" in kwargs: + self.kvcaches = kwargs["kvcaches"] + def _validate_memory_format(self, memory_obj: MemoryObj) -> None: """Validate that the memory object has the expected format. @@ -261,10 +266,14 @@ def _validate_memory_format(self, memory_obj: MemoryObj) -> None: ) def _initialize_attributes(self, kv_caches: List[torch.Tensor]): - if self._attributes_initialized: + if self._attributes_initialized or not kv_caches: return - self.device = kv_caches[0].device + first = kv_caches[0] + if isinstance(first, torch.Tensor): + self.device = first.device + else: + self.device = first[0].device assert self.device.type == "hpu", "The device should be HPU." # HPU vLLM provides kv_caches as List[TensorTuple(k_tensor, v_tensor)], From f02ad538782171e219a0e5ec97fc6d7a694ea063 Mon Sep 17 00:00:00 2001 From: Jason Goldschmidt <92800864+jgoldsch12@users.noreply.github.com> Date: Thu, 14 May 2026 20:18:00 -0400 Subject: [PATCH 13/69] [CORE] Add nixl_endpoint_list for per-worker object-storage endpoint distribution (#2861) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add nixl_endpoint_list for per-worker object-storage endpoint distribution Allow TP workers to target different object-storage endpoints when using the NIXL OBJ backend. When nixl_endpoint_list is set in extra_config, each worker selects an endpoint round-robin by worker_id, overriding any nixl_backend_params.endpoint_override (with a logged warning). Also copies nixl_backend_params into a new dict before mutating it to avoid modifying the original config. Signed-off-by: Jason Goldschmidt * Use local_worker_id instead of worker_id is correct for multi-host deployments because: • worker_id is the global worker ID across all hosts (e.g., host 0 has workers 0-7, host 1 has workers 8-15) • local_worker_id is the worker ID within the local host (e.g., workers 0-7 on each host) • For endpoint distribution, each host should distribute its local workers across endpoints independently • This ensures proper load balancing in multi-host scenarios where each host may have its own set of endpoints Signed-off-by: Jason Goldschmidt --------- Signed-off-by: Jason Goldschmidt --- docs/source/api_reference/configurations.rst | 12 +- .../source/kv_cache/storage_backends/nixl.rst | 30 +++++ .../storage_backend/nixl_storage_backend.py | 30 ++++- tests/disagg/test_nixl_storage_backend.py | 120 +++++++++++++++++- tests/v1/test_nixl_storage.py | 48 ++++++- 5 files changed, 232 insertions(+), 8 deletions(-) diff --git a/docs/source/api_reference/configurations.rst b/docs/source/api_reference/configurations.rst index d624828da23..57549a05f57 100644 --- a/docs/source/api_reference/configurations.rst +++ b/docs/source/api_reference/configurations.rst @@ -334,9 +334,9 @@ Settings for using Nixl as a storage backend instead of disaggregated prefill. T extra_config: # enable_nixl_storage will disable disaggregated prefill mode. enable_nixl_storage: true - nixl_backend: "POSIX" # Options: "GDS", "GDS_MT", "POSIX", "HF3FS" + nixl_backend: "POSIX" # Options: "GDS", "GDS_MT", "POSIX", "HF3FS", "OBJ" nixl_path: "/path/to/storage/" - nixl_file_pool_size: 64 + nixl_pool_size: 64 .. list-table:: :header-rows: 1 @@ -347,11 +347,13 @@ Settings for using Nixl as a storage backend instead of disaggregated prefill. T * - enable_nixl_storage - Whether to enable Nixl storage backend. Values: true/false * - nixl_backend - - Storage backend type. Options: "GDS", "GDS_MT", "POSIX", "HF3FS" + - Storage backend type. Options: "GDS", "GDS_MT", "POSIX", "HF3FS", "OBJ" * - nixl_path - File system path for Nixl storage - * - nixl_file_pool_size - - Number of files in the storage pool + * - nixl_pool_size + - Number of files or objects in the storage pool + * - nixl_endpoint_list + - List of object-storage endpoint URLs for per-worker distribution. Overrides ``nixl_backend_params.endpoint_override`` when set. Additional Storage Configurations diff --git a/docs/source/kv_cache/storage_backends/nixl.rst b/docs/source/kv_cache/storage_backends/nixl.rst index 331e275c2b0..9f599ab1ef6 100644 --- a/docs/source/kv_cache/storage_backends/nixl.rst +++ b/docs/source/kv_cache/storage_backends/nixl.rst @@ -111,6 +111,36 @@ Example ``lmcache-config.yaml`` for AZURE_BLOB backend to offload using Azure Bl account_url: https://.blob.core.windows.net container_name: +Per-Worker Endpoint Distribution +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When using the OBJ backend with multiple tensor-parallel (TP) workers, you can +distribute workers across multiple object-storage endpoints by providing a list of +endpoints via ``nixl_endpoint_list``. Each worker selects an endpoint in +round-robin order based on its ``local_worker_id`` (the worker ID within its host). + +.. code-block:: yaml + + extra_config: + enable_nixl_storage: true + nixl_backend: OBJ + nixl_pool_size: 64 + nixl_path: /mnt/nixl/cache/ + nixl_endpoint_list: + - https://node-0.object-storage:9021 + - https://node-1.object-storage:9021 + - https://node-2.object-storage:9021 + nixl_backend_params: + access_key: + secret_key: + bucket: + region: + +.. note:: + + When ``nixl_endpoint_list`` is set, any ``endpoint_override`` value in + ``nixl_backend_params`` is ignored (a warning is logged). + Dynamic Mode ~~~~~~~~~~~~~ diff --git a/lmcache/v1/storage_backend/nixl_storage_backend.py b/lmcache/v1/storage_backend/nixl_storage_backend.py index 0583985cb89..ae8a8bc9c90 100644 --- a/lmcache/v1/storage_backend/nixl_storage_backend.py +++ b/lmcache/v1/storage_backend/nixl_storage_backend.py @@ -119,10 +119,38 @@ def from_cache_engine_config( enable_presence_cache = extra_config.get("nixl_presence_cache", False) enable_async_put = extra_config.get("nixl_async_put", False) - backend_params = extra_config.get("nixl_backend_params", {}) + backend_params = dict(extra_config.get("nixl_backend_params", {})) use_direct_io = extra_config.get("use_direct_io", False) pool_size = extra_config.get("nixl_pool_size") + backend = extra_config.get("nixl_backend") + + # Per-worker endpoint distribution: if nixl_endpoint_list is set, + # override endpoint_override so each TP worker targets a different + # object-storage endpoint (round-robin by local_worker_id). + endpoint_list = extra_config.get("nixl_endpoint_list") + if endpoint_list is not None and len(endpoint_list) == 0: + raise ValueError("nixl_endpoint_list is set but empty") + if backend == "OBJ" and endpoint_list: + if "endpoint_override" in backend_params: + logger.warning( + "nixl_endpoint_list is set; ignoring " + "nixl_backend_params.endpoint_override (%s)", + backend_params["endpoint_override"], + ) + ep = endpoint_list[metadata.local_worker_id % len(endpoint_list)] + if not ep.startswith(("http://", "https://")): + raise ValueError( + f"nixl_endpoint_list entry {ep!r} is not a valid URL " + "(must start with 'http://' or 'https://')" + ) + backend_params["endpoint_override"] = ep + logger.info( + "Worker %d using endpoint %s (from %d endpoints)", + metadata.local_worker_id, + ep, + len(endpoint_list), + ) path = extra_config.get("nixl_path") enable_prog_thread = extra_config.get("nixl_enable_prog_thread", True) sync_mode_str = extra_config.get("nixl_sync_mode", None) diff --git a/tests/disagg/test_nixl_storage_backend.py b/tests/disagg/test_nixl_storage_backend.py index 4923099d80c..843158fe5b3 100644 --- a/tests/disagg/test_nixl_storage_backend.py +++ b/tests/disagg/test_nixl_storage_backend.py @@ -71,7 +71,7 @@ def create_test_config( config.extra_config = { "enable_nixl_storage": True, "nixl_backend": backend, - "nixl_file_pool_size": 10, + "nixl_pool_size": 10, "nixl_path": tempfile.mkdtemp(), # Create a temporary directory for testing } return config @@ -119,6 +119,124 @@ def test_nixl_storage_config(): assert not NixlStorageConfig.validate_nixl_backend("INVALID", "cpu") +def _make_obj_config( + extra_overrides: dict | None = None, +) -> LMCacheEngineConfig: + """Create a minimal OBJ-backend config for endpoint-list tests.""" + config = LMCacheEngineConfig() + config.nixl_buffer_size = 2**30 # 1 GB + config.nixl_buffer_device = "cpu" + config.extra_config = { + "enable_nixl_storage": True, + "nixl_backend": "OBJ", + "nixl_pool_size": 0, + "nixl_path": tempfile.mkdtemp(), + } + if extra_overrides: + config.extra_config.update(extra_overrides) + return config + + +def _make_metadata(local_worker_id: int = 0) -> LMCacheMetadata: + """Create test metadata with a configurable local_worker_id.""" + return LMCacheMetadata( + model_name="test_model", + worker_id=local_worker_id, + local_world_size=1, + local_worker_id=local_worker_id, + world_size=1, + kv_dtype=torch.bfloat16, + kv_shape=(32, 2, 256, 1024, 128), + ) + + +@pytest.mark.no_shared_allocator +def test_endpoint_list_round_robin(): + """nixl_endpoint_list should assign endpoints to workers round-robin.""" + endpoints = [ + "https://node-0:9021", + "https://node-1:9021", + "https://node-2:9021", + ] + config = _make_obj_config({"nixl_endpoint_list": endpoints}) + + for local_worker_id in range(6): + metadata = _make_metadata(local_worker_id=local_worker_id) + nixl_config = NixlStorageConfig.from_cache_engine_config(config, metadata) + expected = endpoints[local_worker_id % len(endpoints)] + assert nixl_config.backend_params["endpoint_override"] == expected + + +@pytest.mark.no_shared_allocator +def test_endpoint_list_overrides_endpoint_override(): + """nixl_endpoint_list takes precedence over backend_params endpoint_override.""" + endpoints = ["https://node-0:9021"] + config = _make_obj_config( + { + "nixl_endpoint_list": endpoints, + "nixl_backend_params": { + "endpoint_override": "https://should-be-ignored:9021", + "access_key": "key", + }, + } + ) + metadata = _make_metadata(local_worker_id=0) + + nixl_config = NixlStorageConfig.from_cache_engine_config(config, metadata) + + assert nixl_config.backend_params["endpoint_override"] == endpoints[0] + # other params must be preserved + assert nixl_config.backend_params["access_key"] == "key" + + +@pytest.mark.no_shared_allocator +def test_endpoint_list_does_not_mutate_original_config(): + """Setting nixl_endpoint_list must not mutate the original backend_params dict.""" + original_params = {"access_key": "key", "secret_key": "secret"} + config = _make_obj_config( + { + "nixl_endpoint_list": ["https://node-0:9021"], + "nixl_backend_params": original_params, + } + ) + metadata = _make_metadata(local_worker_id=0) + + NixlStorageConfig.from_cache_engine_config(config, metadata) + + assert "endpoint_override" not in original_params + + +@pytest.mark.no_shared_allocator +def test_endpoint_list_empty_raises(): + """nixl_endpoint_list=[] should raise ValueError before any nixl ops.""" + config = _make_obj_config({"nixl_endpoint_list": []}) + metadata = _make_metadata(local_worker_id=0) + + with pytest.raises(ValueError, match="nixl_endpoint_list is set but empty"): + NixlStorageConfig.from_cache_engine_config(config, metadata) + + +@pytest.mark.no_shared_allocator +def test_no_endpoint_list_leaves_backend_params_unchanged(): + """When nixl_endpoint_list is absent, endpoint_override must not be injected.""" + config = _make_obj_config() # no nixl_endpoint_list key + metadata = _make_metadata(local_worker_id=0) + + nixl_config = NixlStorageConfig.from_cache_engine_config(config, metadata) + + assert "endpoint_override" not in nixl_config.backend_params + + +@pytest.mark.no_shared_allocator +def test_endpoint_list_malformed_url_raises(): + """A non-http(s) entry in nixl_endpoint_list should raise ValueError.""" + config = _make_obj_config({"nixl_endpoint_list": ["htps://typo.example.com"]}) + metadata = _make_metadata(local_worker_id=0) + + with pytest.raises(ValueError, match="is not a valid URL"): + NixlStorageConfig.from_cache_engine_config(config, metadata) + + @pytest.mark.no_shared_allocator @pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA") def test_nixl_storage_backend_basic(): diff --git a/tests/v1/test_nixl_storage.py b/tests/v1/test_nixl_storage.py index 1ed4439751c..740aa57a9a9 100644 --- a/tests/v1/test_nixl_storage.py +++ b/tests/v1/test_nixl_storage.py @@ -16,7 +16,10 @@ from lmcache.v1.memory_management import PagedTensorMemoryAllocator from lmcache.v1.metadata import LMCacheMetadata from lmcache.v1.storage_backend import CreateStorageBackends -from lmcache.v1.storage_backend.nixl_storage_backend import NixlStorageBackend +from lmcache.v1.storage_backend.nixl_storage_backend import ( + NixlStorageBackend, + NixlStorageConfig, +) def create_key(chunk_hash: str): @@ -246,6 +249,49 @@ def test_nixl_gds_cpu_backend(): run(config, shape, dtype) +@pytest.mark.no_shared_allocator +def test_nixl_endpoint_list_empty_raises(): + """nixl_endpoint_list=[] should raise ValueError before any nixl ops.""" + BASE_DIR = Path(__file__).parent + config = LMCacheEngineConfig.from_file(BASE_DIR / "data/nixl.yaml") + config.extra_config["nixl_endpoint_list"] = [] + + metadata = LMCacheMetadata( + model_name="Llama-3.1-70B-Instruct", + world_size=1, + local_world_size=1, + worker_id=0, + local_worker_id=0, + kv_dtype=torch.bfloat16, + kv_shape=[2048, 2048], + ) + + with pytest.raises(ValueError, match="nixl_endpoint_list is set but empty"): + NixlStorageConfig.from_cache_engine_config(config, metadata) + + +@pytest.mark.no_shared_allocator +def test_nixl_endpoint_list_malformed_url_raises(): + """A non-http(s) entry in nixl_endpoint_list should raise ValueError.""" + BASE_DIR = Path(__file__).parent + config = LMCacheEngineConfig.from_file(BASE_DIR / "data/nixl.yaml") + config.extra_config["nixl_endpoint_list"] = ["htps://typo.example.com"] + config.extra_config["nixl_backend"] = "OBJ" + + metadata = LMCacheMetadata( + model_name="Llama-3.1-70B-Instruct", + world_size=1, + local_world_size=1, + worker_id=0, + local_worker_id=0, + kv_dtype=torch.bfloat16, + kv_shape=[2048, 2048], + ) + + with pytest.raises(ValueError, match="is not a valid URL"): + NixlStorageConfig.from_cache_engine_config(config, metadata) + + @pytest.mark.no_shared_allocator def test_nixl_posix_backend(): BASE_DIR = Path(__file__).parent From a7934b7945f9a305a5cd1ff01e6f735a46ef5298 Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Thu, 14 May 2026 17:40:42 -0700 Subject: [PATCH 14/69] [CI] allow PyTorch/NVIDIA egress endpoints in release workflows (#3295) [CI] allow PyTorch/NVIDIA egress endpoints in release workflows Signed-off-by: deng451e <838677410@qq.com> Co-authored-by: Roy Huang --- .github/workflows/build_main_artifacts.yml | 4 ++++ .github/workflows/nightly_build.yml | 1 + .github/workflows/publish.yml | 2 ++ 3 files changed, 7 insertions(+) diff --git a/.github/workflows/build_main_artifacts.yml b/.github/workflows/build_main_artifacts.yml index 1e4e103d18d..29e2a6ba0b0 100644 --- a/.github/workflows/build_main_artifacts.yml +++ b/.github/workflows/build_main_artifacts.yml @@ -32,14 +32,18 @@ jobs: api.github.com:443 github.com:443 registry-1.docker.io:443 + index.docker.io:443 production.cloudflare.docker.com:443 auth.docker.io:443 azure.archive.ubuntu.com:80 pypi.org:443 + pypi.nvidia.com:443 files.pythonhosted.org:443 codeload.github.com:443 objects.githubusercontent.com:443 releases.githubusercontent.com:443 + download.pytorch.org:443 + download-r2.pytorch.org:443 - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index a2b5178b424..d5117141790 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -25,6 +25,7 @@ jobs: uploads.github.com:443 github.com:443 registry-1.docker.io:443 + index.docker.io:443 production.cloudflare.docker.com:443 auth.docker.io:443 azure.archive.ubuntu.com:80 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f890f213f5a..da48ec122ab 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -341,6 +341,7 @@ jobs: allowed-endpoints: > github.com:443 registry-1.docker.io:443 + index.docker.io:443 production.cloudflare.docker.com:443 auth.docker.io:443 azure.archive.ubuntu.com:80 @@ -350,6 +351,7 @@ jobs: releases.astral.sh:443 objects.githubusercontent.com:443 pypi.org:443 + pypi.nvidia.com:443 files.pythonhosted.org:443 release-assets.githubusercontent.com:443 wrapdb.mesonbuild.com:443 From 3090e89634c1ab43b56819462fe4eeae7bcdf1ae Mon Sep 17 00:00:00 2001 From: "Tian, Feng" Date: Sat, 16 May 2026 00:06:30 +0800 Subject: [PATCH 15/69] Add sycl implementation of memory_kernels for Intel XPU (#2902) * Add SYCL/XPU port of mem_kernels with Intel optimizations and build scripts Performance optimization for flash_infer on Intel XPU: 1. Added page_buffer_base_offset() and key_value_base_offset() helpers that compute loop-invariant base offsets (incl. integer division for flash_infer block mapping) once before the inner loop. 2. Added submit_multi_layer_kernel_fused_kv() that processes both K and V in a single work-group for non-MLA formats, halving work-group count and avoiding redundant slot lookups, pointer reads, and division. 3. Updated dispatch logic to use fused kernel for non-MLA formats (flash_infer, flash_attention, cross-layer) and original kernel for MLA formats. 4. Applied base-offset hoisting to unilateral kernel as well. Signed-off-by: Feng Tian * Add build instructions for Intel XPU Signed-off-by: Feng Tian * Update csrc/sycl/mem_kernels_sycl.cpp Fix this typo in code Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Signed-off-by: Tian, Feng * Fix typo in setup.py and __init__.py Signed-off-by: Feng Tian * Refine the logic of importing GPU/XPU/HPU connector symbols Signed-off-by: Feng Tian * fix issues raised by cursor 1. add releasing gil lock for single layer transfer 2. fix typo when printing error message 3. add .spv file to ignore file list Signed-off-by: Feng Tian * fix issues raised by cusor 1. change dtype from uint64 to int64 for old_position calculation 2. change potential out-of-bound access in slot_mapping Signed-off-by: Feng Tian * Fix bugs by merging conflicts Signed-off-by: Feng Tian * Update backend loading logic to support XPU backend The non_cuda_equivalents is the base module to provide default kernels, if backend implments optimized or new ops, use them. Signed-off-by: Feng Tian * skip c ops signature check as XPU compiler doesn't generate signature Signed-off-by: Feng Tian * fix issue found in test_non_cuda_equivalents.py The original code override the non_cuda_equivalents import lib. The change uses a merged module to avoid that. Signed-off-by: Feng Tian * fix bugs introduced by merging dev to this PR 1. adopt the name change from `discover_gpu_kv_format` to `normalize_kv_and_discover_format` in (LMCache#3162) 2. adopt the KVLayerGroupsManager change in (LMCache#3078) Signed-off-by: Feng Tian * Not install cuda or hip related libs when build sycl version Signed-off-by: Feng Tian * merge the Intel XPU related contents to the latest docs Signed-off-by: Feng Tian * sync GPUKVFormat enum to latest one Signed-off-by: Feng Tian * Remove SGLang related mem SYCL kernels Signed-off-by: Feng Tian * Fix cpu backend importing c_ops issue Signed-off-by: Feng Tian * fix pre-commit check error on test_xpu_connector_benchmark.py Signed-off-by: Feng Tian --------- Signed-off-by: Feng Tian Signed-off-by: Tian, Feng Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .gitignore | 4 + csrc/sycl/mem_kernels_sycl.cpp | 840 +++++++++ csrc/sycl/mem_kernels_sycl.h | 129 ++ csrc/sycl/pybind_sycl.cpp | 46 + docs/source/getting_started/installation.rst | 25 +- docs/source/production/docker_deployment.rst | 18 + lmcache/__init__.py | 30 +- lmcache/v1/gpu_connector/__init__.py | 59 +- lmcache/v1/gpu_connector/utils.py | 25 +- lmcache/v1/gpu_connector/xpu_connectors.py | 1587 ++++++++++------- lmcache/v1/memory_management.py | 86 - setup.py | 111 +- .../test_xpu_connector_benchmark.py | 6 + .../test_xpu_layerwise_connector_benchmark.py | 26 +- tests/v1/test_c_ops_fallback_parity.py | 7 +- tests/v1/test_gpu_connector.py | 2 +- tests/v1/test_xpu_connector.py | 1103 ++++++++---- tests/v1/utils.py | 5 +- 18 files changed, 2954 insertions(+), 1155 deletions(-) create mode 100644 csrc/sycl/mem_kernels_sycl.cpp create mode 100644 csrc/sycl/mem_kernels_sycl.h create mode 100644 csrc/sycl/pybind_sycl.cpp diff --git a/.gitignore b/.gitignore index 272adae7845..1e31eb8fde9 100644 --- a/.gitignore +++ b/.gitignore @@ -121,6 +121,9 @@ lmcache/experimental/tests /csrc/*_hip.cuh /csrc/*_hip.cpp +# SYCL build artifacts +/build_sycl/ + # Ignore all log files *.log @@ -139,3 +142,4 @@ operator/dist/ operator/cover.out operator/coverage.html operator/Dockerfile.cross +*.spv diff --git a/csrc/sycl/mem_kernels_sycl.cpp b/csrc/sycl/mem_kernels_sycl.cpp new file mode 100644 index 00000000000..1f1b26203af --- /dev/null +++ b/csrc/sycl/mem_kernels_sycl.cpp @@ -0,0 +1,840 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// SYCL implementation of LMCache memory kernels for Intel XPU. +// Ported from csrc/mem_kernels.cu with Intel-XPU-specific +// optimizations. +// +// Performance-critical design choices for Intel XPU (PVC / Arc / +// Battlemage): +// +// 1. Work-group size 256 (vs CUDA's 128) -- Intel XPU EUs have deep +// hardware-thread scheduling; larger work-groups keep the EU ALUs +// fed and hide global-memory latency. +// +// 2. [[sycl::reqd_sub_group_size(16)]] -- locks SIMD lane width to +// 16, which is the native width across all Intel discrete GPU +// families. Avoids the compiler falling back to sub-group 32 on +// PVC (where it would halve occupancy for these kernels). +// +// 3. Compile-time template parameters for DIRECTION and USE_MLA -- +// eliminates run-time branches inside the innermost loop, +// allowing the IGC (Intel Graphics Compiler) to schedule reads +// and writes without control-flow hazards. +// +// 4. Hoisted loop-invariant base offsets -- integer division and +// modulo (used by flash_infer's block-based indexing) are +// computed once before the inner loop and reused via simple +// base+i addressing. This avoids re-executing expensive 32-bit +// division on every loop iteration and helps the IGC schedule +// straight-line loads/stores. +// +// 5. 64-bit (int64_t) bulk transfers -- packs two fp32 / four fp16 / +// eight int8 values into a single 64-bit move, doubling the +// effective bandwidth compared to element-wise copies. +// +// 6. Fused K+V work-groups (non-MLA multi-layer kernels) -- for +// formats that carry both key and value (k_or_v_size == 2), a +// specialised kernel processes both K and V within a single +// work-group. This halves the total work-group count, avoids +// redundant slot-mapping reads and pointer-array lookups, and +// computes the (potentially expensive) block-index division only +// once per token+layer instead of twice. + +// The SYCL standard headers (sycl/accessor.hpp) reference the deprecated +// 'host_buffer' internally even when user code only uses USM pointers. +// Suppress the resulting -Wdeprecated-declarations noise from these headers. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#include +#pragma GCC diagnostic pop + +#include +#include +#include +#include + +#include "mem_kernels_sycl.h" + +#include +#include +#include +#include + +// --------------------------------------------------------------------------- +// Tuning constants +// --------------------------------------------------------------------------- +// Sub-group (SIMD) width -- 16 is native on PVC, DG2, and BMG. +constexpr int INTEL_SUB_GROUP_SIZE = 16; + +// Maximum work-group size. 256 gives best occupancy / latency- +// hiding trade-off on Intel discrete GPUs whose EUs can schedule +// more hardware threads than CUDA SMs (which typically use 128). +// Must be a multiple of INTEL_SUB_GROUP_SIZE. +constexpr int MAX_WG_SIZE = 256; + +// --------------------------------------------------------------------------- +// Host-side helper: round *up* to the nearest multiple of +// INTEL_SUB_GROUP_SIZE so the work-group is always evenly +// divisible into sub-groups. +// --------------------------------------------------------------------------- +inline int round_up_to_sg(int n) { + return ((n + INTEL_SUB_GROUP_SIZE - 1) / INTEL_SUB_GROUP_SIZE) * + INTEL_SUB_GROUP_SIZE; +} + +// --------------------------------------------------------------------------- +// Namespace lmc -- device-side helper functions +// --------------------------------------------------------------------------- +namespace lmc { + +inline bool is_mla(const GPUKVFormat gpu_kv_format) { + return gpu_kv_format == GPUKVFormat::NL_X_NB_BS_HS || // vLLM MLA + gpu_kv_format == GPUKVFormat::NL_X_NBBS_ONE_HS; // SGLang MLA +} + +template +inline int64_t page_buffer_offset(const int k_or_v, const int token_idx, + const int scalar_offset, + const int scalars_per_token, + const int page_buffer_size, + const int block_size) { + // vLLM cross layer + if constexpr (format == GPUKVFormat::NB_NL_TWO_BS_NH_HS) { + return k_or_v * page_buffer_size * scalars_per_token + + token_idx * scalars_per_token + scalar_offset; + } + // vLLM flash attention + else if constexpr (format == GPUKVFormat::NL_X_TWO_NB_BS_NH_HS) { + return k_or_v * page_buffer_size * scalars_per_token + + token_idx * scalars_per_token + scalar_offset; + } + // vLLM flash infer + else if constexpr (format == GPUKVFormat::NL_X_NB_TWO_BS_NH_HS) { + const int block_idx = token_idx / block_size; + const int block_offset = token_idx % block_size; + return block_idx * 2 * block_size * scalars_per_token + + k_or_v * block_size * scalars_per_token + + block_offset * scalars_per_token + scalar_offset; + } + // MLA formats: vLLM (NL_X_NB_BS_HS) and SGLang (NL_X_NBBS_ONE_HS) + else if constexpr (format == GPUKVFormat::NL_X_NB_BS_HS || + format == GPUKVFormat::NL_X_NBBS_ONE_HS) { + return token_idx * scalars_per_token + scalar_offset; + } +} + +/// Loop-invariant base offset for the paged buffer. +/// page_buffer_offset(k_or_v, slot, i, ...) == base_offset(...) + i. +/// Hoisting this out of the inner loop avoids re-computing the +/// integer division / modulo (flash_infer) on every iteration. +template +inline int64_t page_buffer_base_offset(const int k_or_v, const int token_idx, + const int scalars_per_token, + const int page_buffer_size, + const int block_size) { + if constexpr (format == GPUKVFormat::NB_NL_TWO_BS_NH_HS) { + return k_or_v * page_buffer_size * scalars_per_token + + token_idx * scalars_per_token; + } else if constexpr (format == GPUKVFormat::NL_X_TWO_NB_BS_NH_HS) { + return k_or_v * page_buffer_size * scalars_per_token + + token_idx * scalars_per_token; + } else if constexpr (format == GPUKVFormat::NL_X_NB_TWO_BS_NH_HS) { + const int block_idx = token_idx / block_size; + const int block_offset = token_idx % block_size; + return block_idx * 2 * block_size * scalars_per_token + + k_or_v * block_size * scalars_per_token + + block_offset * scalars_per_token; + } else if constexpr (format == GPUKVFormat::NL_X_NB_BS_HS || + format == GPUKVFormat::NL_X_NBBS_ONE_HS) { + return token_idx * scalars_per_token; + } +} + +inline int64_t key_value_offset(const int k_or_v, const int layer_idx, + const int token_idx, const int scalar_offset, + const int scalars_per_token, + const int num_tokens, const int num_layers) { + return k_or_v * num_layers * num_tokens * scalars_per_token + + layer_idx * num_tokens * scalars_per_token + + token_idx * scalars_per_token + scalar_offset; +} + +/// Loop-invariant base offset for the LMCache key_value buffer. +/// key_value_offset(k_or_v, layer, token, i, ...) == base(...) + i. +inline int64_t key_value_base_offset(const int k_or_v, const int layer_idx, + const int token_idx, + const int scalars_per_token, + const int num_tokens, + const int num_layers) { + return k_or_v * num_layers * num_tokens * scalars_per_token + + layer_idx * num_tokens * scalars_per_token + + token_idx * scalars_per_token; +} + +} // namespace lmc + +// --------------------------------------------------------------------------- +// Pointer helper -- returns a kernel-accessible pointer of the given type. +// For XPU tensors the USM device pointer is returned directly. +// For CPU tensors the host pointer is returned; it must have been allocated +// with USM host memory (e.g. via sycl::malloc_host) to be device-accessible. +// --------------------------------------------------------------------------- +template +T* get_kernel_ptr(TENSOR_TYPE& tensor) { + torch::Device device = tensor.device(); + if (device.is_xpu()) { + return static_cast(tensor.data_ptr()); + } else if (device.is_cpu()) { + // USM host pointers are accessible from both host and device in SYCL. + return static_cast(tensor.data_ptr()); + } else { + TORCH_CHECK(false, + "Invalid device. Device must be xpu or cpu (USM pinned)."); + } +} + +// --------------------------------------------------------------------------- +// Kernel-launch helpers -- multi-layer kernels +// --------------------------------------------------------------------------- + +/** + * Submit the multi-layer KV transfer kernel for MLA formats + * (k_or_v_size == 1). + * + * SYCL nd_range mapping (CUDA → SYCL): + * blockIdx.x (token_id) → item.get_group(2) + * blockIdx.y (layer_id) → item.get_group(1) + * threadIdx.x (tid) → item.get_local_id(2) + * blockDim.x (nthreads) → item.get_local_range(2) + * + * Optimizations over the naïve port: + * - DIRECTION is a compile-time bool (no branch in hot loop) + * - Loop-invariant base offsets (including integer division for + * flash_infer) are computed once before the inner loop + * - Work-group size rounded to sub-group multiple for full + * SIMD utilisation + */ +template +void submit_multi_layer_kernel(sycl::queue& queue, scalar_t* key_value_ptr, + scalar_t** page_buffer_ptrs, + const int64_t* slot_mapping_ptr, + int scalars_per_token, int num_tokens, + int num_layers, int page_buffer_size, + int block_size, int skip_prefix_n_tokens, + int k_or_v_size, int wg_size) { + int num_transfer_tokens = num_tokens - skip_prefix_n_tokens; + if (num_transfer_tokens <= 0 || num_layers <= 0) return; + + sycl::range<3> global_range( + static_cast(k_or_v_size), static_cast(num_layers), + static_cast(num_transfer_tokens) * wg_size); + sycl::range<3> local_range(1, 1, static_cast(wg_size)); + + queue.parallel_for( + sycl::nd_range<3>(global_range, local_range), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(16)]] { + const int token_id = static_cast(item.get_group(2)); + const int layer_id = static_cast(item.get_group(1)); + const int k_or_v = static_cast(item.get_group(0)); + const int tid = static_cast(item.get_local_id(2)); + const int num_threads = static_cast(item.get_local_range(2)); + + const int kv_token_id = token_id + skip_prefix_n_tokens; + const int64_t slot_idx = slot_mapping_ptr[kv_token_id]; + scalar_t* paged_buffer_ptr = page_buffer_ptrs[layer_id]; + + if (slot_idx < 0) return; + + // Hoist loop-invariant base offsets (integer division for + // flash_infer happens here, once, not per loop iteration). + const int64_t lmc_base = lmc::key_value_base_offset( + k_or_v, layer_id, kv_token_id, scalars_per_token, num_tokens, + num_layers); + const int64_t vllm_base = lmc::page_buffer_base_offset( + k_or_v, slot_idx, scalars_per_token, page_buffer_size, block_size); + + for (int i = tid; i < scalars_per_token; i += num_threads) { + if constexpr (DIRECTION) { + key_value_ptr[lmc_base + i] = paged_buffer_ptr[vllm_base + i]; + } else { + paged_buffer_ptr[vllm_base + i] = key_value_ptr[lmc_base + i]; + } + } + }); +} + +/** + * Submit a fused K+V multi-layer kernel for non-MLA formats. + * + * Processes both key (k_or_v=0) and value (k_or_v=1) within the + * same work-group, halving the total number of work-groups compared + * to `submit_multi_layer_kernel` with k_or_v_size=2. + * + * Benefits on Intel XPU: + * - Halves work-group dispatch overhead + * - Slot mapping and pointer array are read once per token+layer + * instead of twice (once for K, once for V) + * - Integer division/modulo (flash_infer block mapping) is + * computed once and reused for both K and V + * - Interleaved K+V memory requests help hide latency + */ +template +void submit_multi_layer_kernel_fused_kv( + sycl::queue& queue, scalar_t* key_value_ptr, scalar_t** page_buffer_ptrs, + const int64_t* slot_mapping_ptr, int scalars_per_token, int num_tokens, + int num_layers, int page_buffer_size, int block_size, + int skip_prefix_n_tokens, int wg_size) { + int num_transfer_tokens = num_tokens - skip_prefix_n_tokens; + if (num_transfer_tokens <= 0 || num_layers <= 0) return; + + // Grid: (1, num_layers, num_transfer_tokens * wg_size) + // k_or_v dimension is gone -- both K and V handled in one work-group. + sycl::range<3> global_range( + 1, static_cast(num_layers), + static_cast(num_transfer_tokens) * wg_size); + sycl::range<3> local_range(1, 1, static_cast(wg_size)); + + queue.parallel_for( + sycl::nd_range<3>(global_range, local_range), + [=](sycl::nd_item<3> item) [[sycl::reqd_sub_group_size(16)]] { + const int token_id = static_cast(item.get_group(2)); + const int layer_id = static_cast(item.get_group(1)); + const int tid = static_cast(item.get_local_id(2)); + const int num_threads = static_cast(item.get_local_range(2)); + + const int kv_token_id = token_id + skip_prefix_n_tokens; + const int64_t slot_idx = slot_mapping_ptr[kv_token_id]; + scalar_t* paged_buffer_ptr = page_buffer_ptrs[layer_id]; + + if (slot_idx < 0) return; + + // Base offsets for K (k_or_v=0) and V (k_or_v=1). + // The expensive division/modulo (flash_infer) happens once. + const int64_t lmc_base_k = lmc::key_value_base_offset( + 0, layer_id, kv_token_id, scalars_per_token, num_tokens, + num_layers); + const int64_t lmc_base_v = lmc::key_value_base_offset( + 1, layer_id, kv_token_id, scalars_per_token, num_tokens, + num_layers); + const int64_t vllm_base_k = lmc::page_buffer_base_offset( + 0, slot_idx, scalars_per_token, page_buffer_size, block_size); + const int64_t vllm_base_v = lmc::page_buffer_base_offset( + 1, slot_idx, scalars_per_token, page_buffer_size, block_size); + + for (int i = tid; i < scalars_per_token; i += num_threads) { + if constexpr (DIRECTION) { + // paged buffer → LMCache + key_value_ptr[lmc_base_k + i] = paged_buffer_ptr[vllm_base_k + i]; + key_value_ptr[lmc_base_v + i] = paged_buffer_ptr[vllm_base_v + i]; + } else { + // LMCache → paged buffer + paged_buffer_ptr[vllm_base_k + i] = key_value_ptr[lmc_base_k + i]; + paged_buffer_ptr[vllm_base_v + i] = key_value_ptr[lmc_base_v + i]; + } + } + }); +} + +// --------------------------------------------------------------------------- +// Macros to dispatch multi-layer kernels with a specific GPUKVFormat +// --------------------------------------------------------------------------- +// For MLA formats (k_or_v_size == 1): use the original per-component kernel. +// For non-MLA formats (k_or_v_size == 2): use the fused K+V kernel that +// halves work-groups and avoids redundant slot/pointer/division work. +// --------------------------------------------------------------------------- +#define LAUNCH_KERNEL_WITH_FORMAT(T, DIRECTION, FORMAT) \ + submit_multi_layer_kernel( \ + queue, key_value_ptr, page_buffer_ptrs, slot_mapping_ptr, num_xwords, \ + num_tokens, num_layers, page_buffer_size, block_size, \ + skip_prefix_n_tokens, k_or_v_size, wg_size); + +#define LAUNCH_FUSED_KV_KERNEL_WITH_FORMAT(T, DIRECTION, FORMAT) \ + submit_multi_layer_kernel_fused_kv( \ + queue, key_value_ptr, page_buffer_ptrs, slot_mapping_ptr, num_xwords, \ + num_tokens, num_layers, page_buffer_size, block_size, \ + skip_prefix_n_tokens, wg_size); + +// --------------------------------------------------------------------------- +// multi_layer_kv_transfer -- templated implementation +// --------------------------------------------------------------------------- +template +void multi_layer_kv_transfer_templated( + torch::Tensor& key_value, const torch::Tensor& key_value_ptrs, + const torch::Tensor& slot_mapping, const torch::Device& paged_memory_device, + const int page_buffer_size, const TransferDirection direction, + const GPUKVFormat gpu_kv_format, const int block_size, + const int skip_prefix_n_tokens) { + T* key_value_ptr = get_kernel_ptr(key_value); + T** page_buffer_ptrs = + get_kernel_ptr(key_value_ptrs); + const int64_t* slot_mapping_ptr = + get_kernel_ptr(slot_mapping); + + int num_layers = key_value.size(1); + int num_tokens = key_value.size(2); + int num_origin_elements = key_value.size(3); + int elements_per_xword = sizeof(T) / key_value.element_size(); + int num_xwords = num_origin_elements / elements_per_xword; + + int k_or_v_size = lmc::is_mla(gpu_kv_format) ? 1 : 2; + + // Round up to a sub-group multiple so every sub-group is full. + int wg_size = round_up_to_sg(std::min(num_xwords, MAX_WG_SIZE)); + + const c10::OptionalDeviceGuard device_guard(paged_memory_device); + sycl::queue& queue = + c10::xpu::getCurrentXPUStream(paged_memory_device.index()).queue(); + + // Non-MLA formats use the fused K+V kernel (processes both K and V + // in a single work-group). MLA formats (k_or_v_size==1) use the + // original per-component kernel. + if (k_or_v_size == 2) { + if (direction == TransferDirection::H2D) { + switch (gpu_kv_format) { + case GPUKVFormat::NB_NL_TWO_BS_NH_HS: + LAUNCH_FUSED_KV_KERNEL_WITH_FORMAT(T, false, + GPUKVFormat::NB_NL_TWO_BS_NH_HS); + break; + case GPUKVFormat::NL_X_TWO_NB_BS_NH_HS: + LAUNCH_FUSED_KV_KERNEL_WITH_FORMAT(T, false, + GPUKVFormat::NL_X_TWO_NB_BS_NH_HS); + break; + case GPUKVFormat::NL_X_NB_TWO_BS_NH_HS: + LAUNCH_FUSED_KV_KERNEL_WITH_FORMAT(T, false, + GPUKVFormat::NL_X_NB_TWO_BS_NH_HS); + break; + default: + throw std::runtime_error("Unsupported non-MLA GPUKVFormat"); + } + } else { + switch (gpu_kv_format) { + case GPUKVFormat::NB_NL_TWO_BS_NH_HS: + LAUNCH_FUSED_KV_KERNEL_WITH_FORMAT(T, true, + GPUKVFormat::NB_NL_TWO_BS_NH_HS); + break; + case GPUKVFormat::NL_X_TWO_NB_BS_NH_HS: + LAUNCH_FUSED_KV_KERNEL_WITH_FORMAT(T, true, + GPUKVFormat::NL_X_TWO_NB_BS_NH_HS); + break; + case GPUKVFormat::NL_X_NB_TWO_BS_NH_HS: + LAUNCH_FUSED_KV_KERNEL_WITH_FORMAT(T, true, + GPUKVFormat::NL_X_NB_TWO_BS_NH_HS); + break; + default: + throw std::runtime_error("Unsupported non-MLA GPUKVFormat"); + } + } + } else { + // MLA path (k_or_v_size == 1) + if (direction == TransferDirection::H2D) { + switch (gpu_kv_format) { + case GPUKVFormat::NL_X_NB_BS_HS: + LAUNCH_KERNEL_WITH_FORMAT(T, false, GPUKVFormat::NL_X_NB_BS_HS); + break; + case GPUKVFormat::NL_X_NBBS_ONE_HS: + LAUNCH_KERNEL_WITH_FORMAT(T, false, GPUKVFormat::NL_X_NBBS_ONE_HS); + break; + default: + throw std::runtime_error("Unsupported MLA GPUKVFormat"); + } + } else { + switch (gpu_kv_format) { + case GPUKVFormat::NL_X_NB_BS_HS: + LAUNCH_KERNEL_WITH_FORMAT(T, true, GPUKVFormat::NL_X_NB_BS_HS); + break; + case GPUKVFormat::NL_X_NBBS_ONE_HS: + LAUNCH_KERNEL_WITH_FORMAT(T, true, GPUKVFormat::NL_X_NBBS_ONE_HS); + break; + default: + throw std::runtime_error("Unsupported MLA GPUKVFormat"); + } + } + } +} + +#undef LAUNCH_KERNEL_WITH_FORMAT +#undef LAUNCH_FUSED_KV_KERNEL_WITH_FORMAT + +// --------------------------------------------------------------------------- +// Public API: multi_layer_kv_transfer +// --------------------------------------------------------------------------- +void multi_layer_kv_transfer( + torch::Tensor& key_value, const torch::Tensor& key_value_ptrs, + const torch::Tensor& slot_mapping, const torch::Device& paged_memory_device, + const int page_buffer_size, const TransferDirection direction, + const GPUKVFormat gpu_kv_format, const int block_size, + const int skip_prefix_n_tokens) { + int num_origin_elements = key_value.size(3); + int copy_size = num_origin_elements * key_value.element_size(); + +#define LAUNCH_MULTI_LAYER_KV_TRANSFER(type) \ + do { \ + multi_layer_kv_transfer_templated( \ + key_value, key_value_ptrs, slot_mapping, paged_memory_device, \ + page_buffer_size, direction, gpu_kv_format, block_size, \ + skip_prefix_n_tokens); \ + } while (0) + if (copy_size % 8 == 0) { + LAUNCH_MULTI_LAYER_KV_TRANSFER(int64_t); + } else if (copy_size % 4 == 0) { + LAUNCH_MULTI_LAYER_KV_TRANSFER(int32_t); + } else if (copy_size % 2 == 0) { + LAUNCH_MULTI_LAYER_KV_TRANSFER(int16_t); + } else { + LAUNCH_MULTI_LAYER_KV_TRANSFER(int8_t); + } +#undef LAUNCH_MULTI_LAYER_KV_TRANSFER +} + +// --------------------------------------------------------------------------- +// single_layer_kv_transfer — helper template +// --------------------------------------------------------------------------- +// USE_MLA and IS_D2H are template parameters so the compiler can +// dead-strip the unused branch and emit straight-line code. +template +void single_layer_kv_transfer_impl(sycl::queue& queue, int64_t* lmc_ptr, + int64_t* vllm_ptr, const int64_t* slot_ptr, + int num_tokens, int n, int lmc_stride, + int lmc_value_offset, int block_size, + int vllm_block_key_stride_in_64bit, + int vllm_value_offset, int num_heads, + int head_size_in_64bit, int wg_size) { + if (num_tokens <= 0) return; + + sycl::range<1> global_range(static_cast(num_tokens) * wg_size); + sycl::range<1> local_range(static_cast(wg_size)); + + queue.parallel_for( + sycl::nd_range<1>(global_range, local_range), + [=](sycl::nd_item<1> item) [[sycl::reqd_sub_group_size(16)]] { + const int64_t token_idx = static_cast(item.get_group(0)); + const int64_t slot_idx = slot_ptr[token_idx]; + if (slot_idx < 0) return; + + const int64_t block_idx = slot_idx / block_size; + const int64_t block_offset = slot_idx % block_size; + + const int tid = static_cast(item.get_local_id(0)); + const int nthreads = static_cast(item.get_local_range(0)); + + for (int i = tid; i < n; i += nthreads) { + const int64_t lmc_key_idx = token_idx * lmc_stride + i; + const int head_idx = i / head_size_in_64bit; + const int head_offset = i % head_size_in_64bit; + const int64_t vllm_key_idx = + block_idx * vllm_block_key_stride_in_64bit + + block_offset * num_heads * head_size_in_64bit + + head_idx * head_size_in_64bit + head_offset; + + if constexpr (IS_D2H) { + lmc_ptr[lmc_key_idx] = vllm_ptr[vllm_key_idx]; + if constexpr (!USE_MLA) { + const int64_t lmc_value_idx = lmc_key_idx + lmc_value_offset; + const int64_t vllm_value_idx = vllm_key_idx + vllm_value_offset; + lmc_ptr[lmc_value_idx] = vllm_ptr[vllm_value_idx]; + } + } else { + vllm_ptr[vllm_key_idx] = lmc_ptr[lmc_key_idx]; + if constexpr (!USE_MLA) { + const int64_t lmc_value_idx = lmc_key_idx + lmc_value_offset; + const int64_t vllm_value_idx = vllm_key_idx + vllm_value_offset; + vllm_ptr[vllm_value_idx] = lmc_ptr[lmc_value_idx]; + } + } + } + }); +} + +// --------------------------------------------------------------------------- +// Public API: single_layer_kv_transfer +// --------------------------------------------------------------------------- +void single_layer_kv_transfer(torch::Tensor& lmc_key_value_cache, + torch::Tensor& vllm_key_value_cache, + torch::Tensor& slot_mapping, + const TransferDirection direction, + const GPUKVFormat gpu_kv_format, + const bool token_major) { + int64_t* lmc_key_value_cache_ptr = + get_kernel_ptr(lmc_key_value_cache); + int64_t* vllm_key_value_cache_ptr = + get_kernel_ptr(vllm_key_value_cache); + const int64_t* slot_mapping_ptr = + get_kernel_ptr(slot_mapping); + + int elements_per_entry = 8 / vllm_key_value_cache.element_size(); + + int num_tokens = slot_mapping.size(0); + int num_heads; + int head_size_in_64bit; + int block_size; + + const bool use_mla = lmc::is_mla(gpu_kv_format); + + if (use_mla) { + num_heads = 1; + block_size = vllm_key_value_cache.size(1); + head_size_in_64bit = vllm_key_value_cache.size(2) / elements_per_entry; + } else { + num_heads = vllm_key_value_cache.size(3); + head_size_in_64bit = vllm_key_value_cache.size(4) / elements_per_entry; + block_size = vllm_key_value_cache.size(2); + } + + int lmc_stride; + int lmc_value_offset; + if (use_mla) { + lmc_stride = lmc_key_value_cache.stride(0) / elements_per_entry; + lmc_value_offset = 0; + } else if (token_major) { + lmc_stride = lmc_key_value_cache.stride(0) / elements_per_entry; + lmc_value_offset = lmc_key_value_cache.stride(1) / elements_per_entry; + } else { + lmc_stride = lmc_key_value_cache.stride(1) / elements_per_entry; + lmc_value_offset = lmc_key_value_cache.stride(0) / elements_per_entry; + } + + int vllm_block_key_stride_in_64bit; + int vllm_value_offset; + if (use_mla) { + vllm_block_key_stride_in_64bit = + vllm_key_value_cache.stride(0) / elements_per_entry; + vllm_value_offset = 0; + } else if (gpu_kv_format == GPUKVFormat::NL_X_TWO_NB_BS_NH_HS) { + vllm_block_key_stride_in_64bit = + vllm_key_value_cache.stride(1) / elements_per_entry; + vllm_value_offset = vllm_key_value_cache.stride(0) / elements_per_entry; + } else { // NL_X_NB_TWO_BS_NH_HS + vllm_block_key_stride_in_64bit = + vllm_key_value_cache.stride(0) / elements_per_entry; + vllm_value_offset = vllm_key_value_cache.stride(1) / elements_per_entry; + } + + int n = num_heads * head_size_in_64bit; + int wg_size = round_up_to_sg(std::min(n, MAX_WG_SIZE)); + if (num_tokens <= 0) return; + + const c10::OptionalDeviceGuard device_guard(device_of(vllm_key_value_cache)); + sycl::queue& queue = + c10::xpu::getCurrentXPUStream(vllm_key_value_cache.device().index()) + .queue(); + + auto lmc_ptr = lmc_key_value_cache_ptr; + auto vllm_ptr = vllm_key_value_cache_ptr; + auto slot_ptr = slot_mapping_ptr; + + // Dispatch to 4 compile-time specialisations + // (USE_MLA × IS_D2H) so the inner loop is + // branch-free. + if (use_mla) { + if (direction == TransferDirection::D2H) + single_layer_kv_transfer_impl( + queue, lmc_ptr, vllm_ptr, slot_ptr, num_tokens, n, lmc_stride, + lmc_value_offset, block_size, vllm_block_key_stride_in_64bit, + vllm_value_offset, num_heads, head_size_in_64bit, wg_size); + else + single_layer_kv_transfer_impl( + queue, lmc_ptr, vllm_ptr, slot_ptr, num_tokens, n, lmc_stride, + lmc_value_offset, block_size, vllm_block_key_stride_in_64bit, + vllm_value_offset, num_heads, head_size_in_64bit, wg_size); + } else { + if (direction == TransferDirection::D2H) + single_layer_kv_transfer_impl( + queue, lmc_ptr, vllm_ptr, slot_ptr, num_tokens, n, lmc_stride, + lmc_value_offset, block_size, vllm_block_key_stride_in_64bit, + vllm_value_offset, num_heads, head_size_in_64bit, wg_size); + else + single_layer_kv_transfer_impl( + queue, lmc_ptr, vllm_ptr, slot_ptr, num_tokens, n, lmc_stride, + lmc_value_offset, block_size, vllm_block_key_stride_in_64bit, + vllm_value_offset, num_heads, head_size_in_64bit, wg_size); + } +} + +// --------------------------------------------------------------------------- +// Public API: load_and_reshape_flash (deprecated -- unit tests only) +// --------------------------------------------------------------------------- +void load_and_reshape_flash(torch::Tensor& key_value, torch::Tensor& key_cache, + torch::Tensor& value_cache, + torch::Tensor& slot_mapping, const int layer_idx) { + int64_t* key_value_ptr = get_kernel_ptr(key_value); + int64_t* key_cache_ptr = get_kernel_ptr(key_cache); + int64_t* value_cache_ptr = + get_kernel_ptr(value_cache); + const int64_t* slot_mapping_ptr = + get_kernel_ptr(slot_mapping); + + int elements_per_entry = 8 / key_cache.element_size(); + int num_tokens = slot_mapping.size(0); + int num_heads = key_cache.size(2); + int head_size_in_64bit = key_cache.size(3) / elements_per_entry; + int block_size = key_cache.size(1); + int key_value_stride = key_value.stride(2) / elements_per_entry; + int num_layers = key_value.size(1); + int key_layer_offset = layer_idx * key_value.stride(1) / elements_per_entry; + int value_layer_offset = + (layer_idx + num_layers) * key_value.stride(1) / elements_per_entry; + int block_stride_in_64bit = key_cache.stride(0) / elements_per_entry; + TORCH_CHECK(key_cache.stride(0) == value_cache.stride(0)); + + int n = num_heads * head_size_in_64bit; + int wg_size = round_up_to_sg(std::min(n, MAX_WG_SIZE)); + if (num_tokens <= 0) return; + + sycl::range<1> global_range(static_cast(num_tokens) * wg_size); + sycl::range<1> local_range(static_cast(wg_size)); + + const c10::OptionalDeviceGuard device_guard(device_of(key_cache)); + sycl::queue& queue = + c10::xpu::getCurrentXPUStream(key_cache.device().index()).queue(); + + auto kv_ptr = key_value_ptr; + auto k_ptr = key_cache_ptr; + auto v_ptr = value_cache_ptr; + auto slot_ptr = slot_mapping_ptr; + + queue.parallel_for( + sycl::nd_range<1>(global_range, local_range), + [=](sycl::nd_item<1> item) [[sycl::reqd_sub_group_size(16)]] { + const int64_t token_idx = static_cast(item.get_group(0)); + const int64_t slot_idx = slot_ptr[token_idx]; + if (slot_idx < 0) return; + + const int64_t blk_idx = slot_idx / block_size; + const int64_t blk_off = slot_idx % block_size; + + const int tid = static_cast(item.get_local_id(0)); + const int nthreads = static_cast(item.get_local_range(0)); + + for (int i = tid; i < n; i += nthreads) { + const int64_t tgt_key_idx = + key_layer_offset + token_idx * key_value_stride + i; + const int64_t tgt_value_idx = + value_layer_offset + token_idx * key_value_stride + i; + + const int head_idx = i / head_size_in_64bit; + const int head_offset = i % head_size_in_64bit; + const int64_t src_kv_idx = blk_idx * block_stride_in_64bit + + blk_off * num_heads * head_size_in_64bit + + head_idx * head_size_in_64bit + + head_offset; + + kv_ptr[tgt_key_idx] = k_ptr[src_kv_idx]; + kv_ptr[tgt_value_idx] = v_ptr[src_kv_idx]; + } + }); +} + +// --------------------------------------------------------------------------- +// Public API: reshape_and_cache_back_flash (deprecated -- unit +// tests only) +// --------------------------------------------------------------------------- +void reshape_and_cache_back_flash(torch::Tensor& key_value, + torch::Tensor& key_cache, + torch::Tensor& value_cache, + torch::Tensor& slot_mapping, + const int layer_idx) { + int64_t* key_cache_ptr = get_kernel_ptr(key_cache); + int64_t* value_cache_ptr = + get_kernel_ptr(value_cache); + int64_t* key_value_ptr = get_kernel_ptr(key_value); + const int64_t* slot_mapping_ptr = + get_kernel_ptr(slot_mapping); + + int elements_per_entry = 8 / key_cache.element_size(); + int num_tokens = slot_mapping.size(0); + int num_heads = key_cache.size(2); + int head_size_in_64bit = key_cache.size(3) / elements_per_entry; + int block_size = key_cache.size(1); + int key_value_stride = key_value.stride(2) / elements_per_entry; + int num_layers = key_value.size(1); + int key_layer_offset = layer_idx * key_value.stride(1) / elements_per_entry; + int value_layer_offset = + (layer_idx + num_layers) * key_value.stride(1) / elements_per_entry; + int block_stride_in_64bit = key_cache.stride(0) / elements_per_entry; + TORCH_CHECK(key_cache.stride(0) == value_cache.stride(0)); + + int n = num_heads * head_size_in_64bit; + int wg_size = round_up_to_sg(std::min(n, MAX_WG_SIZE)); + if (num_tokens <= 0) return; + + sycl::range<1> global_range(static_cast(num_tokens) * wg_size); + sycl::range<1> local_range(static_cast(wg_size)); + + const c10::OptionalDeviceGuard device_guard(device_of(key_cache)); + sycl::queue& queue = + c10::xpu::getCurrentXPUStream(key_cache.device().index()).queue(); + + auto kv_ptr = key_value_ptr; + auto k_ptr = key_cache_ptr; + auto v_ptr = value_cache_ptr; + auto slot_ptr = slot_mapping_ptr; + + queue.parallel_for( + sycl::nd_range<1>(global_range, local_range), + [=](sycl::nd_item<1> item) [[sycl::reqd_sub_group_size(16)]] { + const int64_t token_idx = static_cast(item.get_group(0)); + const int64_t slot_idx = slot_ptr[token_idx]; + if (slot_idx < 0) return; + + const int64_t blk_idx = slot_idx / block_size; + const int64_t blk_off = slot_idx % block_size; + + const int tid = static_cast(item.get_local_id(0)); + const int nthreads = static_cast(item.get_local_range(0)); + + for (int i = tid; i < n; i += nthreads) { + const int64_t tgt_key_idx = + key_layer_offset + token_idx * key_value_stride + i; + const int64_t tgt_value_idx = + value_layer_offset + token_idx * key_value_stride + i; + + const int head_idx = i / head_size_in_64bit; + const int head_offset = i % head_size_in_64bit; + const int64_t src_kv_idx = blk_idx * block_stride_in_64bit + + blk_off * num_heads * head_size_in_64bit + + head_idx * head_size_in_64bit + + head_offset; + + k_ptr[src_kv_idx] = kv_ptr[tgt_key_idx]; + v_ptr[src_kv_idx] = kv_ptr[tgt_value_idx]; + } + }); +} + +// --------------------------------------------------------------------------- +// Public API: lmcache_memcpy_async +// --------------------------------------------------------------------------- +void lmcache_memcpy_async(uintptr_t dest, uintptr_t src, size_t nbytes, + TransferDirection direction, + size_t host_buffer_offset, + size_t host_buffer_alignments) { + TORCH_CHECK((host_buffer_alignments & (host_buffer_alignments - 1)) == 0, + "host_buffer_alignments must be power of two"); + + // SYCL USM memcpy infers direction from pointer allocation types. + // The direction parameter is kept for API compatibility with the CUDA + // version but is not needed by the SYCL runtime. + (void)direction; + + sycl::queue& queue = c10::xpu::getCurrentXPUStream().queue(); + + size_t offset = 0; + const size_t mask = host_buffer_alignments - 1; + + while (offset < nbytes) { + size_t current_src = src + offset; + size_t current_dest = dest + offset; + + size_t aligned_area_end = + ((offset + host_buffer_offset) & ~mask) + host_buffer_alignments; + size_t real_end = std::min(host_buffer_offset + nbytes, aligned_area_end); + size_t max_nbytes = real_end - offset - host_buffer_offset; + + // SYCL USM memcpy is direction-agnostic; the runtime resolves + // the copy direction from the pointer allocation types. + queue.memcpy(reinterpret_cast(current_dest), + reinterpret_cast(current_src), max_nbytes); + + offset += max_nbytes; + } +} diff --git a/csrc/sycl/mem_kernels_sycl.h b/csrc/sycl/mem_kernels_sycl.h new file mode 100644 index 00000000000..04cb56cc11a --- /dev/null +++ b/csrc/sycl/mem_kernels_sycl.h @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +enum class TransferDirection : int { + H2D = 0, + D2H = 1, +}; + +/* +Symbol Reference: +NL: number of layers +NB: number of blocks/pages +BS: block/page size +NBBS: block/page buffer size = NB * BS +NH: number of heads +HS: head size +TWO: 2 +ONE: 1 + +_ means a dimension within the same tensor +_X_ means a dimension across a list + +A_X_B_X_C_D_E means: +kv_cache: List[List[torch.Tensor]] +len(kv_cache) = A +len(kv_cache[0]) = B +kv_cache[0][0].shape = (C, D, E) + +The logic for identifying the format currently lives in +`lmcache/v1/gpu_connector/utils.py` +*/ +enum class GPUKVFormat : int { + NB_NL_TWO_BS_NH_HS = 0, + /* + used by: + - vLLM CROSS_LAYER mode + */ + + NL_X_TWO_NB_BS_NH_HS = 1, + /* + used by: + - vLLM non-MLA flash attention + */ + + NL_X_NB_TWO_BS_NH_HS = 2, + /* + used by: + - vLLM non-MLA flash infer + */ + + NL_X_NB_BS_HS = 3, + /* + used by: + - vLLM MLA + */ + + TWO_X_NL_X_NBBS_NH_HS = 4, + /* + used by: + - SGLang MHA (flash attention and flash infer) + */ + + NL_X_NBBS_ONE_HS = 5, + /* + used by: + - SGLang MLA + */ + + NL_X_TWO_NB_NH_BS_HS = 6, + /* + used by: + - vLLM non-MLA flash attention (HND layout) + physical shape per layer: [2, num_blocks, num_heads, block_size, head_size] + */ + + NL_X_NB_TWO_NH_BS_HS = 7, + /* + used by: + - vLLM non-MLA flash infer (HND layout) + physical shape per layer: [num_blocks, 2, num_heads, block_size, head_size] + */ + + NB_NL_TWO_NH_BS_HS = 8, + /* + used by: + - TRT-LLM cross-layer (HND layout) + physical shape: [num_blocks, num_layers, 2, num_heads, block_size, head_size] + */ +}; + +void multi_layer_kv_transfer( + torch::Tensor& key_value, const torch::Tensor& key_value_ptrs, + const torch::Tensor& slot_mapping, const torch::Device& paged_memory_device, + const int page_buffer_size, const TransferDirection direction, + const GPUKVFormat gpu_kv_format, const int block_size = 0, + const int skip_prefix_n_tokens = 0); + +void single_layer_kv_transfer(torch::Tensor& lmc_key_value_cache, + torch::Tensor& vllm_key_value_cache, + torch::Tensor& slot_mapping, + const TransferDirection direction, + const GPUKVFormat gpu_kv_format, + const bool token_major = false); + +// Asynchronous memory copy between host and device buffers. +// Note: the direction parameter is retained for API compatibility with the +// CUDA version but is not used by the SYCL implementation (SYCL USM memcpy +// infers direction from pointer allocation types). +void lmcache_memcpy_async(uintptr_t dest, uintptr_t src, size_t nbytes, + TransferDirection direction, + size_t host_buffer_offset, + size_t host_buffer_alignments); + +// deprecated / unused except in unit tests +void load_and_reshape_flash(torch::Tensor& key_value, torch::Tensor& key_cache, + torch::Tensor& value_cache, + torch::Tensor& slot_mapping, const int layer_idx); + +// deprecated / unused except in unit tests +void reshape_and_cache_back_flash(torch::Tensor& key_value, + torch::Tensor& key_cache, + torch::Tensor& value_cache, + torch::Tensor& slot_mapping, + const int layer_idx); diff --git a/csrc/sycl/pybind_sycl.cpp b/csrc/sycl/pybind_sycl.cpp new file mode 100644 index 00000000000..0935d1333fb --- /dev/null +++ b/csrc/sycl/pybind_sycl.cpp @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 + +// +// Python bindings for the SYCL/XPU memory kernels. +// This module (lmcache.xpu_ops) mirrors the mem-kernel subset of +// lmcache.xpu_ops but targets Intel XPU via SYCL. +// +#include +#include +#include "mem_kernels_sycl.h" + +namespace py = pybind11; + +PYBIND11_MODULE(xpu_ops, m) { + py::enum_(m, "TransferDirection") + .value("H2D", TransferDirection::H2D) + .value("D2H", TransferDirection::D2H) + .export_values(); + py::enum_(m, "GPUKVFormat") + .value("NB_NL_TWO_BS_NH_HS", GPUKVFormat::NB_NL_TWO_BS_NH_HS) + .value("NL_X_TWO_NB_BS_NH_HS", GPUKVFormat::NL_X_TWO_NB_BS_NH_HS) + .value("NL_X_NB_TWO_BS_NH_HS", GPUKVFormat::NL_X_NB_TWO_BS_NH_HS) + .value("NL_X_NB_BS_HS", GPUKVFormat::NL_X_NB_BS_HS) + .value("TWO_X_NL_X_NBBS_NH_HS", GPUKVFormat::TWO_X_NL_X_NBBS_NH_HS) + .value("NL_X_NBBS_ONE_HS", GPUKVFormat::NL_X_NBBS_ONE_HS) + .value("NL_X_TWO_NB_NH_BS_HS", GPUKVFormat::NL_X_TWO_NB_NH_BS_HS) + .value("NL_X_NB_TWO_NH_BS_HS", GPUKVFormat::NL_X_NB_TWO_NH_BS_HS) + .value("NB_NL_TWO_NH_BS_HS", GPUKVFormat::NB_NL_TWO_NH_BS_HS) + .export_values(); + m.def("multi_layer_kv_transfer", &multi_layer_kv_transfer, + py::arg("key_value"), py::arg("key_value_ptrs"), + py::arg("slot_mapping"), py::arg("paged_memory_device"), + py::arg("page_buffer_size"), py::arg("direction"), + py::arg("gpu_kv_format"), py::arg("block_size") = 0, + py::arg("skip_prefix_n_tokens") = 0, + py::call_guard()); + m.def("single_layer_kv_transfer", &single_layer_kv_transfer, + py::arg("lmc_key_value_cache"), py::arg("vllm_key_value_cache"), + py::arg("slot_mapping"), py::arg("direction"), py::arg("gpu_kv_format"), + py::arg("token_major") = false, + py::call_guard()); + m.def("load_and_reshape_flash", &load_and_reshape_flash); + m.def("reshape_and_cache_back_flash", &reshape_and_cache_back_flash); + m.def("lmcache_memcpy_async", &lmcache_memcpy_async, + py::call_guard()); +} diff --git a/docs/source/getting_started/installation.rst b/docs/source/getting_started/installation.rst index f90a256b7e1..21e350fd8b4 100644 --- a/docs/source/getting_started/installation.rst +++ b/docs/source/getting_started/installation.rst @@ -146,6 +146,23 @@ Install LMCache BUILD_WITH_HIP=1 \ uv pip install -e . --no-build-isolation + .. tab-item:: Intel XPU + + .. code-block:: bash + + git clone https://github.com/LMCache/LMCache.git + cd LMCache + + uv venv --python 3.12 + source .venv/bin/activate + + # Need to install these packages manually to avoid build isolation + uv pip install -r requirements/build.txt + + # Build LMCache with SYCL backend. + BUILD_WITH_SYCL=1 uv pip install --no-build-isolation -e . + + .. tab-item:: Docker .. tab-set:: @@ -188,6 +205,12 @@ Install LMCache docker pull rocm/vllm-dev:nightly_0624_rc2_0624_rc2_20250620 + .. tab-item:: Intel XPU + + .. code-block:: bash + + docker pull docker pull intel/vllm:0.17.0-xpu + See :ref:`docker_deployment` for running the container and ROCm images. .. tab-item:: CLI Only @@ -214,7 +237,7 @@ Verify Installation Compatibility Matrix ~~~~~~~~~~~~~~~~~~~~ -✅ compatible · ❌ API incompatible · 🕯️ torch mismatch (use ``--no-build-isolation``) +✅ compatible · ❌ API incompatible · 🕯 torch mismatch (use ``--no-build-isolation``) .. container:: compat-table-scroll diff --git a/docs/source/production/docker_deployment.rst b/docs/source/production/docker_deployment.rst index f1e2dabeda5..2d97727ba0c 100644 --- a/docs/source/production/docker_deployment.rst +++ b/docs/source/production/docker_deployment.rst @@ -51,3 +51,21 @@ Validated environment: ``rocm/vllm-dev:nightly_0624_rc2_0624_rc2_20250620``, MI3 --name lmcache_rocm \ rocm/vllm-dev:nightly_0624_rc2_0624_rc2_20250620 \ bash + +XPU (Intel) +----------- + +The `Intel vLLM XPU hub`__ offers a prebuilt, optimized docker image designed for validating inference performance on the Intel GPU accelerators like ARC770, B60/B70, and future products. + +Validated environment: ``intel/vllm:0.17.0-xpu``, Intel B60 GPU, vLLM V1. + +.. code-block:: bash + + docker run --privileged \ + -it --rm --name vllm-xpu \ + -u root \ + --ipc=host --net=host \ + --cap-add=ALL \ + --device /dev/dri:/dev/dri \ + -v /dev/dri/by-path:/dev/dri/by-path \ + --entrypoint /bin/bash intel/vllm:0.17.0-xpu diff --git a/lmcache/__init__.py b/lmcache/__init__.py index 4c0e350cef0..066f4d78151 100644 --- a/lmcache/__init__.py +++ b/lmcache/__init__.py @@ -4,6 +4,7 @@ from typing import Any import importlib import sys +import types # First Party from lmcache.logging import init_logger @@ -61,10 +62,16 @@ def _get_backend() -> Any: """ Try backends in order, first successful import wins. """ + default_module = importlib.import_module("lmcache.non_cuda_equivalents") # Third Party import torch backend_candidates = [ + ( + "lmcache.xpu_ops", + "xpu_ops", + lambda: torch.xpu.is_available(), + ), ( "lmcache.c_ops", "cuda_ops", @@ -73,8 +80,6 @@ def _get_backend() -> Any: # should extend to more HWs.. ] - imported = False - module = None for module_name, backend_name, predicate in backend_candidates: # 1 Check whether the backend is available before importing try: @@ -93,21 +98,16 @@ def _get_backend() -> Any: continue # 2 Run availability check for the backend try: - module = importlib.import_module(module_name) + backend_module = importlib.import_module(module_name) + merged_module = types.ModuleType("lmcache.c_ops") + merged_module.__dict__.update(default_module.__dict__) + merged_module.__dict__.update(backend_module.__dict__) logger.info("Using backend: %s", module_name) - imported = True - break + return merged_module except Exception as e: logger.warning("Failed to import backend %s: %s", module_name, e) - if not imported: - try: - logger.warning("Fallback to python backend lmcache.non_cuda_equivalents") - module = importlib.import_module("lmcache.non_cuda_equivalents") - logger.info("Using backend: lmcache.non_cuda_equivalents") - except ImportError as e: - raise ImportError("No backend could be imported for lmcache.") from e - return module + return default_module # -------------------------- @@ -115,6 +115,10 @@ def _get_backend() -> Any: # -------------------------- try: _ops = _get_backend() + # override lmcache.c_ops with merged module, + # in which: + # non_cuda_equivalents as base, + # use backend implementation if exists sys.modules["lmcache.c_ops"] = _ops except (ImportError, ModuleNotFoundError): logger.debug("No compute backend loaded; CLI-only mode (torch/numba not installed)") diff --git a/lmcache/v1/gpu_connector/__init__.py b/lmcache/v1/gpu_connector/__init__.py index b15599a87cb..35b422197a5 100644 --- a/lmcache/v1/gpu_connector/__init__.py +++ b/lmcache/v1/gpu_connector/__init__.py @@ -75,39 +75,60 @@ def CreateGPUConnector( torch_dev.set_device(local_worker_id) device = torch.device(f"{torch_device_type}:{local_worker_id}") - if torch_device_type == "xpu": + if torch_device_type == "cuda": # First Party - from lmcache.v1.gpu_connector.xpu_connectors import ( - VLLMPagedMemLayerwiseXPUConnector, - VLLMPagedMemXPUConnectorV2, + from lmcache.v1.gpu_connector.gpu_connectors import ( + VLLMBufferLayerwiseGPUConnector, + VLLMPagedMemGPUConnectorV2, + VLLMPagedMemGPUConnectorV3, + VLLMPagedMemLayerwiseGPUConnector, ) if config.use_layerwise: - return VLLMPagedMemLayerwiseXPUConnector.from_metadata( - metadata, use_xpu=use_gpu, device=device - ) - return VLLMPagedMemXPUConnectorV2.from_metadata(metadata, use_gpu, device) + if config.enable_blending: + return VLLMBufferLayerwiseGPUConnector.from_metadata( + metadata, use_gpu, device, layout_hints=layout_hints + ) + else: + return VLLMPagedMemLayerwiseGPUConnector.from_metadata( + metadata, use_gpu, device, layout_hints=layout_hints + ) - if config.use_layerwise: - if config.enable_blending: - return VLLMBufferLayerwiseGPUConnector.from_metadata( + if config.use_gpu_connector_v3: + return VLLMPagedMemGPUConnectorV3.from_metadata( metadata, use_gpu, device, layout_hints=layout_hints ) else: - return VLLMPagedMemLayerwiseGPUConnector.from_metadata( + return VLLMPagedMemGPUConnectorV2.from_metadata( metadata, use_gpu, device, layout_hints=layout_hints ) + elif torch_device_type == "xpu": + # First Party + from lmcache.v1.gpu_connector.xpu_connectors import ( + VLLMBufferLayerwiseXPUConnector, + VLLMPagedMemLayerwiseXPUConnector, + VLLMPagedMemXPUConnectorV2, + VLLMPagedMemXPUConnectorV3, + ) + + if config.use_layerwise: + if config.enable_blending: + return VLLMBufferLayerwiseXPUConnector.from_metadata( + metadata, use_gpu, device + ) + else: + return VLLMPagedMemLayerwiseXPUConnector.from_metadata( + metadata, use_gpu, device + ) - elif torch_device_type == "cuda": if config.use_gpu_connector_v3: - return VLLMPagedMemGPUConnectorV3.from_metadata( - metadata, use_gpu, device, layout_hints=layout_hints + return VLLMPagedMemXPUConnectorV3.from_metadata( + metadata, use_gpu, device ) else: - return VLLMPagedMemGPUConnectorV2.from_metadata( - metadata, use_gpu, device, layout_hints=layout_hints + return VLLMPagedMemXPUConnectorV2.from_metadata( + metadata, use_gpu, device ) - elif torch_device_type == "hpu": # First Party from lmcache.v1.gpu_connector.hpu_connector import ( @@ -116,7 +137,7 @@ def CreateGPUConnector( return VLLMPagedMemHPUConnectorV2.from_metadata(metadata, use_gpu, device) else: - raise RuntimeError("No supported connector found for the current platform.") + raise RuntimeError(f"No supported {torch_device_type} connector found.") elif engine == EngineType.TRTLLM: # First Party diff --git a/lmcache/v1/gpu_connector/utils.py b/lmcache/v1/gpu_connector/utils.py index 75b2bb37d26..9fea963cf2e 100644 --- a/lmcache/v1/gpu_connector/utils.py +++ b/lmcache/v1/gpu_connector/utils.py @@ -277,24 +277,17 @@ def assert_layerwise_gpu_connector(gpu_connector: "GPUConnectorInterface"): """ # Import at runtime to avoid circular dependency # First Party - from lmcache.v1.gpu_connector.gpu_connectors import ( - SGLangLayerwiseGPUConnector, - VLLMBufferLayerwiseGPUConnector, - VLLMPagedMemLayerwiseGPUConnector, - ) - from lmcache.v1.gpu_connector.xpu_connectors import ( - VLLMPagedMemLayerwiseXPUConnector, + from lmcache.v1.gpu_connector import gpu_connectors, xpu_connectors + + valid_connectors = ( + gpu_connectors.VLLMPagedMemLayerwiseGPUConnector, + gpu_connectors.VLLMBufferLayerwiseGPUConnector, + gpu_connectors.SGLangLayerwiseGPUConnector, + xpu_connectors.VLLMPagedMemLayerwiseXPUConnector, + xpu_connectors.VLLMBufferLayerwiseXPUConnector, ) - assert isinstance( - gpu_connector, - ( - VLLMPagedMemLayerwiseGPUConnector, - VLLMBufferLayerwiseGPUConnector, - SGLangLayerwiseGPUConnector, - VLLMPagedMemLayerwiseXPUConnector, - ), - ) + assert isinstance(gpu_connector, valid_connectors) def get_gpu_kv_shape_description(gpu_kv_format: "lmc_ops.GPUKVFormat") -> str: diff --git a/lmcache/v1/gpu_connector/xpu_connectors.py b/lmcache/v1/gpu_connector/xpu_connectors.py index c93ac6c71e6..1e1a8155fd8 100644 --- a/lmcache/v1/gpu_connector/xpu_connectors.py +++ b/lmcache/v1/gpu_connector/xpu_connectors.py @@ -1,63 +1,33 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright 2024-2025 LMCache Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. # Standard -from typing import List, Optional, Union, cast -import os +from typing import List, Optional, Tuple, Union # Third Party import torch # First Party +from lmcache.integration.vllm.utils import ENGINE_NAME from lmcache.logging import init_logger -from lmcache.utils import EngineType -from lmcache.v1.gpu_connector.gpu_connectors import ( - GPUConnectorInterface, - VLLMPagedMemGPUConnectorV2, -) +from lmcache.utils import EngineType, _lmcache_nvtx_annotate +from lmcache.v1.compute.blend.utils import LMCBlenderBuilder +from lmcache.v1.gpu_connector.gpu_connectors import GPUConnectorInterface from lmcache.v1.gpu_connector.utils import ( - LayoutHints, - _get_head_size_view, - _split_token2d_kv, + assert_is_vllm_flash_attn_or_flash_infer, get_block_size, - get_dtype, - get_head_size, - get_hidden_dim_size, + get_elements_per_layer, get_num_blocks, - get_num_heads, - get_num_layers, - get_page_buffer_size, - is_mla, + get_tokens_per_layer, normalize_kv_and_discover_format, ) -from lmcache.v1.memory_management import ( - MemoryAllocatorInterface, - MemoryFormat, - MemoryObj, -) +from lmcache.v1.memory_management import GPUMemoryAllocator # noqa: E501 +from lmcache.v1.memory_management import MemoryFormat, MemoryObj from lmcache.v1.metadata import LMCacheMetadata +import lmcache.c_ops as lmc_ops logger = init_logger(__name__) -ALLOWED_FORMAT_TRANSITIONS = { - (None, MemoryFormat.KV_MLA_FMT), - (MemoryFormat.KV_MLA_FMT, MemoryFormat.KV_MLA_FMT), - (MemoryFormat.KV_T2D, MemoryFormat.KV_MLA_FMT), -} - -class VLLMPagedMemXPUConnectorV2(VLLMPagedMemGPUConnectorV2): +class VLLMPagedMemXPUConnectorV2(GPUConnectorInterface): """ The GPU KV cache should be a nested tuple of K and V tensors. More specifically, we have: @@ -70,12 +40,46 @@ class VLLMPagedMemXPUConnectorV2(VLLMPagedMemGPUConnectorV2): def __init__( self, + hidden_dim_size: int, + num_layers: int, use_gpu: bool = False, **kwargs, ): - self._attributes_initialized = False + """ + If use_gpu is true, it will create a gpu intermediate buffer. In this + case, it requires the following kwargs: + - chunk_size: The MAX size of the chunk to be copied to GPU. + - dtype: The data type of the intermediate buffer. + """ + self.hidden_dim_size = hidden_dim_size + self.num_layers = num_layers + self.kv_cache_pointers = torch.empty( + num_layers, dtype=torch.uint64, device="cpu" + ) + # Not sure we need a dict here. Maybe a single GPU connector always + # works with a single device? + self.kv_cache_pointers_on_gpu: dict[int, torch.Tensor] = {} + self.page_buffer_size = 0 + self.kvcaches: Optional[List[torch.Tensor]] = None - self.use_gpu = use_gpu + + self.gpu_buffer: Optional[torch.Tensor] = None + self.use_mla = "use_mla" in kwargs and kwargs["use_mla"] + if use_gpu: + assert "chunk_size" in kwargs, ( + "chunk_size should be provided to create a GPU buffer." + ) + assert "dtype" in kwargs, "dtype should be provided to create a GPU buffer." + assert "device" in kwargs, ( + "device should be provided to create a GPU buffer." + ) + shape = self.get_shape(kwargs["chunk_size"]) + self.gpu_buffer = torch.empty( + shape, dtype=kwargs["dtype"], device=kwargs["device"] + ) + + self.store_stream = torch.xpu.Stream() + self.load_stream = torch.xpu.Stream() @classmethod def from_metadata( @@ -83,7 +87,6 @@ def from_metadata( metadata: LMCacheMetadata, use_gpu: bool = False, device: Optional[torch.device] = None, - layout_hints: Optional[LayoutHints] = None, ) -> "VLLMPagedMemXPUConnectorV2": """Create a connector from LMCacheMetadata. @@ -91,16 +94,50 @@ def from_metadata( metadata: The LMCache engine metadata containing model configuration. use_gpu: Whether to use GPU intermediate buffer. device: The device to use for the connector. - layout_hints: Optional hints about KV cache layout from the - serving engine. Returns: A new instance of VLLMPagedMemXPUConnectorV2. """ + # Extract parameters from metadata + # kv_shape: (num_layer, 2 or 1, chunk_size, num_kv_head, head_size) + num_layers = metadata.kv_shape[0] + chunk_size = metadata.kv_shape[2] + num_kv_head = metadata.kv_shape[3] + head_size = metadata.kv_shape[4] + hidden_dim_size = num_kv_head * head_size + return cls( + hidden_dim_size=hidden_dim_size, + num_layers=num_layers, use_gpu=use_gpu, + chunk_size=chunk_size, + dtype=metadata.kv_dtype, + device=device, + use_mla=metadata.use_mla, ) + def _initialize_pointers(self, kv_caches: List[torch.Tensor]) -> torch.Tensor: + self.device = kv_caches[0].device + assert self.device.type == "xpu", "The device should be XPU." + idx = self.device.index + if idx in self.kv_cache_pointers_on_gpu: + return self.kv_cache_pointers_on_gpu[idx] + self.kv_cache_pointers.numpy()[:] = [t.data_ptr() for t in kv_caches] + self.kv_cache_pointers_on_gpu[idx] = torch.empty( + self.num_layers, dtype=torch.uint64, device=self.device + ) + self.kv_cache_pointers_on_gpu[idx].copy_(self.kv_cache_pointers) + + self.gpu_kv_format, kv_caches = normalize_kv_and_discover_format( + kv_caches, EngineType.VLLM + ) + self.num_blocks = get_num_blocks(kv_caches, self.gpu_kv_format) + self.block_size = get_block_size(kv_caches, self.gpu_kv_format) + self.page_buffer_size = self.num_blocks * self.block_size + + return self.kv_cache_pointers_on_gpu[idx] + + @_lmcache_nvtx_annotate def to_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): """Expect a kwarg 'kvcaches' which is a nested tuple of K and V tensors. The kvcaches should correspond to the "WHOLE token sequence". @@ -111,7 +148,7 @@ def to_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): 2. In the case that there is prefix caching, slot_mapping will starts with -1s until the end of the matched prefix. The start and end should NEVER overlap with the prefix caching (which means the - underlying CUDA kernel will never see -1 in slot_mapping) + underlying XPU kernel will never see -1 in slot_mapping) :raises ValueError: If 'kvcaches' is not provided in kwargs. @@ -126,36 +163,50 @@ def to_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): "kvcaches should be provided in kwargs or initialized beforehand." ) + if self.use_mla: + if memory_obj.metadata.fmt != MemoryFormat.KV_MLA_FMT: + raise ValueError( + "The memory object should be in KV_MLA_FMT format in" + " order to be processed by VLLMPagedMemXPUConnector" + ) + else: + if memory_obj.metadata.fmt != MemoryFormat.KV_2LTD: + raise ValueError( + "The memory object should be in KV_2LTD format in" + " order to be processed by VLLMPagedMemXPUConnector" + ) + if "slot_mapping" not in kwargs: raise ValueError("'slot_mapping' should be provided in kwargs.") slot_mapping: torch.Tensor = kwargs["slot_mapping"] - slices = slot_mapping[start:end] - self._initialize_attributes(self.kvcaches) - self._validate_memory_format(memory_obj) - if self.use_mla: - tmp = memory_obj.tensor[0].to(slot_mapping.device) - total_blocks = self.num_blocks * self.block_size - for i, kvcache in enumerate(self.kvcaches): - kvcache.view(total_blocks, self.head_size).index_copy_( - 0, slices, tmp[i] - ) - else: - tmp_k = memory_obj.tensor[0].to(slot_mapping.device) - tmp_v = memory_obj.tensor[1].to(slot_mapping.device) - total_blocks = self.num_blocks * self.block_size - d = self.num_heads * self.head_size - for i, (kcache, vcache) in enumerate(self.kvcaches): - kcache.view(total_blocks, d).index_copy_(0, slices, tmp_k[i]) - vcache.view(total_blocks, d).index_copy_(0, slices, tmp_v[i]) + kv_cache_pointers = self._initialize_pointers(self.kvcaches) + # avoid read/write stream race condition for shared block + # this will only be potentially non-zero for the first + # block lmcache is transferring back + vllm_cached = kwargs.get("vllm_cached_tokens", 0) + skip_prefix_n_tokens = min(end - start, max(0, vllm_cached - start)) + + lmc_ops.multi_layer_kv_transfer( + memory_obj.tensor, + kv_cache_pointers, + slot_mapping[start:end], + self.device, + self.page_buffer_size, + lmc_ops.TransferDirection.H2D, + self.gpu_kv_format, + self.block_size, + skip_prefix_n_tokens, + ) + + @_lmcache_nvtx_annotate def from_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): """Expect a kwarg 'kvcaches' which is a nested tuple of K and V tensors. The kvcaches should correspond to the "WHOLE token sequence". - Will set the memory_obj.metadata.fmt to MemoryFormat.KV_MLA_FMT - if use_mla is True. + Will set the memory_obj.metadata.fmt to MemoryFormat.KV_2LTD. Note: 1. This function expects the 'slot_mapping' is a "full slot mapping" @@ -163,7 +214,7 @@ def from_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): 2. In the case that there is prefix caching, slot_mapping will starts with -1s until the end of the matched prefix. The start and end should NEVER overlap with the prefix caching (which means the - underlying CUDA kernel will never see -1 in slot_mapping) + underlying XPU kernel will never see -1 in slot_mapping) :raises ValueError: If 'kvcaches' is not provided in kwargs, :raises AssertionError: If the memory object does not have a tensor. @@ -180,254 +231,786 @@ def from_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): raise ValueError("'slot_mapping' should be provided in kwargs.") slot_mapping: torch.Tensor = kwargs["slot_mapping"] - slices = slot_mapping[start:end] - self._initialize_attributes(self.kvcaches) - self._validate_memory_format(memory_obj) + + kv_cache_pointers = self._initialize_pointers(self.kvcaches) + + with torch.xpu.stream(self.store_stream): + if self.gpu_buffer is None or end - start != self.gpu_buffer.shape[2]: + lmc_ops.multi_layer_kv_transfer( + memory_obj.tensor, + kv_cache_pointers, + slot_mapping[start:end], + self.kvcaches[0].device, + self.page_buffer_size, + lmc_ops.TransferDirection.D2H, + self.gpu_kv_format, + self.block_size, + ) + else: + # kvcaches -> gpu_buffer -> memobj + assert self.gpu_buffer.device == self.kvcaches[0].device + tmp_gpu_buffer = self.gpu_buffer[:, :, : end - start, :] + lmc_ops.multi_layer_kv_transfer( + tmp_gpu_buffer, + kv_cache_pointers, + slot_mapping[start:end], + self.kvcaches[0].device, + self.page_buffer_size, + lmc_ops.TransferDirection.D2H, + self.gpu_kv_format, + self.block_size, + ) + memory_obj.tensor.copy_(tmp_gpu_buffer, non_blocking=True) + + if not memory_obj.tensor.is_xpu: + # Force a synchronize if the target buffer is NOT XPU device + # NOTE: for better performance, we may not want to sync for every + # memory object + self.store_stream.synchronize() if self.use_mla: - total_blocks = self.num_blocks * self.block_size - tmp = torch.stack( - [ - kvcache.view(total_blocks, self.head_size).index_select(0, slices) - for kvcache in self.kvcaches - ] + memory_obj.metadata.fmt = MemoryFormat.KV_MLA_FMT + + # TODO(Jiayi): need to optimize to enable real batching + def batched_to_gpu(self, memory_objs, starts, ends, **kwargs): + with torch.xpu.stream(self.load_stream): + for memory_obj, start, end in zip(memory_objs, starts, ends, strict=False): + self.to_gpu(memory_obj, start, end, **kwargs) + self.load_stream.synchronize() + + # TODO(Jiayi): need to optimize to enable real batching + def batched_from_gpu(self, memory_objs, starts, ends, **kwargs): + for memory_obj, start, end in zip(memory_objs, starts, ends, strict=False): + self.from_gpu(memory_obj, start, end, **kwargs) + + def get_shape(self, num_tokens: int) -> torch.Size: + kv_size = 1 if self.use_mla else 2 + return torch.Size([kv_size, self.num_layers, num_tokens, self.hidden_dim_size]) + + +class VLLMPagedMemXPUConnectorV3(GPUConnectorInterface): + def __init__( + self, + metadata: LMCacheMetadata, + device: torch.device, + use_gpu: bool = False, + ): + assert device.type == "xpu", "The device should be XPU." + self.metadata = metadata + self.device = device + self.use_mla = metadata.use_mla + self.chunk_size = metadata.chunk_size + self.use_gpu = use_gpu + self.kvcaches: Optional[List[torch.Tensor]] = None + self.page_buffer_size = 0 + + self.init = False + self.group_kv_cache_pointers_on_gpu: Optional[list[torch.Tensor]] = None + self.group_tmp_buffer: Optional[list[torch.Tensor]] = None + + self.store_stream = torch.xpu.Stream() + self.load_stream = torch.xpu.Stream() + + @classmethod + def from_metadata( + cls, + metadata: LMCacheMetadata, + use_gpu: bool = False, + device: Optional[torch.device] = None, + ) -> "VLLMPagedMemXPUConnectorV3": + assert device is not None + return cls(metadata, device, use_gpu) + + def _initialize_kv_cache_pointers(self): + if self.init: + return + assert self.metadata.kv_layer_groups_manager.kv_layer_groups + if self.use_gpu: + # init tmp buffer + tmp_buf_shapes = self.metadata.get_shapes(self.chunk_size) + tmp_buf_dtypes = self.metadata.get_dtypes() + assert len(tmp_buf_shapes) == len(tmp_buf_dtypes) + self.group_tmp_buffer = [ + torch.empty(tmp_buf_shape, dtype=tmp_buf_dtype, device=self.device) + for tmp_buf_shape, tmp_buf_dtype in zip( + tmp_buf_shapes, tmp_buf_dtypes, strict=True + ) + ] + self.group_kv_cache_pointers_on_gpu = [] + for group in self.metadata.kv_layer_groups_manager.kv_layer_groups: + # init kv cache pointers + num_layers = group.num_layers + kv_cache_pointers = torch.empty( + num_layers, dtype=torch.uint64, device="cpu" ) - else: - total_blocks = self.num_blocks * self.block_size - d = self.num_heads * self.head_size - tmp_k = torch.stack( - [ - kvcache[0].view(total_blocks, d).index_select(0, slices) - for kvcache in self.kvcaches - ] + kv_cache_pointers.numpy()[:] = [ + t.data_ptr() + for i, t in enumerate(self.kvcaches) + if i in group.layer_indices + ] + kv_cache_pointers_on_gpu = torch.empty( + num_layers, dtype=torch.uint64, device=self.device ) - tmp_v = torch.stack( - [ - kvcache[1].view(total_blocks, d).index_select(0, slices) - for kvcache in self.kvcaches - ] + kv_cache_pointers_on_gpu.copy_(kv_cache_pointers) + self.group_kv_cache_pointers_on_gpu.append(kv_cache_pointers_on_gpu) + + self.gpu_kv_format, self.kvcaches = normalize_kv_and_discover_format( + self.kvcaches, EngineType.VLLM + ) + self.num_blocks = get_num_blocks(self.kvcaches, self.gpu_kv_format) + self.block_size = get_block_size(self.kvcaches, self.gpu_kv_format) + self.page_buffer_size = self.num_blocks * self.block_size + + self.init = True + logger.info("init kv cache pointers success in VLLMPagedMemXPUConnectorV3") + + @_lmcache_nvtx_annotate + def to_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): + assert memory_obj.raw_tensor is not None + assert "slot_mapping" in kwargs + if self.use_mla: + assert memory_obj.metadata.fmt == MemoryFormat.KV_MLA_FMT + else: + assert memory_obj.metadata.fmt == MemoryFormat.KV_2LTD + + slot_mapping: torch.Tensor = kwargs["slot_mapping"] + self.initialize_kvcaches_ptr(**kwargs) + assert self.kvcaches is not None + assert self.kvcaches[0].device == self.device + self._initialize_kv_cache_pointers() + assert self.group_kv_cache_pointers_on_gpu is not None + + # avoid read/write stream race condition for shared block + # this will only be potentially non-zero for the first + # block lmcache is transferring back + vllm_cached = kwargs.get("vllm_cached_tokens", 0) + skip_prefix_n_tokens = min(end - start, max(0, vllm_cached - start)) + + for i, kv_cache_pointer in enumerate(self.group_kv_cache_pointers_on_gpu): + memory_obj_tensor = memory_obj.get_tensor(i) + assert memory_obj_tensor is not None + lmc_ops.multi_layer_kv_transfer( + memory_obj_tensor, + kv_cache_pointer, + slot_mapping[start:end], + self.device, + self.page_buffer_size, + lmc_ops.TransferDirection.H2D, + self.gpu_kv_format, + self.block_size, + skip_prefix_n_tokens, ) - tmp = torch.stack([tmp_k, tmp_v]) - memory_obj.tensor.copy_(tmp, non_blocking=True) - if not memory_obj.tensor.is_xpu: + @_lmcache_nvtx_annotate + def from_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): + assert memory_obj.raw_tensor is not None + assert "slot_mapping" in kwargs + + slot_mapping: torch.Tensor = kwargs["slot_mapping"] + self.initialize_kvcaches_ptr(**kwargs) + assert self.kvcaches is not None + assert self.kvcaches[0].device == self.device + self._initialize_kv_cache_pointers() + assert self.group_kv_cache_pointers_on_gpu is not None + with torch.xpu.stream(self.store_stream): + if not self.use_gpu or end - start != self.chunk_size: + for i, kv_cache_pointer in enumerate( + self.group_kv_cache_pointers_on_gpu + ): + memory_obj_tensor = memory_obj.get_tensor(i) + assert memory_obj_tensor is not None + lmc_ops.multi_layer_kv_transfer( + memory_obj_tensor, + kv_cache_pointer, + slot_mapping[start:end], + self.device, + self.page_buffer_size, + lmc_ops.TransferDirection.D2H, + self.gpu_kv_format, + self.block_size, + ) + else: + # kvcaches -> gpu_buffer -> memobj + assert self.group_tmp_buffer is not None + for i, kv_cache_pointer in enumerate( + self.group_kv_cache_pointers_on_gpu + ): + tmp_gpu_buffer = self.group_tmp_buffer[i][:, :, : end - start, :] + lmc_ops.multi_layer_kv_transfer( + tmp_gpu_buffer, + kv_cache_pointer, + slot_mapping[start:end], + self.device, + self.page_buffer_size, + lmc_ops.TransferDirection.D2H, + self.gpu_kv_format, + self.block_size, + ) + memory_obj_tensor = memory_obj.get_tensor(i) + assert memory_obj_tensor is not None + memory_obj_tensor.copy_(tmp_gpu_buffer, non_blocking=True) + + if not memory_obj.raw_tensor.is_xpu: # Force a synchronize if the target buffer is NOT XPU device # NOTE: for better performance, we may not want to sync for every # memory object - torch.xpu.synchronize() + self.store_stream.synchronize() if self.use_mla: memory_obj.metadata.fmt = MemoryFormat.KV_MLA_FMT - # TODO(Jiayi): need to optimize to enable real batching def batched_to_gpu(self, memory_objs, starts, ends, **kwargs): + with torch.xpu.stream(self.load_stream): + for memory_obj, start, end in zip(memory_objs, starts, ends, strict=False): + self.to_gpu(memory_obj, start, end, **kwargs) + self.load_stream.synchronize() + + def batched_from_gpu(self, memory_objs, starts, ends, **kwargs): for memory_obj, start, end in zip(memory_objs, starts, ends, strict=False): - self.to_gpu(memory_obj, start, end, **kwargs) + self.from_gpu(memory_obj, start, end, **kwargs) def get_shape(self, num_tokens: int) -> torch.Size: - """Get the shape of the data given the number of tokens. + raise NotImplementedError + + +class VLLMBufferLayerwiseXPUConnector(GPUConnectorInterface): + def __init__( + self, + hidden_dim_size: int, + num_layers: int, + use_gpu: bool = False, + use_double_buffer: bool = True, + **kwargs, + ): + self.hidden_dim_size = hidden_dim_size + self.num_layers = num_layers + + self.kvcaches: Optional[List[torch.Tensor]] = None + + # TODO(Jiayi): remove this hardcode + self.cache_positions = True + + self.fused_rotary_emb = None + + assert use_gpu, "use_gpu must be true in VLLMBufferLayerwiseXPUConnector" + assert "dtype" in kwargs, "dtype should be provided to create a GPU buffer." + assert "device" in kwargs, "device should be provided to create a GPU buffer." + + self.dtype = kwargs["dtype"] + self.device = kwargs["device"] + + self.load_stream = torch.xpu.Stream() + self.store_stream = torch.xpu.Stream() + + self.buffer_mapping: dict[int, MemoryObj] = {} + + # track gap positions between blended chunks + self.current_gap_positions = None + + self.use_gpu = use_gpu + self.gpu_buffer_allocator = None + self.element_size = torch.tensor([], dtype=self.dtype).element_size() + + @classmethod + def from_metadata( + cls, + metadata: LMCacheMetadata, + use_gpu: bool = False, + device: Optional[torch.device] = None, + ) -> "VLLMBufferLayerwiseXPUConnector": + """Create a connector from LMCacheMetadata. Args: - num_tokens: The number of tokens in the data. + metadata: The LMCache engine metadata containing model configuration. + use_gpu: Whether to use GPU intermediate buffer. + device: The device to use for the connector. Returns: - The shape of the KV cache data. + A new instance of VLLMBufferLayerwiseXPUConnector. + """ + # Extract parameters from metadata + # kv_shape: (num_layer, 2 or 1, chunk_size, num_kv_head, head_size) + num_layers = metadata.kv_shape[0] + num_kv_head = metadata.kv_shape[3] + head_size = metadata.kv_shape[4] + hidden_dim_size = num_kv_head * head_size - Raises: - RuntimeError: If attributes have not been initialized yet - (i.e., no kv_caches have been seen). + return cls( + hidden_dim_size=hidden_dim_size, + num_layers=num_layers, + use_gpu=use_gpu, + dtype=metadata.kv_dtype, + device=device, + ) + + def _lazy_initialize_buffer(self, kv_caches): + """ + Lazily initialize the GPU buffer allocator if it is not initialized yet. + Currently, we use the `kv_caches` (kv cache pointer) to determine + the gpu buffer size in gpu connector. + Also, the first request might be a bit slower due to buffer creation. """ - if not self._attributes_initialized: - raise RuntimeError( - "Cannot determine shape before attributes are initialized. " - "Call to_gpu or from_gpu first so that _initialize_attributes " - "can discover the KV cache layout." + if self.use_gpu and self.gpu_buffer_allocator is None: + logger.info("Lazily initializing GPU buffer.") + # NOTE (Jiayi): We use the first layer to determine the gpu buffer size. + # NOTE (Jiayi): Using the exact number of tokens in the first layer + # is okay since fragmentation shouldn't exist in the `gpu_buffer_allocator` + # in layerwise mode. + + self.gpu_kv_format, kv_caches = normalize_kv_and_discover_format( + kv_caches, EngineType.VLLM + ) + assert_is_vllm_flash_attn_or_flash_infer(self.gpu_kv_format) + self.tokens_per_layer = get_tokens_per_layer(kv_caches, self.gpu_kv_format) + self.elements_per_layer = get_elements_per_layer( + kv_caches, self.gpu_kv_format + ) + logger.info( + f"Lazily initializing GPU buffer (max tokens={self.tokens_per_layer})." + ) + gpu_buffer_size = self.elements_per_layer * self.element_size + self.gpu_buffer_allocator = GPUMemoryAllocator( + gpu_buffer_size, device=self.device ) - kv_size = 1 if self.use_mla else 2 - return torch.Size([kv_size, self.num_layers, num_tokens, self.hidden_dim_size]) - def _validate_memory_format(self, memory_obj: MemoryObj) -> None: - """Validate that the memory object has the expected format. + def get_kv(self, layer_id: int) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Get the KV cache for the given layer ID. + This function is used to get the KV cache from the GPU buffer. + """ + if layer_id not in self.buffer_mapping: + raise ValueError(f"Layer {layer_id} is not loaded into GPU buffer.") - Args: - memory_obj: The memory object to validate. + gpu_buffer = self.buffer_mapping[layer_id].tensor + assert gpu_buffer is not None + return gpu_buffer[0], gpu_buffer[1] + + def to_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): + """ """ + + raise NotImplementedError - Raises: - ValueError: If the memory format does not match the expected - format based on whether MLA is in use. + def from_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): + """ """ + + raise NotImplementedError + + @_lmcache_nvtx_annotate + def batched_to_gpu(self, starts: List[int], ends: List[int], **kwargs): """ - if self.use_mla: - if memory_obj.metadata.fmt != MemoryFormat.KV_MLA_FMT: - raise ValueError( - "The memory object should be in KV_MLA_FMT format in" - " order to be processed by VLLMPagedMemXPUConnectorV2" + This function is a generator that moves the KV cache from the memory + objects to buffer GPU memory. In each iteration i, it (1) loads the KV + cache of layer i from CPU -> GPU buffer, (2) recovers the positional + encoding of the layer i-1's KV cache in the GPU buffer, and (3) + moves the KV cache of layer i-2 from GPU buffer to paged GPU memory. + In total, this the generator will yield num_layers + 2 times. + + :param starts: The starting indices of the KV cache in the corresponding + token sequence. + + :param ends: The ending indices of the KV cache in the corresponding + token sequence. + """ + + self.initialize_kvcaches_ptr(**kwargs) + assert self.kvcaches is not None, ( + "kvcaches should be provided in kwargs or initialized beforehand." + ) + + if "slot_mapping" not in kwargs: + raise ValueError("'slot_mapping' should be provided in kwargs.") + + if self.fused_rotary_emb is None and self.cache_positions: + # TODO(Jiayi): Make this more elegant + self.lmc_model = LMCBlenderBuilder.get(ENGINE_NAME).layerwise_model + self.fused_rotary_emb = self.lmc_model.fused_rotary_emb + + slot_mapping: torch.Tensor = kwargs["slot_mapping"] + + self._lazy_initialize_buffer(self.kvcaches) + + num_all_tokens = ends[-1] - starts[0] + slot_mapping_full = slot_mapping[starts[0] : ends[-1]] + + # compute gap positions + gap_mask = torch.ones( + num_all_tokens, dtype=torch.bool, device=slot_mapping_full.device + ) + buf_offset = starts[0] + + for start, end in zip(starts, ends, strict=False): + gap_mask[start - buf_offset : end - buf_offset] = False + + self.current_gap_positions = torch.where(gap_mask)[0] + + buf_offset = starts[0] + if self.cache_positions: + new_positions_full = torch.arange( + starts[0], ends[-1], dtype=torch.int64, device=self.kvcaches[0].device + ) + + buffer_shape = self.get_shape(num_all_tokens) + assert self.gpu_buffer_allocator is not None + compute_gpu_buffer_obj = self.gpu_buffer_allocator.allocate( + buffer_shape, self.dtype, MemoryFormat.KV_2TD + ) + load_gpu_buffer_obj = self.gpu_buffer_allocator.allocate( + buffer_shape, self.dtype, MemoryFormat.KV_2TD + ) + assert compute_gpu_buffer_obj is not None, ( + "Failed to allocate GPU buffer in GPUConnector" + ) + assert load_gpu_buffer_obj is not None, ( + "Failed to allocate GPU buffer in GPUConnector" + ) + assert compute_gpu_buffer_obj.tensor is not None + assert load_gpu_buffer_obj.tensor is not None + + # current_stream = torch.xpu.current_stream() + + if self.cache_positions: + old_positions_full = torch.zeros( + (num_all_tokens,), dtype=torch.int64, device=self.kvcaches[0].device + ) + for layer_id in range(self.num_layers + 2): + if layer_id > 1: + lmc_ops.single_layer_kv_transfer( + self.buffer_mapping[layer_id - 2].tensor, + self.kvcaches[layer_id - 2], + slot_mapping_full, + lmc_ops.TransferDirection.H2D, + self.gpu_kv_format, + token_major=False, # shape is [2, num_tokens, hidden_dim] ) - else: - if memory_obj.metadata.fmt != MemoryFormat.KV_2LTD: - raise ValueError( - "The memory object should be in KV_2LTD format in" - " order to be processed by VLLMPagedMemXPUConnectorV2" + del self.buffer_mapping[layer_id - 2] + + logger.debug(f"Finished loading layer {layer_id - 2} into paged memory") + + if layer_id > 0 and layer_id <= self.num_layers: + # NOTE: wait until both compute and load streams are done + torch.xpu.synchronize() + + # ping-pong the buffers + compute_gpu_buffer_obj, load_gpu_buffer_obj = ( + load_gpu_buffer_obj, + compute_gpu_buffer_obj, ) - def _initialize_attributes(self, kv_caches: List[torch.Tensor]): - """Initialize attributes from the kv_caches using utils functions. + if self.cache_positions: + assert compute_gpu_buffer_obj.tensor is not None - Uses format discovery and utility functions from utils.py to - extract all KV cache parameters lazily on first use. + compute_gpu_buffer_obj.tensor[0] = self.fused_rotary_emb( + old_positions_full, + new_positions_full, + compute_gpu_buffer_obj.tensor[0], + ) - Args: - kv_caches: The KV cache tensors from which to discover - the cache layout and parameters. - """ - if self._attributes_initialized: - return + # gap zeroing after RoPE + if self.current_gap_positions.numel(): + compute_gpu_buffer_obj.tensor[:, self.current_gap_positions] = 0.0 - self.device = kv_caches[0].device - assert self.device.type == "xpu", "The device should be XPU." + self.buffer_mapping[layer_id - 1] = compute_gpu_buffer_obj - self.gpu_kv_format, kv_caches = normalize_kv_and_discover_format( - kv_caches, EngineType.VLLM + logger.debug(f"Finished loading layer {layer_id - 1} into buffer") + + if layer_id < self.num_layers: + memory_objs_layer = yield + + # memobj -> gpu_buffer + with torch.xpu.stream(self.load_stream): + for start, end, memory_obj in zip( + starts, ends, memory_objs_layer, strict=False + ): + assert memory_obj.metadata.fmt == MemoryFormat.KV_2TD + assert load_gpu_buffer_obj.tensor is not None + load_gpu_buffer_obj.tensor[0][ + start - buf_offset : end - buf_offset + ].copy_(memory_obj.tensor[0], non_blocking=True) + + load_gpu_buffer_obj.tensor[1][ + start - buf_offset : end - buf_offset + ].copy_(memory_obj.tensor[1], non_blocking=True) + + if self.cache_positions and layer_id == 0: + old_positions_full[ + start - buf_offset : end - buf_offset + ] = memory_obj.metadata.cached_positions + + elif layer_id == self.num_layers: + yield + + # free the buffer memory + load_gpu_buffer_obj.ref_count_down() + compute_gpu_buffer_obj.ref_count_down() + + assert len(self.buffer_mapping) == 0, ( + "There are still layers in the buffer mapping after " + "releasing the GPU buffers." ) - self.num_layers = get_num_layers(kv_caches, self.gpu_kv_format) - self.num_blocks = get_num_blocks(kv_caches, self.gpu_kv_format) - self.block_size = get_block_size(kv_caches, self.gpu_kv_format) - self.page_buffer_size = get_page_buffer_size(kv_caches, self.gpu_kv_format) - self.hidden_dim_size = get_hidden_dim_size(kv_caches, self.gpu_kv_format) - self.head_size = get_head_size(kv_caches, self.gpu_kv_format) - self.use_mla = is_mla(self.gpu_kv_format) - self.dtype = get_dtype(kv_caches, self.gpu_kv_format) - self.num_heads = ( - 1 if self.use_mla else get_num_heads(kv_caches, self.gpu_kv_format) + + yield + + # TODO(Jiayi): Reduce repetitive operations in `batched_to_gpu` + # and `batched_from_gpu`. + @_lmcache_nvtx_annotate + def batched_from_gpu( + self, + memory_objs: Union[List[List[MemoryObj]], List[MemoryObj]], + starts: List[int], + ends: List[int], + **kwargs, + ): + """ + This function is a generator that moves the KV cache from the paged GPU + memory to the memory objects. The first iteration will prepare some + related metadata and initiate the transfer in the first layer. In each + of the following iterations, it will first wait until the storing of + previous layer finishes, and then initiate string the KV cache of the + current layer one. The storing process of the KV cache is paged GPU + memory -> GPU buffer -> memory objects. The last iteration simply waits + for the last layer to finish. + In total, this the generator will yield num_layers + 1 times. + + :param memory_objs: The memory objects to store the KV cache. The first + dimension is the number of layers, and the second dimension is the + number of memory objects (i.e., number of chunks) for each layer. + + :param starts: The starting indices of the KV cache in the corresponding + token sequence. + + :param ends: The ending indices of the KV cache in the corresponding + token sequence. + + :raises ValueError: If 'kvcaches' is not provided in kwargs. + + :raises ValueError: If 'slot_mapping' is not provided in kwargs. + """ + + self.initialize_kvcaches_ptr(**kwargs) + assert self.kvcaches is not None, ( + "kvcaches should be provided in kwargs or initialized beforehand." ) - self._attributes_initialized = True - logger.info( - "XPU: attributes initialized - format: %s, " - "num_layers: %d, num_blocks: %d, block_size: %d, " - "page_buffer_size: %d, hidden_dim_size: %d, head_size: %d, " - "use_mla: %s, dtype: %s, num_heads: %d", - self.gpu_kv_format, - self.num_layers, - self.num_blocks, - self.block_size, - self.page_buffer_size, - self.hidden_dim_size, - self.head_size, - self.use_mla, - self.dtype, - self.num_heads, + if "slot_mapping" not in kwargs: + raise ValueError("'slot_mapping' should be provided in kwargs.") + + slot_mapping: torch.Tensor = kwargs["slot_mapping"] + + self._lazy_initialize_buffer(self.kvcaches) + + buf_start = 0 + slot_mapping_chunks = [] + buf_starts_ends = [] + old_positions_chunks = [] + for start, end in zip(starts, ends, strict=False): + buf_end = buf_start + end - start + buf_starts_ends.append((buf_start, buf_end)) + slot_mapping_chunks.append(slot_mapping[start:end]) + buf_start = buf_end + if self.cache_positions: + old_positions_chunks.append( + torch.arange( + start, end, device=self.kvcaches[0].device, dtype=torch.int64 + ) + ) + + slot_mapping_full = torch.cat(slot_mapping_chunks, dim=0) + + num_tokens = len(slot_mapping_full) + buffer_shape = self.get_shape(num_tokens) + assert self.gpu_buffer_allocator is not None + tmp_gpu_buffer_obj = self.gpu_buffer_allocator.allocate( + buffer_shape, self.dtype, MemoryFormat.KV_2TD ) + assert tmp_gpu_buffer_obj is not None, ( + "Failed to allocate GPU buffer in GPUConnector" + ) + assert tmp_gpu_buffer_obj.tensor is not None + current_stream = torch.xpu.current_stream() -class VLLMPagedMemLayerwiseXPUConnector(GPUConnectorInterface): - """ - Layerwise paged KV connector for XPU. + for layer_id in range(self.num_layers): + memory_objs_layer = memory_objs[layer_id] + # kvcaches -> gpu_buffer -> memobj + with torch.xpu.stream(self.store_stream): + self.store_stream.wait_stream(current_stream) + lmc_ops.single_layer_kv_transfer( + tmp_gpu_buffer_obj.tensor, + self.kvcaches[layer_id], + slot_mapping_full, + lmc_ops.TransferDirection.D2H, + self.gpu_kv_format, + token_major=False, # shape is [2, num_tokens, hidden_dim] + ) + for (buf_start, buf_end), memory_obj, old_positions in zip( + buf_starts_ends, + memory_objs_layer, + old_positions_chunks, + strict=False, + ): + assert memory_obj.tensor is not None + memory_obj.tensor[0].copy_( + tmp_gpu_buffer_obj.tensor[0][buf_start:buf_end], + non_blocking=True, + ) + memory_obj.tensor[1].copy_( + tmp_gpu_buffer_obj.tensor[1][buf_start:buf_end], + non_blocking=True, + ) + if self.cache_positions: + memory_obj.metadata.cached_positions = old_positions - Implements the *same generator contract* as VLLMPagedMemLayerwiseGPUConnector: - - batched_to_gpu(...) yields num_layers + 2 times - - batched_from_gpu(...) yields num_layers + 1 times + yield + self.store_stream.synchronize() + logger.debug(f"Finished offloading layer {layer_id}") - Transfer is implemented with pure torch ops (index_copy_/index_select). - """ + # free the buffer memory + tmp_gpu_buffer_obj.ref_count_down() + yield + + def get_shape(self, num_tokens: int) -> torch.Size: + return torch.Size([2, num_tokens, self.hidden_dim_size]) + + +class VLLMPagedMemLayerwiseXPUConnector(GPUConnectorInterface): + """ """ def __init__( self, hidden_dim_size: int, num_layers: int, - use_xpu: bool = False, + use_gpu: bool = False, **kwargs, ): self.hidden_dim_size = hidden_dim_size self.num_layers = num_layers - self.use_xpu = use_xpu + self.use_gpu = use_gpu + + self.gpu_buffer_allocator = None - assert "chunk_size" in kwargs, "chunk_size should be provided." - assert "dtype" in kwargs, "dtype should be provided." - assert "device" in kwargs, "device should be provided." + assert "chunk_size" in kwargs, ( + "chunk_size should be provided to create a GPU buffer." + ) + assert "dtype" in kwargs, "dtype should be provided to create a GPU buffer." + assert "device" in kwargs, "device should be provided to create a GPU buffer." self.dtype = kwargs["dtype"] self.device = kwargs["device"] - self.use_mla = "use_mla" in kwargs and kwargs["use_mla"] self.kvcaches: Optional[List[torch.Tensor]] = None - # XPU streams + # All sizes are in bytes + self.element_size = torch.tensor([], dtype=self.dtype).element_size() + self.load_stream = torch.xpu.Stream() self.store_stream = torch.xpu.Stream() - # Optional device staging buffer allocator (same pattern as CUDA class) - self.gpu_buffer_allocator: Optional[MemoryAllocatorInterface] = None + self.use_mla = "use_mla" in kwargs and kwargs["use_mla"] @classmethod def from_metadata( cls, metadata: LMCacheMetadata, - use_xpu: bool = False, + use_gpu: bool = False, device: Optional[torch.device] = None, ) -> "VLLMPagedMemLayerwiseXPUConnector": + """Create a connector from LMCacheMetadata. + + Args: + metadata: The LMCache engine metadata containing model configuration. + use_gpu: Whether to use GPU intermediate buffer. + device: The device to use for the connector. + + Returns: + A new instance of VLLMPagedMemLayerwiseXPUConnector. + """ + # Extract parameters from metadata + # kv_shape: (num_layer, 2 or 1, chunk_size, num_kv_head, head_size) num_layers = metadata.kv_shape[0] + chunk_size = metadata.kv_shape[2] num_kv_head = metadata.kv_shape[3] head_size = metadata.kv_shape[4] hidden_dim_size = num_kv_head * head_size + return cls( hidden_dim_size=hidden_dim_size, num_layers=num_layers, - use_xpu=use_xpu, - chunk_size=metadata.kv_shape[2], + use_gpu=use_gpu, + chunk_size=chunk_size, dtype=metadata.kv_dtype, device=device, use_mla=metadata.use_mla, ) - def _validate_format_transition(self, mem, target_fmt): - current_fmt = mem.metadata.fmt - - if (current_fmt, target_fmt) not in ALLOWED_FORMAT_TRANSITIONS: - raise ValueError( - f"Invalid KV format transition: {current_fmt} -> {target_fmt}" + def _lazy_initialize_buffer(self, kv_caches): + """ + Lazily initialize the GPU buffer allocator if it is not initialized yet. + Currently, we use the `kv_caches` (kv cache pointer) to determine + the gpu buffer size in gpu connector. + Also, the first request might be a bit slower due to buffer creation. + """ + if self.use_gpu and self.gpu_buffer_allocator is None: + logger.info("Lazily initializing GPU buffer.") + # NOTE (Jiayi): We use the first layer to determine the gpu buffer size. + # NOTE (Jiayi): Using the exact number of tokens in the first layer + # is okay since fragmentation shouldn't exist in the `gpu_buffer_allocator` + # in layerwise mode. + + self.gpu_kv_format, kv_caches = normalize_kv_and_discover_format( + kv_caches, EngineType.VLLM ) - - def _lazy_initialize_buffer(self, kv_caches: List[torch.Tensor]) -> None: - # Buffer allocator only needed when use_xpu=True (device staging) - if self.use_xpu and self.gpu_buffer_allocator is None: - # First Party - from lmcache.v1.memory_management import XPUMemoryAllocator - - # Derive size from first layer KV tensor - layer0 = kv_caches[0] - derived_bytes = layer0.numel() * layer0.element_size() - - # Allow override via env variable - staging_bytes = int( - os.getenv("LMCACHE_GPU_STAGING_BUFFER_BYTES", derived_bytes) + assert_is_vllm_flash_attn_or_flash_infer(self.gpu_kv_format) + self.tokens_per_layer = get_tokens_per_layer(kv_caches, self.gpu_kv_format) + self.elements_per_layer = get_elements_per_layer( + kv_caches, self.gpu_kv_format ) - logger.info( - "Initializing staging buffer (derived=%d bytes, final=%d bytes)", - derived_bytes, - staging_bytes, + f"Lazily initializing GPU buffer (max tokens={self.tokens_per_layer})." ) - - self.gpu_buffer_allocator = XPUMemoryAllocator( - size=staging_bytes, - device=self.device, + gpu_buffer_size = self.elements_per_layer * self.element_size + self.gpu_buffer_allocator = GPUMemoryAllocator( + gpu_buffer_size, device=self.device ) def to_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): - raise NotImplementedError("Layerwise uses batched_to_gpu(generator).") + """ """ + + raise NotImplementedError def from_gpu(self, memory_obj: MemoryObj, start: int, end: int, **kwargs): - raise NotImplementedError("Layerwise uses batched_from_gpu(generator).") + """ """ + + raise NotImplementedError - def _batched_to_gpu_gen(self, starts: List[int], ends: List[int], **kwargs): + @_lmcache_nvtx_annotate + def batched_to_gpu(self, starts: List[int], ends: List[int], **kwargs): """ - Generator: CPU token2d -> (optional XPU staging) -> XPU paged KV (per layer). + This function is a generator that moves the KV cache from the memory + objects to paged GPU memory. The first iteration will prepare some + related metadata. In each of the following iterations, it will first + wait until the loading of the previous layer finish, and then load + one layer of KV cache from the memory objects -> GPU buffer -> + paged GPU memory. The last iteration simply waits for the last layer + to finish. + In total, this the generator will yield num_layers + 2 times. + + :param starts: The starting indices of the KV cache in the corresponding + token sequence. + + :param ends: The ending indices of the KV cache in the corresponding + token sequence. + + :raises ValueError: If 'slot_mapping' is not provided in kwargs. """ + self.initialize_kvcaches_ptr(**kwargs) - assert self.kvcaches is not None + assert self.kvcaches is not None, ( + "kvcaches should be provided in kwargs or initialized beforehand." + ) if "slot_mapping" not in kwargs: raise ValueError("'slot_mapping' should be provided in kwargs.") + if "sync" not in kwargs: raise ValueError("'sync' should be provided in kwargs.") @@ -436,235 +1019,129 @@ def _batched_to_gpu_gen(self, starts: List[int], ends: List[int], **kwargs): self._lazy_initialize_buffer(self.kvcaches) - def _ensure_xpu(t: torch.Tensor) -> torch.Tensor: - # Handle both torch.device('xpu:0') and string devices consistently. - if t is None: - return t - if t.device != self.device: - # non_blocking is fine; will be blocking - # if underlying memory isn't pinned - return t.to(self.device, non_blocking=True) - return t - - # Build a single contiguous mapping in the SAME order we will pack chunks. - slot_mapping_chunks = [ - slot_mapping[s:e] for s, e in zip(starts, ends, strict=False) - ] - slot_mapping_full = torch.cat(slot_mapping_chunks, dim=0) + slot_mapping_chunks = [] + for start, end in zip(starts, ends, strict=False): + slot_mapping_chunks.append(slot_mapping[start:end]) - # Move mapping ONCE to device (fixes multiple small H2D copies). - slot_mapping_full = _ensure_xpu(slot_mapping_full) + # TODO(Jiayi): Optimize away this `cat` + slot_mapping_full = torch.cat(slot_mapping_chunks, dim=0) - num_tokens = int(slot_mapping_full.numel()) - if num_tokens <= 0: - for _ in range(self.num_layers): - _ = yield - yield - if sync: - torch.xpu.current_stream().wait_stream(self.load_stream) - yield - return + num_tokens = len(slot_mapping_full) tmp_gpu_buffer_obj: Optional[MemoryObj] = None - if self.use_xpu: - # First Party - from lmcache.v1.memory_management import MemoryFormat - + if self.use_gpu: buffer_shape = self.get_shape(num_tokens) assert self.gpu_buffer_allocator is not None - requested_bytes = ( - int(buffer_shape.numel()) - * torch.empty((), dtype=self.dtype).element_size() - ) - allocator_tensor = getattr(self.gpu_buffer_allocator, "tensor", None) - capacity_bytes: Optional[int] = None - if isinstance(allocator_tensor, torch.Tensor): - capacity_bytes = int( - allocator_tensor.numel() * allocator_tensor.element_size() - ) - allocator_backend = getattr(self.gpu_buffer_allocator, "allocator", None) - allocated_bytes = getattr(allocator_backend, "total_allocated_size", None) tmp_gpu_buffer_obj = self.gpu_buffer_allocator.allocate( buffer_shape, self.dtype, MemoryFormat.KV_T2D ) - if tmp_gpu_buffer_obj is None or tmp_gpu_buffer_obj.tensor is None: - raise RuntimeError( - "Failed to allocate XPU staging buffer for batched_to_gpu: " - f"requested_bytes={requested_bytes}, " - f"capacity_bytes={capacity_bytes}, " - f"allocated_bytes={allocated_bytes}, " - f"allocator_type={type(self.gpu_buffer_allocator).__name__}, " - f"allocator_tensor_device=" - f"{getattr(allocator_tensor, 'device', None)}" - ) + assert tmp_gpu_buffer_obj is not None, ( + "Failed to allocate GPU buffer in GPUConnector" + ) + assert tmp_gpu_buffer_obj.tensor is not None + offset = starts[0] current_stream = torch.xpu.current_stream() - try: - for layer_id in range(self.num_layers): - memory_objs_layer = yield # List[MemoryObj] for this layer - - if sync: - current_stream.wait_stream(self.load_stream) + for layer_id in range(self.num_layers): + memory_objs_layer = yield + if sync: + current_stream.wait_stream(self.load_stream) + if layer_id > 0: + logger.debug(f"Finished loading layer {layer_id - 1}") - with torch.xpu.stream(self.load_stream): - dst_layer = self.kvcaches[layer_id] + # memobj -> gpu_buffer -> kvcaches + with torch.xpu.stream(self.load_stream): + for start, end, memory_obj in zip( + starts, ends, memory_objs_layer, strict=False + ): + # Validate memory format if self.use_mla: - dst_flat = cast( - torch.Tensor, - _get_head_size_view(dst_layer, use_mla=True), + assert memory_obj.metadata.fmt == MemoryFormat.KV_MLA_FMT, ( + f"Expected memory format {MemoryFormat.KV_MLA_FMT}, " + f"got {memory_obj.metadata.fmt}" ) else: - dst_k_flat, dst_v_flat = _get_head_size_view( # type: ignore[misc] - dst_layer, use_mla=False + assert memory_obj.metadata.fmt == MemoryFormat.KV_T2D, ( + f"Expected memory format {MemoryFormat.KV_T2D}, " + f"got {memory_obj.metadata.fmt}" + ) + if self.use_gpu: + tmp_gpu_buffer_obj.tensor[start - offset : end - offset].copy_( + memory_obj.tensor, non_blocking=True ) - - cursor = 0 - - if self.use_xpu: - assert tmp_gpu_buffer_obj is not None - staged = tmp_gpu_buffer_obj.tensor - assert staged is not None - - for s, e, mem in zip( - starts, ends, memory_objs_layer, strict=False - ): - assert mem.tensor is not None - n = int(e - s) - if n <= 0: - continue - - src = _ensure_xpu(mem.tensor) - - staged[cursor : cursor + n].copy_(src, non_blocking=True) - cursor += n - - sl = slot_mapping_full # already intended to be on device - sl = _ensure_xpu(sl) - - if self.use_mla: - staged_xpu = _ensure_xpu(staged) - if staged_xpu.dim() == 2: - dst_flat.index_copy_(0, sl, staged_xpu) - elif staged_xpu.dim() == 3 and staged_xpu.shape[0] == 1: - dst_flat.index_copy_(0, sl, staged_xpu[0]) - else: - raise ValueError( - f"Unexpected MLA staged tensor: {staged_xpu.shape}" - ) - else: - k_tok, v_tok = _split_token2d_kv(staged) - - # Make sure k_tok/v_tok are on XPU before index_copy_. - k_tok = _ensure_xpu(k_tok) - v_tok = _ensure_xpu(v_tok) - - # Keep your reshape logic as-is (only triggers when needed) - if ( - k_tok.dim() == 2 - and dst_k_flat.dim() == 3 - and k_tok.shape[1] - == dst_k_flat.shape[1] * dst_k_flat.shape[2] - ): - k_tok = k_tok.reshape( - k_tok.shape[0], - dst_k_flat.shape[1], - dst_k_flat.shape[2], - ) - if ( - v_tok.dim() == 2 - and dst_v_flat.dim() == 3 - and v_tok.shape[1] - == dst_v_flat.shape[1] * dst_v_flat.shape[2] - ): - v_tok = v_tok.reshape( - v_tok.shape[0], - dst_v_flat.shape[1], - dst_v_flat.shape[2], - ) - - dst_k_flat.index_copy_(0, sl, k_tok) - dst_v_flat.index_copy_(0, sl, v_tok) - else: - for s, e, mem in zip( - starts, ends, memory_objs_layer, strict=False - ): - assert mem.tensor is not None - n = int(e - s) - if n <= 0: - continue - - src = _ensure_xpu(mem.tensor) - sl = slot_mapping_full[cursor : cursor + n] - sl = _ensure_xpu(sl) - cursor += n - - if self.use_mla: - if src.dim() == 2: - dst_flat.index_copy_(0, sl, src) - elif src.dim() == 3 and src.shape[0] == 1: - dst_flat.index_copy_(0, sl, src[0]) - else: - raise ValueError( - f"Unexpected MLA token tensor: {src.shape}" - ) - else: - k_tok, v_tok = _split_token2d_kv(src) - k_tok = _ensure_xpu(k_tok) - v_tok = _ensure_xpu(v_tok) - - if ( - k_tok.dim() == 2 - and dst_k_flat.dim() == 3 - and k_tok.shape[1] - == dst_k_flat.shape[1] * dst_k_flat.shape[2] - ): - k_tok = k_tok.reshape( - k_tok.shape[0], - dst_k_flat.shape[1], - dst_k_flat.shape[2], - ) - if ( - v_tok.dim() == 2 - and dst_v_flat.dim() == 3 - and v_tok.shape[1] - == dst_v_flat.shape[1] * dst_v_flat.shape[2] - ): - v_tok = v_tok.reshape( - v_tok.shape[0], - dst_v_flat.shape[1], - dst_v_flat.shape[2], - ) - - dst_k_flat.index_copy_(0, sl, k_tok) - dst_v_flat.index_copy_(0, sl, v_tok) + lmc_ops.single_layer_kv_transfer( + memory_obj.tensor, + self.kvcaches[layer_id], + slot_mapping[start:end], + lmc_ops.TransferDirection.H2D, + self.gpu_kv_format, + token_major=True, + ) - yield + if self.use_gpu: + lmc_ops.single_layer_kv_transfer( + tmp_gpu_buffer_obj.tensor, + self.kvcaches[layer_id], + slot_mapping_full, + lmc_ops.TransferDirection.H2D, + self.gpu_kv_format, + token_major=True, + ) + yield - if sync: - current_stream.wait_stream(self.load_stream) - finally: - if tmp_gpu_buffer_obj is not None: - tmp_gpu_buffer_obj.ref_count_down() + # synchronize the last layer + if sync: + current_stream.wait_stream(self.load_stream) + + # free the buffer memory + if tmp_gpu_buffer_obj is not None: + tmp_gpu_buffer_obj.ref_count_down() + logger.debug(f"Finished loading all {self.num_layers} layers.") yield - def batched_from_gpu( # type: ignore[override] + @_lmcache_nvtx_annotate + def batched_from_gpu( self, - memory_objs: List[List[MemoryObj]], + memory_objs: Union[List[List[MemoryObj]]], starts: List[int], ends: List[int], **kwargs, ): """ - Generator: XPU paged KV -> (optional XPU staging) -> CPU token2d (per layer). + This function is a generator that moves the KV cache from the paged GPU + memory to the memory objects. The first iteration will prepare some + related metadata and initiate the transfer in the first layer. In each + of the following iterations, it will first wait until the storing of + previous layer finishes, and then initiate string the KV cache of the + current layer one. The storing process of the KV cache is paged GPU + memory -> GPU buffer -> memory objects. The last iteration simply waits + for the last layer to finish. + In total, this the generator will yield num_layers + 1 times. + + :param memory_objs: The memory objects to store the KV cache. The first + dimension is the number of layers, and the second dimension is the + number of memory objects (i.e., number of chunks) for each layer. + + :param starts: The starting indices of the KV cache in the corresponding + token sequence. + + :param ends: The ending indices of the KV cache in the corresponding + token sequence. + + :raises ValueError: If 'slot_mapping' is not provided in kwargs. """ + self.initialize_kvcaches_ptr(**kwargs) - assert self.kvcaches is not None + assert self.kvcaches is not None, ( + "kvcaches should be provided in kwargs or initialized beforehand." + ) if "slot_mapping" not in kwargs: raise ValueError("'slot_mapping' should be provided in kwargs.") + if "sync" not in kwargs: raise ValueError("'sync' should be provided in kwargs.") @@ -673,246 +1150,80 @@ def batched_from_gpu( # type: ignore[override] self._lazy_initialize_buffer(self.kvcaches) - current_stream = torch.xpu.current_stream() - - # ---- helpers (keep local to minimize file-wide changes) ---- - def _flatten_last2_if_needed( - src: torch.Tensor, dst: torch.Tensor - ) -> torch.Tensor: - """ - Make src match dst for the common KV layouts: - - src: [..., H, HS] -> dst: [..., H*HS] - - or already matches - """ - if src.shape == dst.shape: - return src - - # dst has one less trailing dim: [..., D] where D=H*HS - if src.dim() == dst.dim() + 1: - # e.g., src [..., 8, 128] -> dst [..., 1024] - if dst.shape == (*src.shape[:-2], src.shape[-2] * src.shape[-1]): - return src.reshape(*src.shape[:-2], -1) - - # same ndim but dst last dim is flattened (dst ends with D) - if src.dim() == dst.dim(): - if ( - dst.shape[:-1] == src.shape[:-2] - and dst.shape[-1] == src.shape[-2] * src.shape[-1] - ): - return src.reshape(*src.shape[:-2], -1) - - return src # caller will error if still incompatible - - def _copy_kv_into_mem( - mem_tensor: torch.Tensor, k_src: torch.Tensor, v_src: torch.Tensor - ) -> None: - """ - Copy K/V into mem.tensor supporting: - - [2, ..., D] (K in 0, V in 1) - - [..., 2, D] (K in [:,0,:], V in [:,1,:]) - - [2, ..., H, HS] or [..., 2, H, HS] similarly - """ - if mem_tensor.dim() < 3: - raise ValueError( - f"Unexpected output token2d layout: {mem_tensor.shape}" - ) - - # Case A: mem is [2, ...] - if mem_tensor.shape[0] == 2: - k_dst = mem_tensor[0] - v_dst = mem_tensor[1] - k_src2 = _flatten_last2_if_needed(k_src, k_dst) - v_src2 = _flatten_last2_if_needed(v_src, v_dst) - if k_src2.shape != k_dst.shape or v_src2.shape != v_dst.shape: - raise ValueError( - f"KV shape mismatch after reshape: " - f"k src {k_src.shape}->{k_src2.shape} vs dst {k_dst.shape}; " - f"v src {v_src.shape}->{v_src2.shape} vs dst {v_dst.shape}" - ) - k_dst.copy_(k_src2.to(k_dst.device), non_blocking=True) - v_dst.copy_(v_src2.to(v_dst.device), non_blocking=True) - return - - # Case B: mem is [..., 2, ...] - if mem_tensor.shape[1] == 2: - k_dst = mem_tensor[:, 0, ...] - v_dst = mem_tensor[:, 1, ...] - k_src2 = _flatten_last2_if_needed(k_src, k_dst) - v_src2 = _flatten_last2_if_needed(v_src, v_dst) - if k_src2.shape != k_dst.shape or v_src2.shape != v_dst.shape: - raise ValueError( - f"KV shape mismatch after reshape: " - f"k src {k_src.shape}->{k_src2.shape} vs dst {k_dst.shape}; " - f"v src {v_src.shape}->{v_src2.shape} vs dst {v_dst.shape}" - ) - k_dst.copy_(k_src2.to(k_dst.device), non_blocking=True) - v_dst.copy_(v_src2.to(v_dst.device), non_blocking=True) - return - - raise ValueError(f"Unexpected output token2d layout: {mem_tensor.shape}") + slot_mapping_chunks = [] + for start, end in zip(starts, ends, strict=False): + slot_mapping_chunks.append(slot_mapping[start:end]) - slot_mapping_on_device = slot_mapping.to(self.device) + slot_mapping_full = torch.cat(slot_mapping_chunks, dim=0) - # Precompute “full” mapping for batched gather - # NOTE: this assumes starts/ends partition slot_mapping contiguously. - # If not contiguous, concatenation is still correct. - slot_mapping_full = torch.cat( - [slot_mapping_on_device[s:e] for s, e in zip(starts, ends, strict=False)], - dim=0, - ) - total_tokens = int(slot_mapping_full.numel()) + num_tokens = len(slot_mapping_full) - # Optional staging buffer (will be USED when self.use_xpu=True) tmp_gpu_buffer_obj: Optional[MemoryObj] = None - if self.use_xpu: - # First Party - from lmcache.v1.memory_management import MemoryFormat - - # buffer shape uses existing helper; must match how allocator expects KV_T2D - buffer_shape = self.get_shape(total_tokens) + if self.use_gpu: + buffer_shape = self.get_shape(num_tokens) assert self.gpu_buffer_allocator is not None - requested_bytes = ( - int(buffer_shape.numel()) - * torch.empty((), dtype=self.dtype).element_size() - ) - allocator_tensor = getattr(self.gpu_buffer_allocator, "tensor", None) - capacity_bytes: Optional[int] = None - if isinstance(allocator_tensor, torch.Tensor): - capacity_bytes = int( - allocator_tensor.numel() * allocator_tensor.element_size() - ) - allocator_backend = getattr(self.gpu_buffer_allocator, "allocator", None) - allocated_bytes = getattr(allocator_backend, "total_allocated_size", None) tmp_gpu_buffer_obj = self.gpu_buffer_allocator.allocate( buffer_shape, self.dtype, MemoryFormat.KV_T2D ) - if tmp_gpu_buffer_obj is None or tmp_gpu_buffer_obj.tensor is None: - raise RuntimeError( - "Failed to allocate XPU staging buffer for batched_from_gpu: " - f"requested_bytes={requested_bytes}, " - f"capacity_bytes={capacity_bytes}, " - f"allocated_bytes={allocated_bytes}, " - f"allocator_type={type(self.gpu_buffer_allocator).__name__}, " - f"allocator_tensor_device=" - f"{getattr(allocator_tensor, 'device', None)}" - ) - tmp = tmp_gpu_buffer_obj.tensor # staging tensor on device - - try: - for layer_id in range(self.num_layers): - mem_layer = memory_objs[layer_id] - - with torch.xpu.stream(self.store_stream): - self.store_stream.wait_stream(current_stream) + assert tmp_gpu_buffer_obj is not None, ( + "Failed to allocate GPU buffer in GPUConnector" + ) + assert tmp_gpu_buffer_obj.tensor is not None - src_layer = self.kvcaches[layer_id] + offset = starts[0] + current_stream = torch.xpu.current_stream() - if self.use_mla: - src_flat = cast( - torch.Tensor, - _get_head_size_view(src_layer, use_mla=True), + for layer_id in range(self.num_layers): + memory_objs_layer = memory_objs[layer_id] + # kvcaches -> gpu_buffer -> memobj + with torch.xpu.stream(self.store_stream): + self.store_stream.wait_stream(current_stream) + if self.use_gpu: + lmc_ops.single_layer_kv_transfer( + tmp_gpu_buffer_obj.tensor, + self.kvcaches[layer_id], + slot_mapping_full, + lmc_ops.TransferDirection.D2H, + self.gpu_kv_format, + token_major=True, + ) + for start, end, memory_obj in zip( + starts, ends, memory_objs_layer, strict=False + ): + assert memory_obj.tensor is not None + if self.use_gpu: + memory_obj.tensor.copy_( + tmp_gpu_buffer_obj.tensor[start - offset : end - offset], + non_blocking=True, ) - - if self.use_xpu: - gathered_full = src_flat.index_select(0, slot_mapping_full) - # Write into tmp if possible, else fallback to per-chunk - tmp_src = ( - _flatten_last2_if_needed(gathered_full, tmp) - if "tmp" in locals() - else gathered_full - ) - if "tmp" in locals() and tmp_src.shape == tmp.shape: - tmp.copy_(tmp_src, non_blocking=True) - off = 0 - for s, e, mem in zip( - starts, ends, mem_layer, strict=False - ): - assert mem.tensor is not None - n = e - s - chunk = tmp[off : off + n] - off += n - mem.tensor.copy_( - chunk.to(mem.tensor.device), non_blocking=True - ) - else: - for s, e, mem in zip( - starts, ends, mem_layer, strict=False - ): - assert mem.tensor is not None - sl = slot_mapping_on_device[s:e] - gathered = src_flat.index_select(0, sl) - mem.tensor.copy_( - gathered.to(mem.tensor.device), - non_blocking=True, - ) - else: - for s, e, mem in zip(starts, ends, mem_layer, strict=False): - assert mem.tensor is not None - sl = slot_mapping_on_device[s:e] - gathered = src_flat.index_select(0, sl) - mem.tensor.copy_( - gathered.to(mem.tensor.device), non_blocking=True - ) - - # Keep memory format metadata consistent for downstream checks. - target_fmt = MemoryFormat.KV_MLA_FMT - for mem in mem_layer: - self._validate_format_transition(mem, target_fmt) - mem.metadata.fmt = target_fmt - else: - src_k_flat, src_v_flat = _get_head_size_view( - src_layer, use_mla=False + lmc_ops.single_layer_kv_transfer( + memory_obj.tensor, + self.kvcaches[layer_id], + slot_mapping[start:end], + lmc_ops.TransferDirection.D2H, + self.gpu_kv_format, + token_major=True, ) + # Set metadata format + if self.use_mla: + memory_obj.metadata.fmt = MemoryFormat.KV_MLA_FMT - if self.use_xpu: - k_full = src_k_flat.index_select(0, slot_mapping_full) - v_full = src_v_flat.index_select(0, slot_mapping_full) - - # Slice from staging. If tmp exists and can hold - # the layout, use it; otherwise slice k/v directly. - off = 0 - for s, e, mem in zip(starts, ends, mem_layer, strict=False): - assert mem.tensor is not None - n = e - s - - k_chunk = k_full[off : off + n] - v_chunk = v_full[off : off + n] - off += n - - _copy_kv_into_mem(mem.tensor, k_chunk, v_chunk) - - else: - # per-chunk gather (original behavior); - # avoids per-iteration H2D slot_mapping transfers - for s, e, mem in zip(starts, ends, mem_layer, strict=False): - assert mem.tensor is not None - sl = slot_mapping_on_device[s:e] - k = src_k_flat.index_select(0, sl) - v = src_v_flat.index_select(0, sl) - _copy_kv_into_mem(mem.tensor, k, v) - - if sync: - self.store_stream.synchronize() - yield - finally: - if tmp_gpu_buffer_obj is not None: - tmp_gpu_buffer_obj.ref_count_down() + yield + if sync: + self.store_stream.synchronize() + logger.debug(f"Finished offloading layer {layer_id}") - yield + # free the buffer memory + if tmp_gpu_buffer_obj is not None: + tmp_gpu_buffer_obj.ref_count_down() - def batched_to_gpu( - self, - memory_objs: Union[ - List[List[MemoryObj]], List[MemoryObj], List[int], None - ] = None, - starts: Optional[List[int]] = None, - ends: Optional[List[int]] = None, - **kwargs, - ): - return self._batched_to_gpu_gen(starts=starts or [], ends=ends or [], **kwargs) + yield def get_shape(self, num_tokens: int) -> torch.Size: if self.use_mla: + # MLA format: [num_tokens, hidden_dim_size] return torch.Size([num_tokens, self.hidden_dim_size]) - return torch.Size([num_tokens, 2, self.hidden_dim_size]) + else: + # Standard format: [num_tokens, 2, hidden_dim_size] + return torch.Size([num_tokens, 2, self.hidden_dim_size]) diff --git a/lmcache/v1/memory_management.py b/lmcache/v1/memory_management.py index 91f8b8b4204..164cb058f6f 100644 --- a/lmcache/v1/memory_management.py +++ b/lmcache/v1/memory_management.py @@ -2565,89 +2565,3 @@ def batched_free( def __str__(self): return "PDMemoryAllocator" - - -class XPUMemoryAllocator(MemoryAllocatorInterface): - """Allocates memory in the pre-allocated XPU memory.""" - - def __init__( - self, - size: int, - device="xpu", - align_bytes: Optional[int] = None, - use_paging: bool = False, - **kwargs, - ): - self.tensor = torch.empty((size,), dtype=torch.uint8, device=device) - - self.allocator: MemoryAllocatorInterface - if use_paging: - assert "shapes" in kwargs, ( - "shapes must be specified for paged memory allocator" - ) - assert "dtypes" in kwargs, ( - "dtypes must be specified for paged memory allocator" - ) - assert "fmt" in kwargs, "fmt must be specified for paged memory allocator" - self.allocator = PagedTensorMemoryAllocator( - tensor=self.tensor, - shapes=kwargs["shapes"], - dtypes=kwargs["dtypes"], - fmt=kwargs["fmt"], - ) - else: - alloc_kwargs = {} - if align_bytes is not None: - alloc_kwargs["align_bytes"] = align_bytes - self.allocator = TensorMemoryAllocator(self.tensor, **alloc_kwargs) - - self.device_mem_lock = threading.Lock() if not use_paging else nullcontext() - - @_lmcache_nvtx_annotate - def allocate( - self, - shapes: Union[torch.Size, list[torch.Size]], - dtypes: Union[torch.dtype, list[torch.dtype]], - fmt: MemoryFormat = MemoryFormat.KV_2LTD, - allocator_type: Optional[str] = None, - ) -> Optional[MemoryObj]: - with self.device_mem_lock: - return self.allocator.allocate(shapes, dtypes, fmt, str(self)) - - @_lmcache_nvtx_annotate - def batched_allocate( - self, - shapes: Union[torch.Size, list[torch.Size]], - dtypes: Union[torch.dtype, list[torch.dtype]], - batch_size: int, - fmt: MemoryFormat = MemoryFormat.KV_2LTD, - allocator_type: Optional[str] = None, - ) -> Optional[List[MemoryObj]]: - with self.device_mem_lock: - return self.allocator.batched_allocate( - shapes, dtypes, batch_size, fmt, str(self) - ) - - def free(self, memory_obj: MemoryObj, allocator_type: Optional[str] = None): - with self.device_mem_lock: - self.allocator.free(memory_obj) - - def batched_free( - self, - memory_objs: List[MemoryObj], - allocator_type: Optional[str] = None, - update_stats: bool = True, - ): - with self.device_mem_lock: - self.allocator.batched_free(memory_objs) - - def memcheck(self): - with self.device_mem_lock: - return self.allocator.memcheck() - - def close(self): - if torch_dev.is_available(): - torch_dev.synchronize() - - def __str__(self): - return "XPUMemoryAllocator" diff --git a/setup.py b/setup.py index 6ad52dafb21..2124e21e96d 100644 --- a/setup.py +++ b/setup.py @@ -15,13 +15,17 @@ # will run python setup.py sdist --dist-dir dist BUILDING_SDIST = "sdist" in sys.argv or os.environ.get("NO_CUDA_EXT", "0") == "1" -# New environment variable to choose between CUDA and HIP +# New environment variable to choose between CUDA, HIP, and SYCL BUILD_WITH_HIP = os.environ.get("BUILD_WITH_HIP", "0") == "1" +BUILD_WITH_SYCL = os.environ.get("BUILD_WITH_SYCL", "0") == "1" ENABLE_CXX11_ABI = os.environ.get("ENABLE_CXX11_ABI", "1") == "1" def _read_requirements(path: Path) -> list[str]: + if not path.exists(): + return [] + reqs: list[str] = [] for raw in path.read_text().splitlines(): line = raw.strip() @@ -293,14 +297,115 @@ def rocm_extension() -> tuple[list, dict]: return ext_modules, cmdclass +def sycl_extension() -> tuple[list, dict]: + # Third Party + from torch.utils import cpp_extension # Import here + + print("Building SYCL/XPU extensions") + + # Standard + import shutil + + if shutil.which("icpx") is None: + sys.exit("icpx not found. Please source oneAPI setvars.sh at first") + os.environ["CXX"] = "icpx" + oneapi_root = os.environ.get("ONEAPI_ROOT", "/opt/intel/oneapi") + include_dirs = [f"{oneapi_root}/include"] + library_dirs = [f"{oneapi_root}/lib"] + + sycl_sources = [ + "csrc/sycl/pybind_sycl.cpp", + "csrc/sycl/mem_kernels_sycl.cpp", + ] + storage_manager_sources = [ + "csrc/storage_manager/bitmap.cpp", + "csrc/storage_manager/pybind.cpp", + "csrc/storage_manager/ttl_lock.cpp", + "csrc/storage_manager/utils.cpp", + ] + redis_sources = [ + "csrc/storage_backends/redis/pybind.cpp", + "csrc/storage_backends/redis/connector.cpp", + ] + fs_sources = [ + "csrc/storage_backends/fs/pybind.cpp", + "csrc/storage_backends/fs/connector.cpp", + ] + # Use CppExtension with DPC++ compiler (set CXX=icpx before invoking). + # The -fsycl flag enables SYCL compilation and linking. + # Intel XPU optimizations: + # -ftarget-register-alloc-mode=pvc:auto + # Register allocation tuned for Ponte Vecchio / Data Center GPU Max. + # -fno-sycl-id-queries-fit-in-int + # Allow 64-bit index arithmetic in SYCL kernels. + # -ffast-math + # Aggressive FP opts (safe for current implementation: kernels + # only copy data, no FP arithmetic. Review if FP math is added). + # -funroll-loops + # Unroll inner copy loops for better instruction packing. + ext_modules = [ + cpp_extension.SyclExtension( + "lmcache.xpu_ops", + sources=sycl_sources, + include_dirs=include_dirs, + library_dirs=library_dirs, + extra_compile_args={ + "cxx": [ + "-std=c++17", + "-D_GLIBCXX_USE_CXX11_ABI=1", + "-O3", + "-fsycl", + "-fno-sycl-id-queries-fit-in-int", + "-ffast-math", + "-funroll-loops", + # Suppress deprecation warnings from SYCL standard + # headers (sycl/accessor.hpp references the deprecated + # 'host_buffer' internally; our code uses USM only). + "-Wno-deprecated-declarations", + "-Wno-nan-infinity-disabled", + ], + }, + extra_link_args=["-fsycl"], + ), + cpp_extension.CppExtension( + "lmcache.native_storage_ops", + sources=storage_manager_sources, + include_dirs=["csrc/storage_manager"], + extra_compile_args={ + "cxx": ["-D_GLIBCXX_USE_CXX11_ABI=1", "-O3", "-std=c++17"], + }, + ), + cpp_extension.CppExtension( + "lmcache.lmcache_redis", + sources=redis_sources, + include_dirs=["csrc/storage_backends", "csrc/storage_backends/redis"], + extra_compile_args={ + "cxx": ["-D_GLIBCXX_USE_CXX11_ABI=1", "-O3", "-std=c++17"], + }, + ), + cpp_extension.CppExtension( + "lmcache.lmcache_fs", + sources=fs_sources, + include_dirs=["csrc/storage_backends", "csrc/storage_backends/fs"], + extra_compile_args={ + "cxx": ["-O3", "-std=c++17"], + }, + ), + ] + cmdclass = {"build_ext": cpp_extension.BuildExtension} + return ext_modules, cmdclass + + def source_dist_extension() -> tuple[list, dict]: - print("Not building CUDA/HIP extensions for sdist") + print("Not building CUDA/HIP/SYCL extensions for sdist") return [], {} if __name__ == "__main__": if BUILDING_SDIST: get_extension = source_dist_extension + elif BUILD_WITH_SYCL: + get_extension = sycl_extension elif BUILD_WITH_HIP: get_extension = rocm_extension else: @@ -311,6 +416,8 @@ def source_dist_extension() -> tuple[list, dict]: install_requires = _read_requirements(ROOT_DIR / "requirements" / "common.txt") if BUILD_WITH_HIP: core_file = "rocm_core.txt" + elif BUILD_WITH_SYCL: + core_file = "xpu_core.txt" else: # CUDA major selects between cu12 and cu13 vendor pins (cupy, nixl). # Defaults to cu13 (the PyPI build); cu12.9 wheel builds set diff --git a/tests/benchmarks/test_xpu_connector_benchmark.py b/tests/benchmarks/test_xpu_connector_benchmark.py index b171816135c..2f3b29b2c8a 100644 --- a/tests/benchmarks/test_xpu_connector_benchmark.py +++ b/tests/benchmarks/test_xpu_connector_benchmark.py @@ -81,7 +81,13 @@ def _create_connector( use_mla: bool = False, ): return VLLMPagedMemXPUConnectorV2( + hidden_dim_size=hidden_dim, + num_layers=num_layers, use_gpu=use_gpu, + chunk_size=chunk_size, + dtype=dtype, + use_mla=use_mla, + device=device, ) diff --git a/tests/benchmarks/test_xpu_layerwise_connector_benchmark.py b/tests/benchmarks/test_xpu_layerwise_connector_benchmark.py index 5c461ba9b63..e2adc3bf3e7 100644 --- a/tests/benchmarks/test_xpu_layerwise_connector_benchmark.py +++ b/tests/benchmarks/test_xpu_layerwise_connector_benchmark.py @@ -72,7 +72,7 @@ def _create_connector( hidden_dim: int, num_layers: int, *, - use_xpu: bool, + use_gpu: bool, chunk_size: int, dtype: torch.dtype, use_mla: bool = False, @@ -80,7 +80,7 @@ def _create_connector( return VLLMPagedMemLayerwiseXPUConnector( hidden_dim, num_layers, - use_xpu=use_xpu, + use_gpu=use_gpu, chunk_size=chunk_size, dtype=dtype, device=device, @@ -296,13 +296,13 @@ def _build_engine( @pytest.mark.benchmark(group="store") @pytest.mark.parametrize("device_type", DEVICE_PARAMS) @pytest.mark.parametrize("backend", ["cpu", "disk"]) -@pytest.mark.parametrize("use_xpu", [False, True]) +@pytest.mark.parametrize("use_gpu", [True]) @pytest.mark.parametrize("save_unfull_chunk", [False, True]) def test_store_1GB( benchmark, device_type, backend, - use_xpu, + use_gpu, save_unfull_chunk, create_config, autorelease_v1, @@ -339,7 +339,7 @@ def test_store_1GB( device, hidden_dim=num_heads * head_dim, num_layers=num_layers, - use_xpu=use_xpu, + use_gpu=use_gpu, chunk_size=chunk_size, dtype=dtype, use_mla=False, @@ -411,13 +411,13 @@ def setup(): @pytest.mark.benchmark(group="retrieve") @pytest.mark.parametrize("device_type", DEVICE_PARAMS) @pytest.mark.parametrize("backend", BACKENDS) -@pytest.mark.parametrize("use_xpu", [False, True]) +@pytest.mark.parametrize("use_gpu", [True]) @pytest.mark.parametrize("save_unfull_chunk", [False, True]) def test_retrieve_1GB_allhit( benchmark, device_type, backend, - use_xpu, + use_gpu, save_unfull_chunk, create_config, autorelease_v1, @@ -442,7 +442,7 @@ def test_retrieve_1GB_allhit( device, hidden_dim=num_heads * head_dim, num_layers=num_layers, - use_xpu=use_xpu, + use_gpu=use_gpu, chunk_size=chunk_size, dtype=dtype, use_mla=False, @@ -514,13 +514,13 @@ def run_func(): @pytest.mark.benchmark(group="lookup") @pytest.mark.parametrize("device_type", DEVICE_PARAMS) @pytest.mark.parametrize("backend", BACKENDS) -@pytest.mark.parametrize("use_xpu", [False, True]) +@pytest.mark.parametrize("use_gpu", [True]) @pytest.mark.parametrize("save_unfull_chunk", [False, True]) def test_lookup_20K_tokens( benchmark, device_type, backend, - use_xpu, + use_gpu, save_unfull_chunk, create_config, autorelease_v1, @@ -538,16 +538,16 @@ def test_lookup_20K_tokens( block_size = 16 chunk_size = 256 - # use_xpu=True stages per-request chunks on XPU during layerwise store. + # use_gpu=True stages per-request chunks on XPU during layerwise store. # Keeping 10 requests can exceed the default staging pool in pre-population. - num_requests = 8 if use_xpu else 10 + num_requests = 8 if use_gpu else 10 num_repeats = 10 connector = _create_connector( device, hidden_dim=num_heads * head_dim, num_layers=num_layers, - use_xpu=use_xpu, + use_gpu=use_gpu, chunk_size=chunk_size, dtype=dtype, use_mla=False, diff --git a/tests/v1/test_c_ops_fallback_parity.py b/tests/v1/test_c_ops_fallback_parity.py index f1b037fea59..84d34333d15 100644 --- a/tests/v1/test_c_ops_fallback_parity.py +++ b/tests/v1/test_c_ops_fallback_parity.py @@ -173,7 +173,10 @@ def _parse_docstring_params(func): def _get_python_params(func): """Extract [(name, has_default, default_value)] via inspect.signature.""" - sig = inspect.signature(func) + try: + sig = inspect.signature(func) + except ValueError as err: + raise ValueError("no signature found for {!r}".format(func)) from err result = [] for p in sig.parameters.values(): has_default = p.default is not inspect.Parameter.empty @@ -242,7 +245,7 @@ def test_function_signature_parity(func_name): try: py_params = _get_python_params(py_func) except (ValueError, TypeError): - pytest.fail(f"Cannot inspect fallback.{func_name} signature") + pytest.skip(f"Cannot inspect fallback.{func_name} signature") c_has_names = _has_real_names(c_params) diff --git a/tests/v1/test_gpu_connector.py b/tests/v1/test_gpu_connector.py index d3ef933e06d..824cdf4b01a 100644 --- a/tests/v1/test_gpu_connector.py +++ b/tests/v1/test_gpu_connector.py @@ -779,7 +779,7 @@ def test_vllm_paged_connector_v2_to_gpu_bench(benchmark): chunk_size = 256 - allocator = GPUMemoryAllocator(1024 * 1024 * 1024) + allocator = GPUMemoryAllocator(1024 * 1024 * 1024, device) gpu_kv_src = generate_kv_cache_paged_list_tensors(num_blocks, device, block_size) gpu_kv_dst = generate_kv_cache_paged_list_tensors(num_blocks, device, block_size) diff --git a/tests/v1/test_xpu_connector.py b/tests/v1/test_xpu_connector.py index 0cf04f0def7..7bbd955e681 100644 --- a/tests/v1/test_xpu_connector.py +++ b/tests/v1/test_xpu_connector.py @@ -1,457 +1,834 @@ # SPDX-License-Identifier: Apache-2.0 -# tests/v1/test_xpu_connector.py +# Standard +from contextlib import nullcontext +from unittest.mock import patch +import random +import threading # Third Party import pytest import torch # First Party +from lmcache.v1.gpu_connector.utils import get_dtype from lmcache.v1.gpu_connector.xpu_connectors import ( + VLLMBufferLayerwiseXPUConnector, VLLMPagedMemLayerwiseXPUConnector, VLLMPagedMemXPUConnectorV2, + VLLMPagedMemXPUConnectorV3, ) -from lmcache.v1.memory_management import MemoryFormat, PinMemoryAllocator -from lmcache.v1.metadata import LMCacheMetadata -from tests.v1.utils import ( - check_paged_kv_cache_equal, - generate_kv_cache_paged_list_tensors, +from lmcache.v1.memory_management import ( + GPUMemoryAllocator, + MemoryFormat, + PagedTensorMemoryAllocator, + PinMemoryAllocator, + TensorMemoryAllocator, ) +from lmcache.v1.metadata import LMCacheMetadata +if torch.xpu.is_available(): + try: + # First Party + import lmcache.c_ops as lmc_ops + except ImportError: + lmc_ops = None +else: + lmc_ops = None -def _skip_if_no_xpu(): - if not hasattr(torch, "xpu") or not torch.xpu.is_available(): - pytest.skip("torch.xpu is not available") +# Mock c_ops when not available +if lmc_ops is None: + class MockGPUKVFormat: + NL_X_TWO_NB_BS_NH_HS = 0 + NL_X_NB_TWO_BS_NH_HS = 1 + NL_X_NB_BS_HS = 2 -def _make_unique_slot_mapping( - *, total_slots: int, num_tokens: int, device: torch.device -) -> torch.Tensor: - # Unique indices avoids overwriting the same slot multiple times. - return torch.randperm(total_slots, device=device, dtype=torch.int64)[:num_tokens] + class MockCOps: + GPUKVFormat = MockGPUKVFormat + lmc_ops = MockCOps() -def _pack_slot_mapping( - slot_mapping: torch.Tensor, starts: list[int], ends: list[int] -) -> torch.Tensor: - return torch.cat( - [slot_mapping[s:e] for s, e in zip(starts, ends, strict=False)], - dim=0, - ) +# Local +from .utils import ( + check_paged_kv_cache_equal, + check_paged_kv_cache_equal_with_mla, + generate_kv_cache_paged_list_tensors, + recover_gpu_connector_states, +) -@pytest.mark.parametrize("use_gpu", [False, True]) -def test_xpu_connector_roundtrip_non_layerwise(use_gpu: bool): - _skip_if_no_xpu() - device = torch.device("xpu:0") +@pytest.fixture(autouse=True, scope="module") +def patch_pin_allocator(): + def fake_pin_init(self, size: int, use_paging: bool = False, **kwargs): + """ + :param int size: The size of the pinned memory in bytes. + """ - num_layers = 2 - num_blocks = 4 + self._unregistered = False + self.buffer = torch.empty(size, dtype=torch.uint8, pin_memory=True) + + if use_paging: + assert "shapes" in kwargs, ( + "shapes must be specified for paged memory allocator" + ) + assert "dtypes" in kwargs, ( + "dtypes must be specified for paged memory allocator" + ) + assert "fmt" in kwargs, "fmt must be specified for paged memory allocator" + self.allocator = PagedTensorMemoryAllocator( + tensor=self.buffer, + shapes=kwargs["shapes"], + dtypes=kwargs["dtypes"], + fmt=kwargs["fmt"], + ) + else: + self.allocator = TensorMemoryAllocator(self.buffer) + + self.host_mem_lock = threading.Lock() if not use_paging else nullcontext() + + def fake_pin_close(self): + if not self._unregistered: + torch.xpu.synchronize() + self._unregistered = True + + with ( + patch( + "lmcache.v1.memory_management.PinMemoryAllocator.__init__", fake_pin_init + ), + patch("lmcache.v1.memory_management.PinMemoryAllocator.close", fake_pin_close), + ): + yield + + +@pytest.mark.parametrize("use_gpu", [True, False]) +@pytest.mark.parametrize( + "gpu_kv_format", + [ + lmc_ops.GPUKVFormat.NL_X_TWO_NB_BS_NH_HS, # vllm non-MLA flash attention + lmc_ops.GPUKVFormat.NL_X_NB_TWO_BS_NH_HS, # vllm non-MLA flash infer + lmc_ops.GPUKVFormat.NL_X_NB_BS_HS, + ], # vllm MLA +) +@pytest.mark.skipif( + not torch.xpu.is_available(), + reason="TODO: Add non-XPU implementation to VLLMPagedMemXPUConnectorV2", +) +def test_vllm_paged_connector_v2_with_gpu_and_mla(use_gpu, gpu_kv_format): + use_mla = gpu_kv_format == lmc_ops.GPUKVFormat.NL_X_NB_BS_HS + num_blocks = 100 block_size = 16 - head_size = 64 - num_tokens = 32 + num_layers = 32 + num_heads = 1 if use_mla else 8 + head_size = 128 + device = "xpu" + hidden_dim = num_heads * head_size + + num_tokens = 800 + chunk_size = 256 + + allocator = PinMemoryAllocator(1024 * 1024 * 1024) - kvcaches = generate_kv_cache_paged_list_tensors( + gpu_kv_src = generate_kv_cache_paged_list_tensors( num_blocks=num_blocks, + device=device, block_size=block_size, - num_layers=num_layers, - head_size=head_size, + gpu_kv_format=gpu_kv_format, + ) + gpu_kv_dst = generate_kv_cache_paged_list_tensors( + num_blocks=num_blocks, device=device, + block_size=block_size, + gpu_kv_format=gpu_kv_format, ) + dtype = get_dtype(gpu_kv_src, gpu_kv_format) - # Derive actual dims from generated KV (avoid helper defaults mismatch) - _, _, num_heads_actual, head_size_actual = kvcaches[0][0].shape - hidden_dim_actual = num_heads_actual * head_size_actual - - total_slots = num_blocks * block_size - slot_mapping = _make_unique_slot_mapping( - total_slots=total_slots, num_tokens=num_tokens, device=device - ) + slot_mapping = random.sample(range(0, num_blocks * block_size), num_tokens) + slot_mapping = torch.tensor(slot_mapping, device=device, dtype=torch.int64) - pin_alloc = PinMemoryAllocator(size=1024 * 1024 * 64) - memobj = pin_alloc.allocate( - torch.Size([2, num_layers, num_tokens, hidden_dim_actual]), - torch.bfloat16, - MemoryFormat.KV_2LTD, - ) + # Check the gpu_kv is not the same before copying + with pytest.raises(AssertionError): + if use_mla: + check_paged_kv_cache_equal_with_mla( + gpu_kv_src, gpu_kv_dst, slot_mapping, head_size + ) + else: + check_paged_kv_cache_equal( + gpu_kv_src, + gpu_kv_dst, + slot_mapping, + num_heads, + head_size, + gpu_kv_format, + ) - meta = LMCacheMetadata( - model_name="xpu_test", - world_size=1, - local_world_size=1, - worker_id=0, - local_worker_id=0, - kv_dtype=torch.bfloat16, - kv_shape=(num_layers, 2, num_tokens, num_heads_actual, head_size_actual), + connector = VLLMPagedMemXPUConnectorV2( + hidden_dim, + num_layers, + use_gpu=use_gpu, + chunk_size=chunk_size, + dtype=dtype, + device=device, + use_mla=use_mla, ) - conn = VLLMPagedMemXPUConnectorV2.from_metadata( - meta, + connector2 = VLLMPagedMemXPUConnectorV2( + hidden_dim, + num_layers, use_gpu=use_gpu, + chunk_size=chunk_size, + dtype=dtype, device=device, + use_mla=use_mla, ) - - try: - # XPU -> CPU (KV_2LTD in memobj) - conn.from_gpu( - memobj, - start=0, - end=num_tokens, + assert connector.use_mla == use_mla + assert connector2.use_mla == use_mla + for start in range(0, num_tokens, chunk_size): + end = min(start + chunk_size, num_tokens) + shape = connector.get_shape(end - start) + memory_obj = allocator.allocate(shape, dtype) + connector.from_gpu( + memory_obj, + start, + end, + kvcaches=gpu_kv_src, slot_mapping=slot_mapping, - kvcaches=kvcaches, + offset=0, ) - - # CPU -> XPU into fresh caches - kvcaches_dst = generate_kv_cache_paged_list_tensors( - num_blocks=num_blocks, - block_size=block_size, - num_layers=num_layers, - head_size=head_size_actual, - device=device, - ) - for t in kvcaches_dst: - t.zero_() - - conn.to_gpu( - memobj, - start=0, - end=num_tokens, + recover_gpu_connector_states(connector) + if use_mla: + assert memory_obj.metadata.fmt == MemoryFormat.KV_MLA_FMT + else: + assert memory_obj.metadata.fmt == MemoryFormat.KV_2LTD + connector2.to_gpu( + memory_obj, + start, + end, + kvcaches=gpu_kv_dst, slot_mapping=slot_mapping, - kvcaches=kvcaches_dst, + offset=0, ) + allocator.free(memory_obj) + assert allocator.memcheck() + if use_mla: + check_paged_kv_cache_equal_with_mla( + gpu_kv_src, gpu_kv_dst, slot_mapping, head_size + ) + else: check_paged_kv_cache_equal( - kvcaches, - kvcaches_dst, - slot_mapping, - num_heads=num_heads_actual, - head_size=head_size_actual, + gpu_kv_src, gpu_kv_dst, slot_mapping, num_heads, head_size, gpu_kv_format ) - finally: - memobj.ref_count_down() - pin_alloc.close() + allocator.close() + + +@pytest.mark.parametrize("use_gpu", [True, False]) +@pytest.mark.parametrize("num_groups", [1, 2, 3]) +@pytest.mark.parametrize( + "gpu_kv_format", + [ + lmc_ops.GPUKVFormat.NL_X_TWO_NB_BS_NH_HS, # vllm non-MLA flash attention + lmc_ops.GPUKVFormat.NL_X_NB_TWO_BS_NH_HS, # vllm non-MLA flash infer + lmc_ops.GPUKVFormat.NL_X_NB_BS_HS, + ], # vllm MLA +) +@pytest.mark.skipif( + not torch.xpu.is_available(), + reason="TODO: Add non-XPU implementation to VLLMPagedMemXPUConnectorV3", +) +def test_vllm_paged_connector_v3_with_gpu_and_mla(use_gpu, num_groups, gpu_kv_format): + use_mla = gpu_kv_format == lmc_ops.GPUKVFormat.NL_X_NB_BS_HS + head_sizes = [64, 66, 66] + dtypes = [torch.uint8, torch.bfloat16, torch.uint8] + num_blocks = 100 + block_size = 16 + num_heads = 1 if use_mla else 8 + device = "xpu" + num_tokens = 800 + chunk_size = 256 + + allocator = PinMemoryAllocator(1024 * 1024 * 1024) + + # generate kv cache tensors + src_kv_groups: list[list] = [] + dst_kv_groups: list[list] = [] + src_kv_caches: dict[str, torch.Tensor] = {} + dst_kv_caches: dict[str, torch.Tensor] = {} + for i in range(num_groups): + for groups, kv_caches in [ + (src_kv_groups, src_kv_caches), + (dst_kv_groups, dst_kv_caches), + ]: + kv_group = generate_kv_cache_paged_list_tensors( + num_blocks=num_blocks, + device=device, + block_size=block_size, + dtype=dtypes[i], + num_layers=8, + head_size=head_sizes[i], + gpu_kv_format=gpu_kv_format, + ) + groups.append(kv_group) + for j, layer_tensor in enumerate(kv_group): + kv_caches[f"{i}-{j}"] = layer_tensor + + slot_mapping = random.sample(range(0, num_blocks * block_size), num_tokens) + slot_mapping = torch.tensor(slot_mapping, device=device, dtype=torch.int64) + + # Check the kv group is not the same before copying + with pytest.raises(AssertionError): + for i in range(num_groups): + if use_mla: + check_paged_kv_cache_equal_with_mla( + src_kv_groups[i], dst_kv_groups[i], slot_mapping, head_sizes[i] + ) + else: + check_paged_kv_cache_equal( + src_kv_groups[i], + dst_kv_groups[i], + slot_mapping, + num_heads, + head_sizes[i], + gpu_kv_format, + ) + # create metadata and init kv layer groups + metadata = _create_metadata(use_mla, src_kv_caches, gpu_kv_format) + metadata2 = _create_metadata(use_mla, dst_kv_caches, gpu_kv_format) -@pytest.mark.parametrize("use_gpu", [False, True]) -def test_xpu_connector_roundtrip_layerwise(use_gpu: bool): - _skip_if_no_xpu() - device = torch.device("xpu:0") + # connector will copy with src_kv_groups + connector = VLLMPagedMemXPUConnectorV3( + metadata=metadata, + use_gpu=use_gpu, + device=slot_mapping.device, + ) + # connector2 will copy with dst_kv_groups + connector2 = VLLMPagedMemXPUConnectorV3( + metadata=metadata2, + use_gpu=use_gpu, + device=slot_mapping.device, + ) + assert connector.use_mla == use_mla + assert connector2.use_mla == use_mla + + # copy from src_kv_groups to memory_obj, + # and then copy from memory_obj to dst_kv_groups + for start in range(0, num_tokens, chunk_size): + end = min(start + chunk_size, num_tokens) + memory_obj = allocator.allocate( + metadata.get_shapes(end - start), metadata.get_dtypes() + ) + connector.from_gpu( + memory_obj, + start, + end, + kvcaches=list(src_kv_caches.values()), + slot_mapping=slot_mapping, + offset=0, + ) + if use_mla: + assert memory_obj.metadata.fmt == MemoryFormat.KV_MLA_FMT + else: + assert memory_obj.metadata.fmt == MemoryFormat.KV_2LTD + connector2.to_gpu( + memory_obj, + start, + end, + kvcaches=list(dst_kv_caches.values()), + slot_mapping=slot_mapping, + offset=0, + ) + allocator.free(memory_obj) + assert allocator.memcheck() + + # Check the kv group is same after copying + for i in range(num_groups): + if use_mla: + check_paged_kv_cache_equal_with_mla( + src_kv_groups[i], dst_kv_groups[i], slot_mapping, head_sizes[i] + ) + else: + check_paged_kv_cache_equal( + src_kv_groups[i], + dst_kv_groups[i], + slot_mapping, + num_heads, + head_sizes[i], + gpu_kv_format, + ) + allocator.close() - num_layers = 4 - num_blocks = 8 + +@pytest.mark.parametrize("use_gpu", [True]) +@pytest.mark.parametrize( + "gpu_kv_format", + [ + lmc_ops.GPUKVFormat.NL_X_TWO_NB_BS_NH_HS, # vllm non-MLA flash attention + lmc_ops.GPUKVFormat.NL_X_NB_TWO_BS_NH_HS, # vllm non-MLA flash infer + ], +) +@pytest.mark.skipif( + not torch.xpu.is_available(), + reason="TODO: Add non-XPU implementation to VLLMPagedMemLayerwiseXPUConnector", +) +def test_layerwise_vllm_paged_connector_with_gpu(use_gpu, gpu_kv_format): + num_blocks = 100 block_size = 16 - head_size = 64 - num_tokens = 64 + num_layers = 32 + num_heads = 8 + head_size = 128 + device = "xpu" + hidden_dim = num_heads * head_size - kvcaches = generate_kv_cache_paged_list_tensors( + num_tokens = 800 + chunk_size = 256 + + allocator = PinMemoryAllocator(1024 * 1024 * 1024) + + gpu_kv_src = generate_kv_cache_paged_list_tensors( num_blocks=num_blocks, + device=device, block_size=block_size, - num_layers=num_layers, - head_size=head_size, + gpu_kv_format=gpu_kv_format, + ) + gpu_kv_dst = generate_kv_cache_paged_list_tensors( + num_blocks=num_blocks, + device=device, + block_size=block_size, + gpu_kv_format=gpu_kv_format, + ) + dtype = get_dtype(gpu_kv_src, gpu_kv_format) + + slot_mapping = random.sample(range(0, num_blocks * block_size), num_tokens) + slot_mapping = torch.tensor(slot_mapping, device=device, dtype=torch.int64) + + # Check the gpu_kv is not the same before copying + with pytest.raises(AssertionError): + check_paged_kv_cache_equal( + gpu_kv_src, gpu_kv_dst, slot_mapping, num_heads, head_size, gpu_kv_format + ) + + connector = VLLMPagedMemLayerwiseXPUConnector( + hidden_dim, + num_layers, + use_gpu=use_gpu, + chunk_size=chunk_size, + dtype=dtype, device=device, ) - # Derive actual dims from generated KV - _, _, num_heads_actual, head_size_actual = kvcaches[0][0].shape - hidden_dim_actual = num_heads_actual * head_size_actual + # from gpu to cpu + starts = [] + ends = [] + memory_objs = [] + + for start in range(0, num_tokens, chunk_size): + end = min(start + chunk_size, num_tokens) + shape_single_layer = connector.get_shape(end - start) + memory_objs_multi_layer = [] + + for layer_id in range(num_layers): + mem_obj_single_layer = allocator.allocate( + shape_single_layer, dtype, fmt=MemoryFormat.KV_T2D + ) + memory_objs_multi_layer.append(mem_obj_single_layer) + + starts.append(start) + ends.append(end) + memory_objs.append(memory_objs_multi_layer) + + memory_objs = [list(row) for row in zip(*memory_objs, strict=False)] - total_slots = num_blocks * block_size - slot_mapping = _make_unique_slot_mapping( - total_slots=total_slots, num_tokens=num_tokens, device=device + mem_obj_generator = connector.batched_from_gpu( + memory_objs, + starts, + ends, + kvcaches=gpu_kv_src, + slot_mapping=slot_mapping, + sync=True, ) - meta = LMCacheMetadata( - model_name="xpu_test_layerwise", - world_size=1, - local_world_size=1, - worker_id=0, - local_worker_id=0, - kv_dtype=torch.bfloat16, - kv_shape=(num_layers, 2, num_tokens, num_heads_actual, head_size_actual), + for layer_id in range(num_layers + 1): + next(mem_obj_generator) + + # from cpu to gpu + mem_obj_consumer = connector.batched_to_gpu( + starts, + ends, + kvcaches=gpu_kv_dst, + slot_mapping=slot_mapping, + sync=True, + ) + next(mem_obj_consumer) + for layer_id in range(num_layers): + mem_obj_consumer.send(memory_objs[layer_id]) + next(mem_obj_consumer) + + # free all mem objs + for mem_obj_multi_layer in memory_objs: + for mem_obj in mem_obj_multi_layer: + mem_obj.ref_count_down() + + assert allocator.memcheck() + + assert connector.gpu_buffer_allocator.memcheck() + + check_paged_kv_cache_equal( + gpu_kv_src, gpu_kv_dst, slot_mapping, num_heads, head_size, gpu_kv_format + ) + + allocator.close() + + +@pytest.mark.parametrize("use_gpu", [True]) +@pytest.mark.skipif( + not torch.xpu.is_available(), + reason="TODO: Add non-XPU implementation to VLLMPagedMemLayerwiseXPUConnector", +) +def test_batched_layerwise_vllm_paged_connector_with_gpu(use_gpu): + num_blocks = 100 + block_size = 16 + num_layers = 32 + num_heads = 8 + head_size = 128 + device = "xpu" + hidden_dim = num_heads * head_size + + num_tokens_1 = 800 + num_tokens_2 = 500 + num_tokens_total = num_tokens_1 + num_tokens_2 + chunk_size = 256 + + allocator = PinMemoryAllocator(1024 * 1024 * 1024) + + gpu_kv_src = generate_kv_cache_paged_list_tensors(num_blocks, device, block_size) + gpu_kv_dst = generate_kv_cache_paged_list_tensors(num_blocks, device, block_size) + dtype = gpu_kv_src[0][0].dtype + + slot_mapping_total = random.sample( + range(0, num_blocks * block_size), num_tokens_total + ) + slot_mapping_total = torch.tensor( + slot_mapping_total, device=device, dtype=torch.int64 ) - conn = VLLMPagedMemLayerwiseXPUConnector.from_metadata( - meta, - use_xpu=use_gpu, + # Check the gpu_kv is not the same before copying + with pytest.raises(AssertionError): + check_paged_kv_cache_equal(gpu_kv_src, gpu_kv_dst, slot_mapping_total) + + connector = VLLMPagedMemLayerwiseXPUConnector( + hidden_dim, + num_layers, + use_gpu=use_gpu, + chunk_size=chunk_size, + dtype=dtype, device=device, ) - pin_alloc = PinMemoryAllocator(size=1024 * 1024 * 256) + # from gpu to cpu + starts_1 = [] + ends_1 = [] + memory_objs_1 = [] + + for start in range(0, num_tokens_1, chunk_size): + end = min(start + chunk_size, num_tokens_1) + shape_single_layer = connector.get_shape(end - start) + memory_objs_multi_layer = [] - # Per-layer list-of-chunks. We use 1 chunk: [0, num_tokens) - memobjs_by_layer = [ - [ - pin_alloc.allocate( - torch.Size([num_tokens, 2, hidden_dim_actual]), - torch.bfloat16, - MemoryFormat.KV_T2D, + for layer_id in range(num_layers): + mem_obj_single_layer = allocator.allocate( + shape_single_layer, dtype, fmt=MemoryFormat.KV_T2D ) - ] - for _ in range(num_layers) - ] + memory_objs_multi_layer.append(mem_obj_single_layer) - try: - # XPU -> CPU (layerwise generator): yields num_layers + 1 times - gen = conn.batched_from_gpu( - memobjs_by_layer, - starts=[0], - ends=[num_tokens], - slot_mapping=slot_mapping, - sync=True, - kvcaches=kvcaches, - ) + starts_1.append(start) + ends_1.append(end) + memory_objs_1.append(memory_objs_multi_layer) - # Drive generator: one yield per layer + final yield - for _ in range(num_layers + 1): - next(gen) - - # CPU -> XPU into fresh caches (layerwise generator): - kvcaches_dst = generate_kv_cache_paged_list_tensors( - num_blocks=num_blocks, - block_size=block_size, - num_layers=num_layers, - head_size=head_size_actual, - device=device, - ) - for t in kvcaches_dst: - t.zero_() + memory_objs_1 = [list(row) for row in zip(*memory_objs_1, strict=False)] - gen2 = conn.batched_to_gpu( - starts=[0], - ends=[num_tokens], - slot_mapping=slot_mapping, - sync=True, - kvcaches=kvcaches_dst, - ) + starts_2 = [] + ends_2 = [] + memory_objs_2 = [] + for start in range(num_tokens_1, num_tokens_total, chunk_size): + end = min(start + chunk_size, num_tokens_total) + shape_single_layer = connector.get_shape(end - start) + memory_objs_multi_layer = [] - next(gen2) # layer 0 expects send() for layer_id in range(num_layers): - gen2.send(memobjs_by_layer[layer_id]) + mem_obj_single_layer = allocator.allocate( + shape_single_layer, dtype, fmt=MemoryFormat.KV_T2D + ) + memory_objs_multi_layer.append(mem_obj_single_layer) - # After the last send, generator is at "yield # after last layer" - next(gen2) # advances to "yield # final" + starts_2.append(start) + ends_2.append(end) + memory_objs_2.append(memory_objs_multi_layer) - check_paged_kv_cache_equal( - kvcaches, - kvcaches_dst, - slot_mapping, - num_heads=num_heads_actual, - head_size=head_size_actual, - ) - finally: - for layer in memobjs_by_layer: - for m in layer: - m.ref_count_down() - pin_alloc.close() - - -@pytest.mark.parametrize("use_gpu", [False, True]) -def test_xpu_connector_roundtrip_non_layerwise_multi_chunk( - use_gpu: bool, -) -> None: - _skip_if_no_xpu() - device = torch.device("xpu:0") - - num_layers = 2 - num_blocks = 6 - block_size = 8 - head_size = 64 - total_tokens = 32 - - starts = [0, 7, 19] - ends = [4, 13, 25] - - kvcaches = generate_kv_cache_paged_list_tensors( - num_blocks=num_blocks, - block_size=block_size, - num_layers=num_layers, - head_size=head_size, - device=device, + memory_objs_2 = [list(row) for row in zip(*memory_objs_2, strict=False)] + + mem_obj_generator_1 = connector.batched_from_gpu( + memory_objs_1, + starts_1, + ends_1, + kvcaches=gpu_kv_src, + slot_mapping=slot_mapping_total, + sync=True, ) - _, _, num_heads_actual, head_size_actual = kvcaches[0][0].shape - hidden_dim_actual = num_heads_actual * head_size_actual - slot_mapping = _make_unique_slot_mapping( - total_slots=num_blocks * block_size, - num_tokens=total_tokens, - device=device, + mem_obj_generator_1 = connector.batched_from_gpu( + memory_objs_1, + starts_1, + ends_1, + kvcaches=gpu_kv_src, + slot_mapping=slot_mapping_total, + sync=True, ) - packed_slot_mapping = _pack_slot_mapping(slot_mapping, starts, ends) - meta = LMCacheMetadata( - model_name="xpu_test_non_layerwise_multi_chunk", - world_size=1, - local_world_size=1, - worker_id=0, - local_worker_id=0, - kv_dtype=torch.bfloat16, - kv_shape=(num_layers, 2, total_tokens, num_heads_actual, head_size_actual), + mem_obj_generator_2 = connector.batched_from_gpu( + memory_objs_2, + starts_2, + ends_2, + kvcaches=gpu_kv_src, + slot_mapping=slot_mapping_total, + sync=False, ) - conn = VLLMPagedMemXPUConnectorV2.from_metadata( - meta, - use_gpu=use_gpu, - device=device, + + for layer_id in range(num_layers + 1): + next(mem_obj_generator_1) + next(mem_obj_generator_2) + + # from cpu to gpu + mem_obj_consumer_1 = connector.batched_to_gpu( + starts_1, + ends_1, + kvcaches=gpu_kv_dst, + slot_mapping=slot_mapping_total, + sync=False, + ) + mem_obj_consumer_2 = connector.batched_to_gpu( + starts_2, + ends_2, + kvcaches=gpu_kv_dst, + slot_mapping=slot_mapping_total, + sync=True, ) - pin_alloc = PinMemoryAllocator(size=1024 * 1024 * 64) - memobjs = [] - try: - for s, e in zip(starts, ends, strict=False): - n = e - s - memobj = pin_alloc.allocate( - torch.Size([2, num_layers, n, hidden_dim_actual]), - torch.bfloat16, - MemoryFormat.KV_2LTD, - ) - conn.from_gpu( - memobj, - start=s, - end=e, - slot_mapping=slot_mapping, - kvcaches=kvcaches, - ) - memobjs.append((s, e, memobj)) - - kvcaches_dst = generate_kv_cache_paged_list_tensors( - num_blocks=num_blocks, - block_size=block_size, - num_layers=num_layers, - head_size=head_size_actual, - device=device, - ) - for layer in kvcaches_dst: - layer.zero_() - - for s, e, memobj in memobjs: - conn.to_gpu( - memobj, - start=s, - end=e, - slot_mapping=slot_mapping, - kvcaches=kvcaches_dst, - ) + next(mem_obj_consumer_1) + next(mem_obj_consumer_2) + for layer_id in range(num_layers): + mem_obj_consumer_1.send(memory_objs_1[layer_id]) + mem_obj_consumer_2.send(memory_objs_2[layer_id]) + next(mem_obj_consumer_1) + next(mem_obj_consumer_2) + + # free all mem objs + for mem_obj_multi_layer in memory_objs_1: + for mem_obj in mem_obj_multi_layer: + mem_obj.ref_count_down() + + for mem_obj_multi_layer in memory_objs_2: + for mem_obj in mem_obj_multi_layer: + mem_obj.ref_count_down() + + assert allocator.memcheck() + + assert connector.gpu_buffer_allocator.memcheck() + + check_paged_kv_cache_equal( + gpu_kv_src, + gpu_kv_dst, + slot_mapping_total, + num_heads, + head_size, + ) - check_paged_kv_cache_equal( - kvcaches, - kvcaches_dst, - packed_slot_mapping, - num_heads=num_heads_actual, - head_size=head_size_actual, - ) - finally: - for _, _, memobj in memobjs: - memobj.ref_count_down() - pin_alloc.close() + allocator.close() -@pytest.mark.parametrize("use_xpu", [False, True]) -def test_xpu_connector_roundtrip_layerwise_multi_chunk(use_xpu: bool) -> None: - _skip_if_no_xpu() - device = torch.device("xpu:0") +@pytest.mark.skip(reason="This test is skipped due to vllm dependency") +@pytest.mark.parametrize("use_gpu", [True]) +@pytest.mark.skipif( + not torch.xpu.is_available(), + reason="TODO: Add non-XPU implementation to VLLMBufferLayerwiseXPUConnector", +) +def test_layerwise_vllm_buffer_connector_with_gpu(use_gpu): + num_blocks = 100 + block_size = 16 + num_layers = 32 + num_heads = 8 + head_size = 128 + device = "xpu" + hidden_dim = num_heads * head_size - num_layers = 4 - num_blocks = 8 - block_size = 8 - head_size = 64 - total_tokens = 40 + num_tokens = 800 + chunk_size = 256 - starts = [0, 9, 21] - ends = [5, 15, 30] + allocator = PinMemoryAllocator(1024 * 1024 * 1024) - kvcaches = generate_kv_cache_paged_list_tensors( - num_blocks=num_blocks, - block_size=block_size, - num_layers=num_layers, - head_size=head_size, + gpu_kv_src = generate_kv_cache_paged_list_tensors(num_blocks, device, block_size) + gpu_kv_dst = generate_kv_cache_paged_list_tensors(num_blocks, device, block_size) + dtype = gpu_kv_src[0][0].dtype + + slot_mapping = random.sample(range(0, num_blocks * block_size), num_tokens) + slot_mapping = torch.tensor(slot_mapping, device=device, dtype=torch.int64) + + # Check the gpu_kv is not the same before copying + with pytest.raises(AssertionError): + check_paged_kv_cache_equal( + gpu_kv_src, gpu_kv_dst, slot_mapping, num_heads, head_size + ) + + connector = VLLMBufferLayerwiseXPUConnector( + hidden_dim, + num_layers, + use_gpu=use_gpu, + dtype=dtype, device=device, ) - _, _, num_heads_actual, head_size_actual = kvcaches[0][0].shape - hidden_dim_actual = num_heads_actual * head_size_actual + # from gpu to cpu + starts = [] + ends = [] + memory_objs = [] - slot_mapping = _make_unique_slot_mapping( - total_slots=num_blocks * block_size, - num_tokens=total_tokens, - device=device, + for start in range(0, num_tokens, chunk_size): + end = min(start + chunk_size, num_tokens) + shape_single_layer = connector.get_shape(end - start) + memory_objs_multi_layer = [] + + for layer_id in range(num_layers): + mem_obj_single_layer = allocator.allocate( + shape_single_layer, dtype, fmt=MemoryFormat.KV_2TD + ) + memory_objs_multi_layer.append(mem_obj_single_layer) + + starts.append(start) + ends.append(end) + memory_objs.append(memory_objs_multi_layer) + + memory_objs = [list(row) for row in zip(*memory_objs, strict=False)] + + mem_obj_generator = connector.batched_from_gpu( + memory_objs, + starts, + ends, + kvcaches=gpu_kv_src, + slot_mapping=slot_mapping, ) - packed_slot_mapping = _pack_slot_mapping(slot_mapping, starts, ends) - meta = LMCacheMetadata( - model_name="xpu_test_layerwise_multi_chunk_gpu", - world_size=1, - local_world_size=1, - worker_id=0, - local_worker_id=0, - kv_dtype=torch.bfloat16, - kv_shape=(num_layers, 2, total_tokens, num_heads_actual, head_size_actual), + for layer_id in range(num_layers + 1): + next(mem_obj_generator) + + # from cpu to gpu + mem_obj_consumer = connector.batched_to_gpu( + starts, + ends, + kvcaches=gpu_kv_dst, + slot_mapping=slot_mapping, ) - conn = VLLMPagedMemLayerwiseXPUConnector.from_metadata( - meta, - use_xpu=use_xpu, - device=device, + next(mem_obj_consumer) + for layer_id in range(num_layers): + mem_obj_consumer.send(memory_objs[layer_id]) + next(mem_obj_consumer) + + # free all mem objs + for mem_obj_multi_layer in memory_objs: + for mem_obj in mem_obj_multi_layer: + mem_obj.ref_count_down() + + assert allocator.memcheck() + + assert connector.gpu_buffer_allocator.memcheck() + + check_paged_kv_cache_equal( + gpu_kv_src, gpu_kv_dst, slot_mapping, num_heads, head_size ) - pin_alloc = PinMemoryAllocator(size=1024 * 1024 * 128) - memobjs_by_layer = [] - for _ in range(num_layers): - per_layer = [] - for s, e in zip(starts, ends, strict=False): - n = e - s - per_layer.append( - pin_alloc.allocate( - torch.Size([n, 2, hidden_dim_actual]), - torch.bfloat16, - MemoryFormat.KV_T2D, - ) - ) - memobjs_by_layer.append(per_layer) + allocator.close() - try: - producer = conn.batched_from_gpu( - memobjs_by_layer, - starts=starts, - ends=ends, - slot_mapping=slot_mapping, - sync=True, - kvcaches=kvcaches, - ) - for _ in range(num_layers + 1): - next(producer) - if use_xpu: - assert conn.gpu_buffer_allocator is not None - else: - assert conn.gpu_buffer_allocator is None - - kvcaches_dst = generate_kv_cache_paged_list_tensors( - num_blocks=num_blocks, - block_size=block_size, - num_layers=num_layers, - head_size=head_size_actual, - device=device, - ) - for layer in kvcaches_dst: - layer.zero_() +@pytest.mark.skipif( + not torch.xpu.is_available(), + reason="TODO: Add non-XPU implementation to VLLMPagedMemXPUConnectorV2", +) +def test_vllm_paged_connector_v2_to_gpu_bench(benchmark): + """ + VLLMPagedMemXPUConnectorV2.to_gpu() micro-benchmark. - consumer = conn.batched_to_gpu( - starts=starts, - ends=ends, - slot_mapping=slot_mapping, - sync=True, - kvcaches=kvcaches_dst, - ) - next(consumer) - for layer_id in range(num_layers): - consumer.send(memobjs_by_layer[layer_id]) - next(consumer) + This test is to measure the performance of + VLLMPagedMemXPUConnectorV2.to_gpu() when both KV caches and + memobject are on GPU. - check_paged_kv_cache_equal( - kvcaches, - kvcaches_dst, - packed_slot_mapping, - num_heads=num_heads_actual, - head_size=head_size_actual, - ) - finally: - for layer in memobjs_by_layer: - for memobj in layer: - memobj.ref_count_down() - pin_alloc.close() + """ + num_blocks = 100 + block_size = 16 + num_layers = 32 + num_heads = 8 + head_size = 128 + device = "xpu" + hidden_dim = num_heads * head_size + + chunk_size = 256 + + allocator = GPUMemoryAllocator(1024 * 1024 * 1024, device) + + gpu_kv_src = generate_kv_cache_paged_list_tensors(num_blocks, device, block_size) + gpu_kv_dst = generate_kv_cache_paged_list_tensors(num_blocks, device, block_size) + + slot_mapping = random.sample(range(0, num_blocks * block_size), chunk_size) + slot_mapping = torch.tensor(slot_mapping, device=device, dtype=torch.int64) + + connector = VLLMPagedMemXPUConnectorV2(hidden_dim, num_layers) + shape = connector.get_shape(chunk_size) + memory_obj = allocator.allocate(shape, gpu_kv_src[0][0].dtype) + connector.from_gpu( + memory_obj, + 0, + chunk_size, + kvcaches=gpu_kv_src, + slot_mapping=slot_mapping, + offset=0, + ) + recover_gpu_connector_states(connector) + assert memory_obj.metadata.fmt == MemoryFormat.KV_2LTD + benchmark.pedantic( + connector.to_gpu, + args=(memory_obj, 0, chunk_size), + kwargs={ + "kvcaches": gpu_kv_dst, + "slot_mapping": slot_mapping, + "offset": 0, + }, + rounds=100, + iterations=1000, + warmup_rounds=10, + ) + allocator.free(memory_obj) + assert allocator.memcheck() + + allocator.close() + + +def _create_metadata(use_mla, kv_caches, gpu_kv_format): + # First Party + from lmcache.v1.gpu_connector.utils import get_block_size, get_num_blocks + from lmcache.v1.kv_layer_groups import KVLayerGroupsManager + + num_heads = 1 if use_mla else 8 + metadata = LMCacheMetadata( + model_name="test", + world_size=8, + local_world_size=8, + worker_id=0, + local_worker_id=0, + kv_dtype=torch.bfloat16, + kv_shape=(32, 2, 256, num_heads, 128), + use_mla=use_mla, + ) + kv_list = list(kv_caches.values()) + metadata.kv_layer_groups_manager = KVLayerGroupsManager( + kv_list, + gpu_kv_format=gpu_kv_format, + num_blocks=get_num_blocks(kv_list, gpu_kv_format), + block_size=get_block_size(kv_list, gpu_kv_format), + ) + return metadata diff --git a/tests/v1/utils.py b/tests/v1/utils.py index 88a4711e38e..94826b20f80 100644 --- a/tests/v1/utils.py +++ b/tests/v1/utils.py @@ -25,7 +25,7 @@ from lmcache.v1.metadata import LMCacheMetadata # Conditional import for CUDA-only operations -if torch.cuda.is_available(): +if torch.cuda.is_available() or torch.xpu.is_available(): try: # First Party import lmcache.c_ops as lmc_ops @@ -34,6 +34,7 @@ lmc_ops = None else: # Mock c_ops when CUDA is not available + # First Party lmc_ops = None # Define mock GPUKVFormat enum if c_ops is not available @@ -310,6 +311,8 @@ def generate_kv_cache_paged_list_tensors( shape = [2, num_blocks, num_heads, block_size, head_size] elif gpu_kv_format == lmc_ops.GPUKVFormat.NL_X_NB_TWO_NH_BS_HS: shape = [num_blocks, 2, num_heads, block_size, head_size] + else: + raise ValueError(f"Unsupported gpu_kv_format: {gpu_kv_format}") for i in range(num_layers): # TODO(chunxiaozheng): support more dtypes From 9117259f994f11458dbbe1642894350917b020f6 Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Fri, 15 May 2026 10:48:08 -0700 Subject: [PATCH 16/69] [CD] switch all workflows to egress audit mode (#3297) Signed-off-by: deng451e <838677410@qq.com> --- .github/workflows/actionlint.yml | 7 +-- .github/workflows/build_cli_artifacts.yml | 9 +-- .github/workflows/build_cu129_artifacts.yml | 18 +----- .github/workflows/build_doc.yml | 4 +- .github/workflows/build_main_artifacts.yml | 18 +----- .github/workflows/code_quality_checks.yml | 9 +-- .github/workflows/codeql.yml | 26 +------- .github/workflows/nightly_build.yml | 41 +----------- .github/workflows/publish.yml | 69 ++------------------- .github/workflows/scorecard.yml | 15 +---- .github/workflows/stale_bot.yml | 4 +- .github/workflows/test.yml | 2 +- 12 files changed, 19 insertions(+), 203 deletions(-) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index fc47f54f357..71f05488b71 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -35,12 +35,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - github.com:443 - registry-1.docker.io:443 - production.cloudflare.docker.com:443 - auth.docker.io:443 + egress-policy: audit - name: "Checkout" uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 diff --git a/.github/workflows/build_cli_artifacts.yml b/.github/workflows/build_cli_artifacts.yml index a1e522ec47a..3b21cf523ae 100644 --- a/.github/workflows/build_cli_artifacts.yml +++ b/.github/workflows/build_cli_artifacts.yml @@ -39,14 +39,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - api.github.com:443 - github.com:443 - azure.archive.ubuntu.com:80 - pypi.org:443 - files.pythonhosted.org:443 - objects.githubusercontent.com:443 + egress-policy: audit - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 diff --git a/.github/workflows/build_cu129_artifacts.yml b/.github/workflows/build_cu129_artifacts.yml index 8c6caf3d668..dc12c188a01 100644 --- a/.github/workflows/build_cu129_artifacts.yml +++ b/.github/workflows/build_cu129_artifacts.yml @@ -28,23 +28,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - api.github.com:443 - github.com:443 - registry-1.docker.io:443 - index.docker.io:443 - production.cloudflare.docker.com:443 - auth.docker.io:443 - azure.archive.ubuntu.com:80 - pypi.org:443 - files.pythonhosted.org:443 - codeload.github.com:443 - objects.githubusercontent.com:443 - releases.githubusercontent.com:443 - download.pytorch.org:443 - download-r2.pytorch.org:443 - pypi.nvidia.com:443 + egress-policy: audit - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 diff --git a/.github/workflows/build_doc.yml b/.github/workflows/build_doc.yml index 3419dae68b6..64247a757e8 100644 --- a/.github/workflows/build_doc.yml +++ b/.github/workflows/build_doc.yml @@ -19,7 +19,7 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: - egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + egress-policy: audit - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 @@ -66,7 +66,7 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: - egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + egress-policy: audit - name: Fetch doc artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 diff --git a/.github/workflows/build_main_artifacts.yml b/.github/workflows/build_main_artifacts.yml index 29e2a6ba0b0..d8fba7ba4bc 100644 --- a/.github/workflows/build_main_artifacts.yml +++ b/.github/workflows/build_main_artifacts.yml @@ -27,23 +27,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - api.github.com:443 - github.com:443 - registry-1.docker.io:443 - index.docker.io:443 - production.cloudflare.docker.com:443 - auth.docker.io:443 - azure.archive.ubuntu.com:80 - pypi.org:443 - pypi.nvidia.com:443 - files.pythonhosted.org:443 - codeload.github.com:443 - objects.githubusercontent.com:443 - releases.githubusercontent.com:443 - download.pytorch.org:443 - download-r2.pytorch.org:443 + egress-policy: audit - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 diff --git a/.github/workflows/code_quality_checks.yml b/.github/workflows/code_quality_checks.yml index 8a27bd26c22..553d7bb5e0e 100644 --- a/.github/workflows/code_quality_checks.yml +++ b/.github/workflows/code_quality_checks.yml @@ -47,14 +47,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: true - egress-policy: block - allowed-endpoints: > - github.com:443 - pypi.org:443 - files.pythonhosted.org:443 - crates.io:443 - index.crates.io:443 - static.crates.io:443 + egress-policy: audit - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 18cd7c3d855..ed56272b15e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -63,31 +63,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: true - egress-policy: block - allowed-endpoints: > - github.com:443 - api.github.com:443 - objects.githubusercontent.com:443 - github-releases.githubusercontent.com:443 - raw.githubusercontent.com:443 - packages.githubusercontent.com:443 - pypi.org:443 - files.pythonhosted.org:443 - registry.npmjs.org:443 - registry-1.docker.io:443 - production.cloudflare.docker.com:443 - auth.docker.io:443 - azure.archive.ubuntu.com:80 - security.ubuntu.com:80 - storage.googleapis.com:443 - *.github.com:443 - *.githubusercontent.com:443 - pypi.org:443 - files.pythonhosted.org:443 - registry.npmjs.org:443 - *.docker.io:443 - *.ubuntu.com:80 - storage.googleapis.com:443 + egress-policy: audit - name: Checkout repository uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index d5117141790..6160988fc1a 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -19,24 +19,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - api.github.com:443 - uploads.github.com:443 - github.com:443 - registry-1.docker.io:443 - index.docker.io:443 - production.cloudflare.docker.com:443 - auth.docker.io:443 - azure.archive.ubuntu.com:80 - pypi.org:443 - files.pythonhosted.org:443 - objects.githubusercontent.com:443 - releases.githubusercontent.com:443 - codeload.github.com:443 - download.pytorch.org:443 - download-r2.pytorch.org:443 - pypi.nvidia.com:443 + egress-policy: audit - name: Checkout code uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 @@ -138,27 +121,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - github.com:443 - registry-1.docker.io:443 - production.cloudflare.docker.com:443 - auth.docker.io:443 - azure.archive.ubuntu.com:80 - pypi.org:443 - files.pythonhosted.org:443 - wheels.vllm.ai:443 - release-assets.githubusercontent.com:443 - wrapdb.mesonbuild.com:443 - nvcr.io:443 - layers.nvcr.io:443 - archive.ubuntu.com:80 - security.ubuntu.com:80 - astral.sh:443 - releases.astral.sh:443 - download.pytorch.org:443 - download-r2.pytorch.org:443 - pypi.nvidia.com:443 + egress-policy: audit - name: Login to DockerHub if: vars.DOCKERHUB_USERNAME != '' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index da48ec122ab..b20e7ba088e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -118,14 +118,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - ghcr.io:443 - pkg-containers.githubusercontent.com:443 - test.pypi.org:443 - tuf-repo-cdn.sigstore.dev:443 - fulcio.sigstore.dev:443 - rekor.sigstore.dev:443 + egress-policy: audit - name: Fetch release artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 @@ -157,14 +150,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - ghcr.io:443 - pkg-containers.githubusercontent.com:443 - test.pypi.org:443 - tuf-repo-cdn.sigstore.dev:443 - fulcio.sigstore.dev:443 - rekor.sigstore.dev:443 + egress-policy: audit - name: Fetch CLI release artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 @@ -200,16 +186,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - api.github.com:443 - uploads.github.com:443 - ghcr.io:443 - pkg-containers.githubusercontent.com:443 - upload.pypi.org:443 - tuf-repo-cdn.sigstore.dev:443 - fulcio.sigstore.dev:443 - rekor.sigstore.dev:443 + egress-policy: audit - name: Fetch release artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 @@ -245,16 +222,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - api.github.com:443 - uploads.github.com:443 - ghcr.io:443 - pkg-containers.githubusercontent.com:443 - upload.pypi.org:443 - tuf-repo-cdn.sigstore.dev:443 - fulcio.sigstore.dev:443 - rekor.sigstore.dev:443 + egress-policy: audit - name: Fetch CLI release artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 @@ -290,10 +258,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - api.github.com:443 - uploads.github.com:443 + egress-policy: audit - name: Fetch cu12.9 release artifacts uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 @@ -337,29 +302,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - github.com:443 - registry-1.docker.io:443 - index.docker.io:443 - production.cloudflare.docker.com:443 - auth.docker.io:443 - azure.archive.ubuntu.com:80 - archive.ubuntu.com:80 - security.ubuntu.com:80 - astral.sh:443 - releases.astral.sh:443 - objects.githubusercontent.com:443 - pypi.org:443 - pypi.nvidia.com:443 - files.pythonhosted.org:443 - release-assets.githubusercontent.com:443 - wrapdb.mesonbuild.com:443 - nvcr.io:443 - layers.nvcr.io:443 - wheels.vllm.ai:443 - download.pytorch.org:443 - download-r2.pytorch.org:443 + egress-policy: audit - name: Login to DockerHub uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index f483a9d7fe0..0c61cd5d4f7 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -37,20 +37,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: false - egress-policy: block - allowed-endpoints: > - github.com:443 - api.github.com:443 - index.docker.io:443 - www.bestpractices.dev:443 - oss-fuzz-build-logs.storage.googleapis.com:443 - api.osv.dev:443 - api.deps.dev:443 - fulcio.sigstore.dev:443 - tuf-repo-cdn.sigstore.dev:443 - rekor.sigstore.dev:443 - auth.docker.io:443 - api.scorecard.dev:443 + egress-policy: audit - name: "Checkout code" uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 diff --git a/.github/workflows/stale_bot.yml b/.github/workflows/stale_bot.yml index 27033b0135b..64e0ec334da 100644 --- a/.github/workflows/stale_bot.yml +++ b/.github/workflows/stale_bot.yml @@ -25,9 +25,7 @@ jobs: uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: disable-sudo-and-containers: true - egress-policy: block - allowed-endpoints: > - api.github.com:443 + egress-policy: audit - name: "Stale Action" uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v10.1.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd3f010c0bd..62e36f2ff0c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,7 +70,7 @@ jobs: - name: "Harden Runner" uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 with: - egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + egress-policy: audit - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 From 86f3c751519d6e3cdd09c658b0241174f1d0e238 Mon Sep 17 00:00:00 2001 From: Roy Huang Date: Fri, 15 May 2026 14:11:41 -0700 Subject: [PATCH 17/69] [Operator][CI] Date-based nightly tag and release-driven flow (#3293) Signed-off-by: royyhuang --- .github/workflows/operator_nightly.yml | 5 ++--- .github/workflows/operator_release.yml | 11 +++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/operator_nightly.yml b/.github/workflows/operator_nightly.yml index d2f8db621a9..22490eab762 100644 --- a/.github/workflows/operator_nightly.yml +++ b/.github/workflows/operator_nightly.yml @@ -41,9 +41,8 @@ jobs: - name: Compute nightly tag id: tag run: | - DATE=$(date -u +%Y%m%d) - SHA=$(git rev-parse --short HEAD) - echo "TAG=nightly-${DATE}-${SHA}" >> "$GITHUB_OUTPUT" + DATE=$(date -u +%Y-%m-%d) + echo "TAG=nightly-${DATE}" >> "$GITHUB_OUTPUT" - name: Build and push image uses: docker/build-push-action@v6 diff --git a/.github/workflows/operator_release.yml b/.github/workflows/operator_release.yml index 2887e95ce9e..d0484cfff17 100644 --- a/.github/workflows/operator_release.yml +++ b/.github/workflows/operator_release.yml @@ -1,9 +1,8 @@ name: "Operator: Release" on: - push: - tags: - - "operator-v*" + release: + types: [published] env: IMAGE_NAME: lmcache/lmcache-operator @@ -19,6 +18,8 @@ defaults: jobs: release: name: Build, push, and release + # Filter out main lmcache `v*` releases — only fire for operator-prefixed tags. + if: startsWith(github.ref_name, 'operator-v') runs-on: ubuntu-latest steps: - name: Harden Runner @@ -63,11 +64,9 @@ jobs: - name: Build installer manifest run: make build-installer IMG=${{ env.IMAGE_NAME }}:${{ steps.version.outputs.VERSION }} - - name: Create GitHub Release + - name: Attach installer to GitHub Release uses: softprops/action-gh-release@v2 with: - name: "Operator ${{ steps.version.outputs.VERSION }}" - generate_release_notes: true files: operator/dist/install.yaml - name: Update latest release alias From 359f27f64465eeb6a74bf34c765c6fba04019449 Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Fri, 15 May 2026 16:09:31 -0700 Subject: [PATCH 18/69] [CD] add sm_120 to cu129 wheel build (#3299) --- .github/workflows/build_cu129_artifacts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_cu129_artifacts.yml b/.github/workflows/build_cu129_artifacts.yml index dc12c188a01..1dc7dbf211a 100644 --- a/.github/workflows/build_cu129_artifacts.yml +++ b/.github/workflows/build_cu129_artifacts.yml @@ -76,7 +76,7 @@ jobs: - name: Build cu12.9 CUDA wheels with cibuildwheel env: CIBW_MANYLINUX_X86_64_IMAGE: docker.io/pytorch/manylinux2_28-builder:cuda12.9 - CIBW_ENVIRONMENT: 'TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0;8.6;8.9;9.0;10.0" ENABLE_CXX11_ABI=1 LMCACHE_CUDA_MAJOR=12 SETUPTOOLS_SCM_PRETEND_VERSION=${{ env.CU129_VERSION }} PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cu129' + CIBW_ENVIRONMENT: 'TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0;8.6;8.9;9.0;10.0;12.0" ENABLE_CXX11_ABI=1 LMCACHE_CUDA_MAJOR=12 SETUPTOOLS_SCM_PRETEND_VERSION=${{ env.CU129_VERSION }} PIP_EXTRA_INDEX_URL=https://download.pytorch.org/whl/cu129' CIBW_REPAIR_WHEEL_COMMAND_LINUX: >- auditwheel repair --plat manylinux_2_28_x86_64 From d63f68dbd426dd3967e442f5e7f4e1d1230d9e15 Mon Sep 17 00:00:00 2001 From: Roy Huang Date: Fri, 15 May 2026 16:13:34 -0700 Subject: [PATCH 19/69] [Test] Operator end-to-end smoke suite (#3289) * [Test] Operator: smoke suite Signed-off-by: royyhuang Co-authored-by: Yihua Cheng --- .buildkite/operator/integration/pipeline.yaml | 28 ++ .buildkite/operator/integration/tests.yaml | 30 ++ .github/workflows/operator_test_e2e.yml | 75 ----- operator/AGENTS.md | 275 +++++++++++++++- operator/DESIGN.md | 14 +- operator/Makefile | 256 ++------------- operator/README.md | 93 +++++- operator/api/v1alpha1/lmcacheengine_types.go | 5 + .../lmcache.lmcache.ai_lmcacheengines.yaml | 9 +- operator/go.mod | 2 +- .../internal/controller/reconcile_helpers.go | 39 ++- operator/make/build.mk | 43 +++ operator/make/deploy.mk | 24 ++ operator/make/dev.mk | 21 ++ operator/make/e2e-gpu.mk | 175 +++++++++++ operator/make/e2e.mk | 64 ++++ operator/make/lint.mk | 22 ++ operator/make/tools.mk | 83 +++++ operator/test/e2e/auth_smoke_test.go | 155 +++++++++ operator/test/e2e/crd_smoke_test.go | 249 +++++++++++++++ operator/test/e2e/e2e_suite_test.go | 158 +++++++--- operator/test/e2e/e2e_test.go | 45 +-- .../test/e2e/field_coverage_smoke_test.go | 133 ++++++++ operator/test/e2e/lifecycle_smoke_test.go | 146 +++++++++ operator/test/e2e/runtime_smoke_test.go | 117 +++++++ operator/test/e2e/smoke_helpers_test.go | 261 ++++++++++++++++ .../test/e2e/vllm_integration_smoke_test.go | 293 ++++++++++++++++++ operator/test/utils/fixtures.go | 82 +++++ .../golden/kv_transfer_config_minimal.json | 8 + .../test/utils/fixtures/lmc_custom_port.yaml | 18 ++ operator/test/utils/fixtures/lmc_minimal.yaml | 19 ++ operator/test/utils/fixtures/lmc_runtime.yaml | 22 ++ .../utils/fixtures/lmc_servicemonitor.yaml | 20 ++ .../lmc_with_redis_l2_authsecret.yaml | 22 ++ .../test/utils/fixtures/vllm_deployment.yaml | 122 ++++++++ operator/test/utils/http.go | 157 ++++++++++ operator/test/utils/lmc.go | 209 +++++++++++++ operator/test/utils/portforward.go | 173 +++++++++++ operator/test/utils/runner.go | 65 ++++ operator/test/utils/utils.go | 81 ----- operator/test/utils/wait.go | 113 +++++++ 41 files changed, 3431 insertions(+), 495 deletions(-) create mode 100644 .buildkite/operator/integration/pipeline.yaml create mode 100644 .buildkite/operator/integration/tests.yaml delete mode 100644 .github/workflows/operator_test_e2e.yml create mode 100644 operator/make/build.mk create mode 100644 operator/make/deploy.mk create mode 100644 operator/make/dev.mk create mode 100644 operator/make/e2e-gpu.mk create mode 100644 operator/make/e2e.mk create mode 100644 operator/make/lint.mk create mode 100644 operator/make/tools.mk create mode 100644 operator/test/e2e/auth_smoke_test.go create mode 100644 operator/test/e2e/crd_smoke_test.go create mode 100644 operator/test/e2e/field_coverage_smoke_test.go create mode 100644 operator/test/e2e/lifecycle_smoke_test.go create mode 100644 operator/test/e2e/runtime_smoke_test.go create mode 100644 operator/test/e2e/smoke_helpers_test.go create mode 100644 operator/test/e2e/vllm_integration_smoke_test.go create mode 100644 operator/test/utils/fixtures.go create mode 100644 operator/test/utils/fixtures/golden/kv_transfer_config_minimal.json create mode 100644 operator/test/utils/fixtures/lmc_custom_port.yaml create mode 100644 operator/test/utils/fixtures/lmc_minimal.yaml create mode 100644 operator/test/utils/fixtures/lmc_runtime.yaml create mode 100644 operator/test/utils/fixtures/lmc_servicemonitor.yaml create mode 100644 operator/test/utils/fixtures/lmc_with_redis_l2_authsecret.yaml create mode 100644 operator/test/utils/fixtures/vllm_deployment.yaml create mode 100644 operator/test/utils/http.go create mode 100644 operator/test/utils/lmc.go create mode 100644 operator/test/utils/portforward.go create mode 100644 operator/test/utils/runner.go create mode 100644 operator/test/utils/wait.go diff --git a/.buildkite/operator/integration/pipeline.yaml b/.buildkite/operator/integration/pipeline.yaml new file mode 100644 index 00000000000..f8ef73a0b89 --- /dev/null +++ b/.buildkite/operator/integration/pipeline.yaml @@ -0,0 +1,28 @@ +steps: + - label: "Detect operator changes" + key: "operator_detect_changes" + command: | + set -eu + + BASE_REF="origin/dev" + + if ! git fetch --quiet origin dev 2>/dev/null; then + echo "WARN: could not fetch $$BASE_REF; running tests defensively." + buildkite-agent pipeline upload .buildkite/operator/integration/tests.yaml + exit 0 + fi + + if ! CHANGED=$(git diff --name-only "$$BASE_REF...HEAD" -- operator/ .buildkite/operator/ 2>/dev/null); then + echo "WARN: git diff failed; running tests defensively." + buildkite-agent pipeline upload .buildkite/operator/integration/tests.yaml + exit 0 + fi + + if [ -z "$$CHANGED" ]; then + echo "No operator or operator-pipeline changes vs $$BASE_REF; skipping integration tests." + exit 0 + fi + + echo "Operator changes detected:" + echo "$$CHANGED" + buildkite-agent pipeline upload .buildkite/operator/integration/tests.yaml diff --git a/.buildkite/operator/integration/tests.yaml b/.buildkite/operator/integration/tests.yaml new file mode 100644 index 00000000000..c64b8a6c526 --- /dev/null +++ b/.buildkite/operator/integration/tests.yaml @@ -0,0 +1,30 @@ +env: + GPU_KIND_CLUSTER: "operator-test-e2e-gpu-${BUILDKITE_BUILD_NUMBER}" + +steps: + - label: "Operator Integration: test-e2e-gpu-kind" + key: "operator_test_e2e_gpu_kind" + timeout_in_minutes: 90 + command: | + echo $$PWD # for debugging + + cd operator + make test-e2e-gpu-kind + + - label: "Cleanup Kind cluster" + key: "operator_cleanup_kind" + depends_on: "operator_test_e2e_gpu_kind" + allow_dependency_failure: true + timeout_in_minutes: 10 + command: | + cd operator + make cleanup-test-e2e-gpu-kind + + - label: "Cleanup workspace" + depends_on: "operator_cleanup_kind" + allow_dependency_failure: true + command: | + export TARGET="$$PWD" + echo "Deleting current workspace $$TARGET" + cd / + rm -rf "$$TARGET" || true diff --git a/.github/workflows/operator_test_e2e.yml b/.github/workflows/operator_test_e2e.yml deleted file mode 100644 index 9a7eac5939c..00000000000 --- a/.github/workflows/operator_test_e2e.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: "Operator: E2E Tests" - -on: - push: - branches: - - dev - - "release-**" - pull_request: - branches: - - dev - - "release-**" - -permissions: - contents: read - -defaults: - run: - shell: bash - working-directory: operator - -jobs: - changes: - name: Detect changes - runs-on: ubuntu-latest - permissions: - pull-requests: read - outputs: - operator: ${{ steps.filter.outputs.operator == 'true' }} - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - if: github.event_name == 'push' - with: - fetch-depth: 2 - - - uses: dorny/paths-filter@v3 - if: github.event_name == 'push' || github.event_name == 'pull_request' - id: filter - with: - filters: | - operator: - - 'operator/**' - - '.github/workflows/operator_test_e2e.yml' - - test-e2e: - needs: changes - if: needs.changes.outputs.operator == 'true' - name: Run on Ubuntu - runs-on: ubuntu-latest - steps: - - name: Harden Runner - uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 - with: - egress-policy: audit - - - name: Checkout - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version-file: operator/go.mod - - - name: Install the latest version of kind - run: | - curl -Lo ./kind "https://kind.sigs.k8s.io/dl/latest/kind-linux-$(go env GOARCH)" - chmod +x ./kind - sudo mv ./kind /usr/local/bin/kind - - - name: Verify kind installation - run: kind version - - - name: Running Test e2e - run: | - go mod tidy - make test-e2e diff --git a/operator/AGENTS.md b/operator/AGENTS.md index 9b38617775d..bdaae7be395 100644 --- a/operator/AGENTS.md +++ b/operator/AGENTS.md @@ -1,5 +1,277 @@ # operator - AI Agent Guide +## Smoke Test Suite + +The smoke suite under `test/e2e/` validates the operator end-to-end against +a real Kubernetes cluster. It is all-Go, built on Ginkgo/Gomega, and gated +by Go build tags so unit tests run without it. + +### Naming convention + +Targets follow a consistent `-kind` / `-cluster` suffix: + +| Suffix | What it means | +|---|---| +| `-kind` | Self-contained: the target creates a fresh Kind cluster, runs the suite, and deletes the cluster on exit. | +| `-cluster` | Uses whatever cluster the current `KUBECONFIG` / context points at (OpenShift, EKS, k3s, an existing Kind cluster, …). Requires `IMG=` pointing at a registry the cluster can pull from. | + +### Targets (M1, no-GPU) + +```bash +make test-e2e-kind # local Kind, ~5 min +make test-e2e-cluster IMG=/: # existing cluster +``` + +### Targets (M2, GPU) + +```bash +make test-e2e-gpu-kind # local Kind, ~30 min +make test-e2e-gpu-cluster IMG=/: # existing GPU cluster +``` + +Both run the M1 + M2 specs (M2 = runtime `/conf` round-trip + vLLM +integration, under the `e2e_gpu` build tag). Pick `test-e2e-gpu-kind` +when you have GPUs on the dev box and want a self-contained Kind +cluster; pick `test-e2e-gpu-cluster` when you're targeting an +existing OpenShift / EKS / GKE GPU cluster. See *GPU tier* below for +details. + +#### `test-e2e-kind` — local Kind run + +Builds the manager image, loads it into a dedicated Kind cluster +(`operator-test-e2e-` by default), installs CRDs, deploys the controller, +runs every `//go:build e2e` spec under `test/e2e/`, then tears the +cluster down. No prereqs beyond Kind + Docker on `$PATH`. + +#### `test-e2e-cluster` — existing cluster (OpenShift, EKS, k3s, …) + +For when you already have a cluster running and just want the suite to +install/deploy → test → undeploy against it. Prereqs: + +1. **Image pushed to a reachable registry.** The cluster nodes must be + able to `docker pull` `$IMG`. On OpenShift the simplest path is the + internal registry — see *Pushing to OpenShift's internal registry* + below. +2. **`KUBECONFIG` / current-context** points at the target cluster. +3. Pass the in-cluster pull URL via `IMG=`. The target fails fast if + `IMG` is missing or still the default `controller:latest`. + +```bash +oc config use-context admin # or kubectl config use-context +export IMG=image-registry.openshift-image-registry.svc:5000/lmcache-operator-system/operator:v0.0.1 +make test-e2e-cluster IMG=$IMG +``` + +Under the hood this sets `SMOKE_SKIP_IMAGE_LOAD=true` so the suite +neither rebuilds nor `kind load`s. The cluster is left intact after +the run. + +#### Pushing to OpenShift's internal registry + +```bash +# One-time per cluster: expose the registry on a default route +oc patch configs.imageregistry.operator.openshift.io/cluster \ + --type merge -p '{"spec":{"defaultRoute":true}}' + +# Log Docker into the registry using your oc session +oc registry login --to=$HOME/.docker/config.json --skip-check + +# Build, tag, push +oc create namespace lmcache-operator-system --dry-run=client -o yaml | oc apply -f - +export OCP_REGISTRY=$(oc get route default-route -n openshift-image-registry -o jsonpath='{.spec.host}') +make docker-build IMG=controller:latest +docker tag controller:latest $OCP_REGISTRY/lmcache-operator-system/operator:v0.0.1 +docker push $OCP_REGISTRY/lmcache-operator-system/operator:v0.0.1 +``` + +The push uses the external route; pods inside the cluster pull via the +in-cluster service name (`image-registry.openshift-image-registry.svc:5000/...`). +Both names point at the same image; only the hostnames differ. + +#### OpenShift caveats + +- **PodSecurity admission**: test namespaces are pre-labeled + `pod-security.kubernetes.io/enforce=privileged` so the operator's + DaemonSet (which sets `hostIPC=true` + `privileged=true`) is accepted + at admission time. Harmless on clusters that don't enforce PodSecurity. +- **SCC (Security Context Constraints)**: M1 smokes never wait for + DaemonSet pods to schedule, so SCC isn't a blocker. If you need pods + to actually run later (M2/M3 GPU tier), grant the LMCache + ServiceAccount the `privileged` SCC explicitly. + +### Specs included in M1 + +| Spec file | Coverage | +|---|---| +| `crd_smoke_test.go` | harness sanity check + minimal-CR shape + custom-port propagation | +| `lifecycle_smoke_test.go` | port update propagation, delete + ownerRef GC, invalid `sizeGB` rejection | +| `field_coverage_smoke_test.go` | ServiceMonitor (auto-skipped if CRD absent), `extraArgs` override, `resourceOverrides` | +| `auth_smoke_test.go` | cross-namespace `authSecretRef` mirroring + env-var injection | + +### Specs included in M2 (GPU, build tag `e2e_gpu`) + +| Spec file | Coverage | +|---|---| +| `runtime_smoke_test.go` | HTTP `/conf` round-trip — proves CR field values reach the live LMCache server (not just the K8s objects) by asserting `mp.port` / `mp.chunk_size` / `mp.max_workers` / `mp.hash_algorithm` / `http.http_port` against the running pod's `/conf` payload. | +| `vllm_integration_smoke_test.go` | vLLM + LMCache round-trip — spins up a vLLM `Deployment` configured against the operator's `-connection` ConfigMap with `--no-enable-prefix-caching`, sends the same long prompt twice, and asserts `lmcache:num_hit_tokens` on the LMCache `/metrics` endpoint increments on the second call. | + +### Prerequisites + +What you need to install / configure before each target. The "Host +config" column is **only needed once per host** — subsequent runs +reuse it. Detailed rationale for each item lives in the per-target +sections below; this table is the quick-reference. + +| Target | Tools | Host config | Cluster reqs | +|---|---|---|---| +| `test-e2e-kind` | `kind`, `kubectl`, `docker` | — | (cluster created fresh) | +| `test-e2e-cluster` | `kubectl` | — | `KUBECONFIG` → target cluster; pass `IMG=` (registry the cluster can pull from) | +| `test-e2e-gpu-kind` | `kind`, `kubectl`, `docker`, `helm` | NVIDIA driver + `nvidia-container-toolkit` installed, **plus** the two `nvidia-ctk` commands below | (cluster created fresh) | +| `test-e2e-gpu-cluster` | `kubectl`, `helm` | — | `KUBECONFIG` → GPU cluster; GPU node labeled `nvidia.com/gpu.present=true`; `nvidia` `RuntimeClass` installed; pass `IMG=` | + +#### Tool install hints + +```bash +# kind +go install sigs.k8s.io/kind/cmd/kind@latest + +# kubectl — distro-specific; e.g. `curl -LO https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl` +# helm v3 — https://helm.sh/docs/intro/install/ +# docker — distro-specific +``` + +#### GPU host one-time setup (only for `test-e2e-gpu-kind`) + +```bash +# Configure docker to default to the nvidia runtime, then toggle the +# volume-mount-marker mechanism the inline Kind config uses to inject +# GPUs into the worker container. Restart docker once at the end. +sudo nvidia-ctk runtime configure --runtime=docker --set-as-default --cdi.enabled +sudo nvidia-ctk config --set accept-nvidia-visible-devices-as-volume-mounts=true --in-place +sudo systemctl restart docker +``` + +The target fails fast with a copy-pasteable fix command if either +piece of host config is missing. + +**Not needed**: `nvkind`. An earlier iteration used it; the current +target uses the NVIDIA GPU Operator instead and gets GPU passthrough +via an inline Kind config. See `make/e2e-gpu.mk` for the rationale. + +### Helper library (`test/utils/`) + +| File | Purpose | +|---|---| +| `lmc.go` | `ApplyLMC`, `WaitLMCReconciled`, `WaitLMCPhase`, `WaitLMCReady`, `GetConnectionConfig` (typed parser), `PatchLMCSpec`, `DeleteLMCAndWaitGC` | +| `portforward.go` | `PortForward(spec, ports...)` — wraps `kubectl port-forward`, waits for the local port to accept TCP | +| `fixtures.go` | `go:embed`-backed fixture/golden loader | +| `runner.go` | `RunMake` / `RunFromOperator` — runs commands from the operator/ root **without** `os.Chdir` | +| `utils.go` | Legacy exec helpers retained for the kubebuilder-template Manager spec | + +### Adding a new smoke spec + +1. Drop the YAML under `test/utils/fixtures/.yaml`. +2. Use `utils.NewLMCFromFixture(...)` to load and override `name`/`namespace`. +3. Call the helpers above; do **not** call `os.Chdir` or read paths + relative to the working directory — fixtures resolve via `go:embed`, + and shell commands accept their working directory through `cmd.Dir`. +4. Wrap the spec body with `recordOnFailure(nsName)` in `AfterEach` so + failures dump controller logs, events, pod descriptions, and the CR yaml. + +### GPU tier (M2) + +Two entry points share the same `e2e_gpu`-tagged specs: + +#### `test-e2e-gpu-kind` — self-contained Kind cluster + +Best when you have GPUs on the dev box and don't want to wire up an +external cluster. The target hand-rolls a Kind cluster config that +mounts `/dev/null` at `/var/run/nvidia-container-devices/all` in the +worker — combined with the host setup below, +nvidia-container-runtime sees that marker and injects all GPU +devices + driver libraries into the Kind worker container. Then the +target installs the +[NVIDIA GPU Operator](https://github.com/NVIDIA/gpu-operator), which +runs a toolkit daemonset that installs nvidia-container-toolkit +*inside the Kind node* and registers a `nvidia` containerd runtime +handler. After that, pods with `runtimeClassName: nvidia` (which the +LMCache DaemonSet and the test-side vLLM Deployment already use) +get NVML / libcuda injected. The cluster is auto-deleted on exit — +same trap pattern as `test-e2e-kind`. All in-cluster manifests +(including the Kind config) are inlined into the Makefile. + +We explored several alternatives before settling on GPU Operator: +the bare device plugin alone fails with `Failed to initialize NVML: +ERROR_LIBRARY_NOT_FOUND` because pods scheduled by Kind's inner +containerd don't inherit the worker's library mounts; manually +apt-installing the toolkit via `docker exec` works but breaks if +the kindest/node image changes; nvkind didn't reliably produce +GPU-passthrough markers on the target host. GPU Operator does the +toolkit install as a Kubernetes-native DaemonSet, which is the most +robust path. Trade-off: helm install takes ~10 min (operator + +ClusterPolicy + 5 daemonsets), versus ~30 s for the bare plugin +when it works. + +Host one-time setup (NOT automated): + +```bash +# 1. NVIDIA driver + nvidia-container-toolkit installed (distro-specific). + +# 2. Configure docker + nvidia-container-runtime: +sudo nvidia-ctk runtime configure --runtime=docker --set-as-default --cdi.enabled +sudo nvidia-ctk config --set accept-nvidia-visible-devices-as-volume-mounts=true --in-place +sudo systemctl restart docker + +# 3. helm + kubectl + kind on PATH. +``` + +Then: + +```bash +make test-e2e-gpu-kind +``` + +The Makefile target fails fast with a copy-pasteable fix command if +either `Default Runtime: nvidia` is missing from `docker info` or +`accept-nvidia-visible-devices-as-volume-mounts=true` is missing +from `/etc/nvidia-container-runtime/config.toml`. + +Single-node is intentional: the LMCache DaemonSet and the test-side +vLLM Deployment both schedule onto the same (only) worker, which is +what the kv-cache transfer needs anyway (hostIPC + cudaIPC require +colocation). + +**Side effect of step 2 to be aware of**: after the flip, every +docker container on the host — not just Kind workers — starts +through nvidia-container-runtime. Non-GPU workloads still work but +go through one extra hook on startup. + +The `nvidia` RuntimeClass is registered by nvkind. Pods that need +GPU access (the LMCache DaemonSet, the test-side vLLM Deployment) +already reference `runtimeClassName: nvidia`. + +#### `test-e2e-gpu-cluster` — existing GPU cluster + +Use when targeting OpenShift / EKS / GKE GPU clusters. Prerequisites: + +1. At least one node has `nvidia.com/gpu.present=true` and the + `nvidia` RuntimeClass installed. +2. `KUBECONFIG` points at that cluster, the operator image is pushed + to a registry the cluster can pull from, and `IMG=` references it. +3. The cluster can pull `lmcache/vllm-openai:latest` (default for + both the LMCache DaemonSet and the test-side vLLM workload). + Override with `VLLM_IMAGE=` if you mirror it elsewhere. +4. Internet egress for Hugging Face model downloads, OR the model is + already on the node. Default model is `Qwen/Qwen2.5-0.5B`; override + with `VLLM_MODEL=/`. +5. To run only the `/conf` spec and skip the heavyweight vLLM + round-trip, set `SKIP_VLLM_INTEGRATION=true`. + +Timeout is 60 min — cold image pulls + model download routinely eat +20+ min before the first inference. + +--- + ## Project Structure **Single-group layout (default):** @@ -12,7 +284,8 @@ internal/webhook/* Validation/defaulting (if present) config/crd/bases/* Generated CRDs (DO NOT EDIT) config/rbac/role.yaml Generated RBAC (DO NOT EDIT) config/samples/* Example CRs (edit these) -Makefile Build/test/deploy commands +Makefile Top-level orchestrator: vars + `include make/*.mk` +make/*.mk Targets by concern (dev / e2e / e2e-gpu / lint / build / deploy / tools) PROJECT Kubebuilder metadata Auto-generated (DO NOT EDIT) ``` diff --git a/operator/DESIGN.md b/operator/DESIGN.md index 863c9a34590..76cbf63ad2c 100644 --- a/operator/DESIGN.md +++ b/operator/DESIGN.md @@ -340,7 +340,19 @@ OnEvent(LMCacheEngine create/update/delete): **Secondary watches:** DaemonSet, Pods (readiness changes → update endpoints), Nodes (new GPU node → DaemonSet auto-schedules) -**Finalizer** `lmcache.ai/cleanup`: on CR deletion, cascading delete of owned resources. +**Deletion / cleanup**: every child resource the operator creates +(DaemonSet, lookup Service, metrics Service, connection ConfigMap, +managed RESP auth Secret, optional ServiceMonitor) carries an +`ownerReference` to the LMCacheEngine, so Kubernetes garbage +collection cascade-deletes them when the CR goes away. **No finalizer +is used.** An earlier design added a `lmcache.ai/cleanup` finalizer +to mirror that GC behavior, but it was a no-op that only created +deadlocks when the controller pod was not running (e.g. during +cluster issues or a single-step `kubectl delete -k config/default`). +The reconciler now actively strips that legacy finalizer from any CR +it sees, so migration from older operator versions is automatic. +Finalizers will return when we need to clean up state K8s GC cannot +reach (Redis L2 keys, federation deregistration, etc.). --- diff --git a/operator/Makefile b/operator/Makefile index 98ead3eaf26..915fb7e50dc 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -1,3 +1,8 @@ +# Top-level Makefile orchestrator. Targets and their recipes live in +# make/*.mk by concern (dev / e2e / e2e-gpu / lint / build / deploy / +# tools). Each sub-file carries its own `##@ Category` header so the +# `make help` awk command groups them correctly. + # Image URL to use all building/pushing image targets IMG ?= controller:latest @@ -25,240 +30,25 @@ all: build ##@ General # The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk command is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - +# beneath their categories. Categories come from `##@ ` lines, targets +# from `: ... ## `. $(MAKEFILE_LIST) expands to this +# Makefile + every included sub-file, so all sub-files are scanned. .PHONY: help help: ## Display this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) -##@ Development - -.PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - "$(CONTROLLER_GEN)" rbac:roleName=manager-role crd:allowDangerousTypes=true webhook paths="./..." output:crd:artifacts:config=config/crd/bases - -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - "$(CONTROLLER_GEN)" object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: fmt -fmt: ## Run go fmt against code. - go fmt ./... - -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... - -.PHONY: test -test: manifests generate fmt vet setup-envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out - -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# CertManager is installed by default; skip with: -# - CERT_MANAGER_INSTALL_SKIP=true -KIND_CLUSTER ?= operator-test-e2e - -.PHONY: setup-test-e2e -setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist - @command -v $(KIND) >/dev/null 2>&1 || { \ - echo "Kind is not installed. Please install Kind manually."; \ - exit 1; \ - } - @case "$$($(KIND) get clusters)" in \ - *"$(KIND_CLUSTER)"*) \ - echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ - *) \ - echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ - $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ - esac - -.PHONY: test-e2e -test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. - KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v - $(MAKE) cleanup-test-e2e - -.PHONY: cleanup-test-e2e -cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests - @$(KIND) delete cluster --name $(KIND_CLUSTER) - -.PHONY: install-hooks -install-hooks: golangci-lint ## Install git pre-commit hooks (requires pre-commit: pip install pre-commit). - @command -v pre-commit >/dev/null 2>&1 || { \ - echo "pre-commit not found. Install with: pip install pre-commit"; \ - exit 1; \ - } - cd "$(shell git rev-parse --show-toplevel)" && pre-commit install -c operator/.pre-commit-config.yaml - @echo "Pre-commit hooks installed (operator)." - -.PHONY: lint -lint: golangci-lint ## Run golangci-lint linter - "$(GOLANGCI_LINT)" run - -.PHONY: lint-fix -lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes - "$(GOLANGCI_LINT)" run --fix - -.PHONY: lint-config -lint-config: golangci-lint ## Verify golangci-lint linter configuration - "$(GOLANGCI_LINT)" config verify - -##@ Build - -.PHONY: build -build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go - -.PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. - go run ./cmd/main.go - -# If you wish to build the manager image targeting other platforms you can use the --platform flag. -# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. -# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -.PHONY: docker-build -docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} - -# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ -# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) -# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/amd64 -.PHONY: docker-buildx -docker-buildx: ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name operator-builder - $(CONTAINER_TOOL) buildx use operator-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm operator-builder - rm Dockerfile.cross - -.PHONY: build-installer -build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. - mkdir -p dist - cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} - "$(KUSTOMIZE)" build config/default > dist/install.yaml - -##@ Deployment - -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - @out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \ - if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" apply -f -; else echo "No CRDs to install; skipping."; fi - -.PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - @out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \ - if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi - -.PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} - "$(KUSTOMIZE)" build config/default | "$(KUBECTL)" apply -f - - -.PHONY: undeploy -undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - "$(KUSTOMIZE)" build config/default | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f - - -##@ Dependencies - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p "$(LOCALBIN)" - -## Tool Binaries -KUBECTL ?= kubectl -KIND ?= kind -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest -GOLANGCI_LINT = $(LOCALBIN)/golangci-lint - -## Tool Versions -KUSTOMIZE_VERSION ?= v5.8.1 -CONTROLLER_TOOLS_VERSION ?= v0.20.1 - -#ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) -ENVTEST_VERSION ?= $(shell v='$(call gomodver,sigs.k8s.io/controller-runtime)'; \ - [ -n "$$v" ] || { echo "Set ENVTEST_VERSION manually (controller-runtime replace has no tag)" >&2; exit 1; }; \ - printf '%s\n' "$$v" | sed -E 's/^v?([0-9]+)\.([0-9]+).*/release-\1.\2/') - -#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) -ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \ - [ -n "$$v" ] || { echo "Set ENVTEST_K8S_VERSION manually (k8s.io/api replace has no tag)" >&2; exit 1; }; \ - printf '%s\n' "$$v" | sed -E 's/^v?[0-9]+\.([0-9]+).*/1.\1/') - -GOLANGCI_LINT_VERSION ?= v2.8.0 -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. -$(KUSTOMIZE): $(LOCALBIN) - $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) - -.PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. -$(CONTROLLER_GEN): $(LOCALBIN) - $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) - -.PHONY: setup-envtest -setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. - @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." - @"$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path || { \ - echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ - exit 1; \ - } - -.PHONY: envtest -envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. -$(ENVTEST): $(LOCALBIN) - $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) - -.PHONY: golangci-lint -golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. -$(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) - @test -f .custom-gcl.yml && { \ - echo "Building custom golangci-lint with plugins..." && \ - $(GOLANGCI_LINT) custom --destination $(LOCALBIN) --name golangci-lint-custom && \ - mv -f $(LOCALBIN)/golangci-lint-custom $(GOLANGCI_LINT); \ - } || true - -# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist -# $1 - target path with name of binary -# $2 - package url which can be installed -# $3 - specific version of package -define go-install-tool -@[ -f "$(1)-$(3)" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)" ] || { \ -set -e; \ -package=$(2)@$(3) ;\ -echo "Downloading $${package}" ;\ -rm -f "$(1)" ;\ -GOBIN="$(LOCALBIN)" go install $${package} ;\ -mv "$(LOCALBIN)/$$(basename "$(1)")" "$(1)-$(3)" ;\ -} ;\ -ln -sf "$$(realpath "$(1)-$(3)")" "$(1)" -endef - -define gomodver -$(shell go list -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' $(1) 2>/dev/null) -endef +# Order matters only when one file uses a macro `define`d in another: +# - tools.mk defines `gomodver` + `go-install-tool` (used by every +# tool-fetcher target it ships). +# - dev/e2e/e2e-gpu/lint reference $(KIND), $(KUBECTL), etc. defined +# in tools.mk — but those are simple variable references and Make +# resolves them lazily, so include order doesn't matter for that. +# Listing tools.mk first is just stylistic so readers see the +# variables before the targets that consume them. +include make/tools.mk +include make/dev.mk +include make/build.mk +include make/deploy.mk +include make/lint.mk +include make/e2e.mk +include make/e2e-gpu.mk diff --git a/operator/README.md b/operator/README.md index dd5b2801f35..9e3d92f74f7 100644 --- a/operator/README.md +++ b/operator/README.md @@ -342,10 +342,101 @@ make manifests # Generate CRD YAML + RBAC make build # Compile operator binary make fmt # go fmt make vet # go vet -make test # Run unit tests +make test # Run unit tests (envtest, CPU-only) make lint # Run golangci-lint ``` +### End-to-End Tests + +Four `make` targets cover the e2e tiers. The `-kind` variants create +a throwaway Kind cluster and tear it down on exit; the `-cluster` +variants run against whatever your current `KUBECONFIG` points at. +M2 (GPU) targets additionally run a runtime HTTP check against the +LMCache server and a vLLM round-trip that proves the KV cache +stores on the first request and retrieves on the second. + +```bash +make test-e2e-kind # local Kind, no GPU, ~5 min +make test-e2e-cluster IMG= # existing cluster, no GPU +make test-e2e-gpu-kind # local Kind, GPU, ~30 min +make test-e2e-gpu-cluster IMG= # existing GPU cluster +``` + +#### Tool prerequisites + +| Target | Tools to install | +|---|---| +| `test-e2e-kind` | `kind`, `kubectl`, `docker` | +| `test-e2e-cluster` | `kubectl` (cluster access via `KUBECONFIG`) | +| `test-e2e-gpu-kind` | `kind`, `kubectl`, `docker`, `helm` (v3) | +| `test-e2e-gpu-cluster` | `kubectl`, `helm` (cluster access via `KUBECONFIG`) | + +```bash +# Install kind (the other tools are distro-specific) +go install sigs.k8s.io/kind/cmd/kind@latest +``` + +#### One-time host setup for `test-e2e-gpu-kind` + +Beyond the tools above, your host needs the NVIDIA driver and +`nvidia-container-toolkit` installed (distro-specific), plus two +`nvidia-ctk` commands that flip docker's default runtime to `nvidia` +and toggle the volume-mount-based GPU injection mechanism the +target's inline Kind config relies on: + +```bash +sudo nvidia-ctk runtime configure --runtime=docker --set-as-default --cdi.enabled +sudo nvidia-ctk config --set accept-nvidia-visible-devices-as-volume-mounts=true --in-place +sudo systemctl restart docker +``` + +The target fails fast with a copy-pasteable fix command if either +piece of host config is missing. Note that flipping docker's default +runtime is a **host-wide** change — every container on the host then +starts through `nvidia-container-runtime`. Non-GPU workloads still +work but go through one extra hook. + +The target installs the [NVIDIA GPU Operator](https://github.com/NVIDIA/gpu-operator) +inside the Kind cluster to handle the toolkit / containerd reconfig +that lets pods scheduled by Kind's inner containerd see the driver +libraries. That install takes 5-10 min, which is the bulk of the +`test-e2e-gpu-kind` runtime. (`nvkind` is **not** required — we +tried it and the current target uses a hand-rolled Kind config +instead.) + +#### Cluster prerequisites for `test-e2e-gpu-cluster` + +The existing cluster must have: + +- at least one node labeled `nvidia.com/gpu.present=true` +- the `nvidia` `RuntimeClass` installed (GPU Operator or equivalent) +- the operator image pushed to a registry the cluster's image puller can reach (pass as `IMG=…`) + +#### Knobs (env vars) + +| Variable | Default | Used by | +|---|---|---| +| `KIND_CLUSTER` | `operator-test-e2e-` | `test-e2e-kind` — set to target an existing Kind cluster | +| `GPU_KIND_CLUSTER` | `operator-test-e2e-gpu-` | `test-e2e-gpu-kind` — same idea | +| `KEEP_CLUSTER_ON_FAILURE` | unset | `test-e2e-gpu-kind` — set to `1` to keep the cluster alive after a failure for live debugging | +| `VLLM_MODEL` | `Qwen/Qwen2.5-0.5B` | vLLM integration spec — Hugging Face model id | +| `VLLM_IMAGE` | `lmcache/vllm-openai:latest` | vLLM integration spec | +| `SKIP_VLLM_INTEGRATION` | unset | set to `true` to skip the heavyweight vLLM spec but still run the runtime HTTP check | + +If a GPU run fails during setup, use the diagnostic target: + +```bash +make diagnose-test-e2e-gpu-kind GPU_KIND_CLUSTER= +``` + +It dumps `ClusterPolicy` status, GPU-Operator pod state, events, +toolkit/device-plugin daemonset logs, `/dev/nvidia*` inside the Kind +worker, and the `nvidia` stanza of the worker's containerd config. + +For deeper design notes (why GPU Operator over the bare device plugin, +how the volume-mount marker propagates GPUs into the Kind worker, the +test specs themselves), see [`AGENTS.md`](./AGENTS.md). + ### Pushing a Custom Operator Image ```bash diff --git a/operator/api/v1alpha1/lmcacheengine_types.go b/operator/api/v1alpha1/lmcacheengine_types.go index 04f66c2bf63..fa4d25cd9b1 100644 --- a/operator/api/v1alpha1/lmcacheengine_types.go +++ b/operator/api/v1alpha1/lmcacheengine_types.go @@ -97,6 +97,11 @@ type ServerSpec struct { // L1BackendSpec defines the L1 memory cache configuration. type L1BackendSpec struct { // sizeGB is the L1 cache size in gigabytes. Required, must be > 0. + // The CRD-level constraint (exclusiveMinimum=0) rejects invalid values + // at admission time so the controller never sees them; the in-Go + // ValidateSpec keeps the same rule for defense in depth. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:ExclusiveMinimum=true SizeGB float64 `json:"sizeGB"` } diff --git a/operator/config/crd/bases/lmcache.lmcache.ai_lmcacheengines.yaml b/operator/config/crd/bases/lmcache.lmcache.ai_lmcacheengines.yaml index 80ed6143ad9..934eb0fd0ca 100644 --- a/operator/config/crd/bases/lmcache.lmcache.ai_lmcacheengines.yaml +++ b/operator/config/crd/bases/lmcache.lmcache.ai_lmcacheengines.yaml @@ -1205,8 +1205,13 @@ spec: description: l1 defines the L1 memory cache configuration. properties: sizeGB: - description: sizeGB is the L1 cache size in gigabytes. Required, - must be > 0. + description: |- + sizeGB is the L1 cache size in gigabytes. Required, must be > 0. + The CRD-level constraint (exclusiveMinimum=0) rejects invalid values + at admission time so the controller never sees them; the in-Go + ValidateSpec keeps the same rule for defense in depth. + exclusiveMinimum: true + minimum: 0 type: number required: - sizeGB diff --git a/operator/go.mod b/operator/go.mod index 002dd9969a4..d2683b24c81 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -11,6 +11,7 @@ require ( k8s.io/apimachinery v0.35.0 k8s.io/client-go v0.35.0 sigs.k8s.io/controller-runtime v0.23.1 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -97,5 +98,4 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/operator/internal/controller/reconcile_helpers.go b/operator/internal/controller/reconcile_helpers.go index 12bbf7add65..6da6b563228 100644 --- a/operator/internal/controller/reconcile_helpers.go +++ b/operator/internal/controller/reconcile_helpers.go @@ -36,13 +36,30 @@ import ( "github.com/LMCache/LMCache/internal/resources" ) -// handleFinalizer adds or processes the finalizer. Returns (err, done). -// If done is true, the caller should return immediately. +// handleFinalizer migrates any LMCacheEngine carrying the legacy +// "lmcache.ai/cleanup" finalizer off of it. Owner references on every +// child resource (DaemonSet, Services, ConfigMap, managed Secret) +// already cause K8s garbage collection to cascade-delete them when +// the CR goes away, so no finalizer work is necessary today. +// +// Why this changed: a no-op finalizer blocks CR deletion whenever the +// controller pod isn't running (e.g. during cluster issues or a +// `kubectl delete -k config/default` that takes down the operator +// alongside the CRDs), with no upside since GC already does the work. +// See discussion in https://github.com/LMCache/LMCache/issues/2693. +// +// We may re-introduce a finalizer in the future for state K8s GC +// can't reach — e.g. evicting L2 Redis keys on CR delete, federation +// deregistration. When that happens, this function will grow real +// cleanup work alongside the legacy migration. +// +// Returns (err, done). If done is true the caller must return. func (r *LMCacheEngineReconciler) handleFinalizer(ctx context.Context, engine *lmcachev1alpha1.LMCacheEngine) (error, bool) { + // On the deletion path we still need to clear the legacy finalizer + // if a CR created by an older operator version is being deleted, + // otherwise K8s would block on it forever. if engine.DeletionTimestamp != nil { if controllerutil.ContainsFinalizer(engine, finalizerName) { - // Owned resources are cleaned up by K8s garbage collection via ownerRefs. - // Remove the finalizer to allow deletion. controllerutil.RemoveFinalizer(engine, finalizerName) if err := r.Update(ctx, engine); err != nil { return err, true @@ -51,15 +68,17 @@ func (r *LMCacheEngineReconciler) handleFinalizer(ctx context.Context, engine *l return nil, true } - // Add finalizer if not present. - if !controllerutil.ContainsFinalizer(engine, finalizerName) { - controllerutil.AddFinalizer(engine, finalizerName) + // Pro-active migration on the create/update path: strip the + // legacy finalizer from any CR that still has it from a prior + // operator version, so future deletes don't deadlock. + if controllerutil.ContainsFinalizer(engine, finalizerName) { + controllerutil.RemoveFinalizer(engine, finalizerName) if err := r.Update(ctx, engine); err != nil { return err, true } - // Return done=true so the controller requeues with a fresh Get. - // Continuing here would risk a resourceVersion conflict because - // the Update changed the object on the server. + // Return done=true so the controller requeues with a fresh + // Get; continuing here would race with the resourceVersion + // bump the Update just produced on the server. return nil, true } diff --git a/operator/make/build.mk b/operator/make/build.mk new file mode 100644 index 00000000000..afdbff11c06 --- /dev/null +++ b/operator/make/build.mk @@ -0,0 +1,43 @@ +##@ Build + +.PHONY: build +build: manifests generate fmt vet ## Build manager binary. + go build -o bin/manager cmd/main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./cmd/main.go + +# If you wish to build the manager image targeting other platforms you can use the --platform flag. +# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. +# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +.PHONY: docker-build +docker-build: ## Build docker image with the manager. + $(CONTAINER_TOOL) build -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + $(CONTAINER_TOOL) push ${IMG} + +# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: +# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ +# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) +# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. +PLATFORMS ?= linux/amd64 +.PHONY: docker-buildx +docker-buildx: ## Build and push docker image for the manager for cross-platform support + # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + - $(CONTAINER_TOOL) buildx create --name operator-builder + $(CONTAINER_TOOL) buildx use operator-builder + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx rm operator-builder + rm Dockerfile.cross + +.PHONY: build-installer +build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. + mkdir -p dist + cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} + "$(KUSTOMIZE)" build config/default > dist/install.yaml diff --git a/operator/make/deploy.mk b/operator/make/deploy.mk new file mode 100644 index 00000000000..7d337b91599 --- /dev/null +++ b/operator/make/deploy.mk @@ -0,0 +1,24 @@ +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + @out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \ + if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" apply -f -; else echo "No CRDs to install; skipping."; fi + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + @out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \ + if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} + "$(KUSTOMIZE)" build config/default | "$(KUBECTL)" apply -f - + +.PHONY: undeploy +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + "$(KUSTOMIZE)" build config/default | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f - diff --git a/operator/make/dev.mk b/operator/make/dev.mk new file mode 100644 index 00000000000..0a835094b31 --- /dev/null +++ b/operator/make/dev.mk @@ -0,0 +1,21 @@ +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + "$(CONTROLLER_GEN)" rbac:roleName=manager-role crd:allowDangerousTypes=true webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + "$(CONTROLLER_GEN)" object:headerFile="hack/boilerplate.go.txt" paths="./..." + +.PHONY: fmt +fmt: ## Run gofmt against code (covers build-tagged files that `go fmt ./...` skips). + gofmt -w . + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: manifests generate fmt vet setup-envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out diff --git a/operator/make/e2e-gpu.mk b/operator/make/e2e-gpu.mk new file mode 100644 index 00000000000..afeea2e6606 --- /dev/null +++ b/operator/make/e2e-gpu.mk @@ -0,0 +1,175 @@ +##@ E2E (GPU) + +# Creates a 2-node Kind cluster (control-plane + GPU worker) and +# installs the NVIDIA GPU Operator. The Operator handles toolkit +# install + containerd reconfig inside the worker so pods with +# `runtimeClassName: nvidia` get NVML / libcuda injected. +# +# Full design notes (why GPU Operator vs bare device plugin, host +# prereqs, single-node intent) live in operator/AGENTS.md under the +# "GPU tier (M2)" section. Keep this header lean. +ifndef GPU_KIND_CLUSTER +GPU_KIND_CLUSTER := operator-test-e2e-gpu-$(shell date +%s%N | tail -c 9) +endif +HELM ?= helm + +# GPU Operator helm flags worth a note (the rest are defaults): +# driver.enabled=false — host docker's nvidia runtime already +# injects /dev/nvidia* + driver libs into +# the Kind worker; we don't want a kernel +# driver installed inside the cluster. +# toolkit.enabled=true — toolkit daemonset writes containerd's +# config.toml + registers a `nvidia` +# runtime handler. Pods with +# runtimeClassName=nvidia then get NVML. +# cdi.enabled=true — modern CDI annotation path; needs +# containerd 1.7+ (kindest/node has it). +# CONTAINERD_CONFIG / — Kind's containerd config + socket are at +# CONTAINERD_SOCKET non-default paths; without overriding, +# the toolkit daemonset edits a phantom +# config and ClusterPolicy never goes Ready. +# validator.driver.env — DISABLE_DEV_CHAR_SYMLINK_CREATION; Kind +# nodes can't mknod /dev/char/* and the +# driver validator pod would loop without +# this. Documented gpu-operator-on-kind +# workaround. +.PHONY: setup-test-e2e-gpu-kind +setup-test-e2e-gpu-kind: ## Create a Kind GPU cluster (inline config) + install the NVIDIA GPU Operator. + @command -v $(KIND) >/dev/null 2>&1 || { echo "kind not found. Install Kind."; exit 1; } + @command -v $(HELM) >/dev/null 2>&1 || { echo "helm not found. Install helm v3."; exit 1; } + @command -v $(KUBECTL) >/dev/null 2>&1 || { echo "kubectl not found."; exit 1; } + @docker info 2>/dev/null | grep -q "Default Runtime: nvidia" || { \ + echo "ERROR: Docker default-runtime is not 'nvidia'."; \ + echo " Fix: sudo nvidia-ctk runtime configure --runtime=docker --set-as-default --cdi.enabled && sudo systemctl restart docker"; \ + exit 1; \ + } + @grep -E '^\s*accept-nvidia-visible-devices-as-volume-mounts\s*=\s*true' /etc/nvidia-container-runtime/config.toml >/dev/null 2>&1 || { \ + echo "ERROR: 'accept-nvidia-visible-devices-as-volume-mounts' is not true in /etc/nvidia-container-runtime/config.toml."; \ + echo " Fix: sudo nvidia-ctk config --set accept-nvidia-visible-devices-as-volume-mounts=true --in-place && sudo systemctl restart docker"; \ + echo " Without this, the volume-mount marker won't propagate into the Kind worker."; \ + exit 1; \ + } + @echo "==> Creating Kind GPU cluster '$(GPU_KIND_CLUSTER)' (1 control-plane + 1 worker, all GPUs)..." + @printf '%s\n' \ + 'kind: Cluster' \ + 'apiVersion: kind.x-k8s.io/v1alpha4' \ + 'name: $(GPU_KIND_CLUSTER)' \ + 'nodes:' \ + '- role: control-plane' \ + '- role: worker' \ + ' labels:' \ + ' nvidia.com/gpu.present: "true"' \ + ' extraMounts:' \ + ' - hostPath: /dev/null' \ + ' containerPath: /var/run/nvidia-container-devices/all' \ + | $(KIND) create cluster --config=- + @echo "==> Installing the NVIDIA GPU Operator (this takes 5-10 minutes)..." + $(HELM) repo add nvidia https://helm.ngc.nvidia.com/nvidia >/dev/null 2>&1 || true + $(HELM) repo update nvidia >/dev/null + @$(HELM) upgrade -i gpu-operator nvidia/gpu-operator \ + --kube-context=kind-$(GPU_KIND_CLUSTER) \ + --namespace gpu-operator --create-namespace \ + --set driver.enabled=false \ + --set toolkit.enabled=true \ + --set cdi.enabled=true \ + --set toolkit.env[0].name=CONTAINERD_CONFIG \ + --set-string toolkit.env[0].value=/etc/containerd/config.toml \ + --set toolkit.env[1].name=CONTAINERD_SOCKET \ + --set-string toolkit.env[1].value=/run/containerd/containerd.sock \ + --set toolkit.env[2].name=CONTAINERD_RUNTIME_CLASS \ + --set-string toolkit.env[2].value=nvidia \ + --set toolkit.env[3].name=CONTAINERD_SET_AS_DEFAULT \ + --set-string toolkit.env[3].value=true \ + --set validator.driver.env[0].name=DISABLE_DEV_CHAR_SYMLINK_CREATION \ + --set-string validator.driver.env[0].value=true \ + --wait --timeout=15m \ + || { $(MAKE) --no-print-directory diagnose-test-e2e-gpu-kind GPU_KIND_CLUSTER=$(GPU_KIND_CLUSTER); exit 1; } + @echo "==> Waiting for the node to advertise nvidia.com/gpu (up to 5 min)..." + @gpus=""; \ + for i in $$(seq 1 150); do \ + gpus=$$($(KUBECTL) --context=kind-$(GPU_KIND_CLUSTER) get nodes \ + -o jsonpath='{.items[?(@.metadata.labels.nvidia\.com/gpu\.present=="true")].status.allocatable.nvidia\.com/gpu}' 2>/dev/null || true); \ + case "$$gpus" in [1-9]*) echo "==> Node advertises $$gpus GPU(s)"; exit 0 ;; esac; \ + sleep 2; \ + done; \ + echo ""; \ + echo "ERROR: node never advertised nvidia.com/gpu after 5 min."; \ + $(MAKE) --no-print-directory diagnose-test-e2e-gpu-kind GPU_KIND_CLUSTER=$(GPU_KIND_CLUSTER); \ + exit 1 + +.PHONY: diagnose-test-e2e-gpu-kind +diagnose-test-e2e-gpu-kind: ## Dump gpu-operator state for the named cluster (for setup failures). + @echo "" + @echo "==========================================================================" + @echo "GPU Operator diagnostic dump for cluster: $(GPU_KIND_CLUSTER)" + @echo "==========================================================================" + @echo "" + @echo "--- kubectl get nodes (allocatable / capacity) ---" + @$(KUBECTL) --context=kind-$(GPU_KIND_CLUSTER) get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n allocatable: "}{.status.allocatable}{"\n capacity: "}{.status.capacity}{"\n"}{end}' || true + @echo "" + @echo "--- ClusterPolicy status ---" + @$(KUBECTL) --context=kind-$(GPU_KIND_CLUSTER) get clusterpolicy -o jsonpath='{.items[0].status}' 2>/dev/null | head -c 2000 || echo "(no ClusterPolicy yet)" + @echo "" + @echo "" + @echo "--- gpu-operator pods ---" + @$(KUBECTL) --context=kind-$(GPU_KIND_CLUSTER) get pods -n gpu-operator -o wide || true + @echo "" + @echo "--- gpu-operator events (last 40) ---" + @$(KUBECTL) --context=kind-$(GPU_KIND_CLUSTER) get events -n gpu-operator --sort-by=.lastTimestamp 2>/dev/null | tail -40 || true + @echo "" + @echo "--- toolkit daemonset logs ---" + @$(KUBECTL) --context=kind-$(GPU_KIND_CLUSTER) logs -n gpu-operator -l app=nvidia-container-toolkit-daemonset --tail=60 --all-containers 2>&1 | head -120 || true + @echo "" + @echo "--- device-plugin daemonset logs ---" + @$(KUBECTL) --context=kind-$(GPU_KIND_CLUSTER) logs -n gpu-operator -l app=nvidia-device-plugin-daemonset --tail=60 --all-containers 2>&1 | head -120 || true + @echo "" + @echo "--- driver-validator pod logs (if running) ---" + @$(KUBECTL) --context=kind-$(GPU_KIND_CLUSTER) logs -n gpu-operator -l app=nvidia-operator-validator --tail=60 --all-containers 2>&1 | head -120 || true + @echo "" + @echo "--- /dev/nvidia* inside the Kind worker container ---" + @(docker exec $(GPU_KIND_CLUSTER)-worker sh -c 'ls /dev/nvidia* 2>&1' || \ + docker exec $(GPU_KIND_CLUSTER)-control-plane sh -c 'ls /dev/nvidia* 2>&1' || \ + echo "(no nvidia devices visible)") | head -30 || true + @echo "" + @echo "--- /etc/containerd/config.toml nvidia stanza (inside Kind worker) ---" + @docker exec $(GPU_KIND_CLUSTER)-worker sh -c 'grep -A8 "nvidia" /etc/containerd/config.toml 2>&1 || echo "(no nvidia stanza)"' || true + @echo "==========================================================================" + +.PHONY: test-e2e-gpu-kind +test-e2e-gpu-kind: manifests generate fmt vet ## Self-contained GPU smoke on a Kind cluster (build + load + run + cleanup). + @echo "==> Using Kind GPU cluster: $(GPU_KIND_CLUSTER) (auto-deleted after the run unless KEEP_CLUSTER_ON_FAILURE=1)" + @trap 'rc=$$?; \ + if [ "$$rc" -ne 0 ] && [ "$(KEEP_CLUSTER_ON_FAILURE)" = "1" ]; then \ + echo ""; \ + echo "==> KEEP_CLUSTER_ON_FAILURE=1 set; leaving cluster '$(GPU_KIND_CLUSTER)' alive."; \ + echo " To diagnose: make diagnose-test-e2e-gpu-kind GPU_KIND_CLUSTER=$(GPU_KIND_CLUSTER)"; \ + echo " To delete: make cleanup-test-e2e-gpu-kind GPU_KIND_CLUSTER=$(GPU_KIND_CLUSTER)"; \ + else \ + $(MAKE) --no-print-directory cleanup-test-e2e-gpu-kind GPU_KIND_CLUSTER=$(GPU_KIND_CLUSTER); \ + fi; \ + exit $$rc' EXIT INT TERM; \ + $(MAKE) --no-print-directory setup-test-e2e-gpu-kind GPU_KIND_CLUSTER=$(GPU_KIND_CLUSTER) && \ + KIND=$(KIND) KIND_CLUSTER=$(GPU_KIND_CLUSTER) \ + go test -tags=e2e,e2e_gpu ./test/e2e/ -v -ginkgo.v -timeout 60m + +.PHONY: cleanup-test-e2e-gpu-kind +cleanup-test-e2e-gpu-kind: ## Delete the Kind GPU cluster (idempotent). + @if $(KIND) get clusters 2>/dev/null | grep -qx "$(GPU_KIND_CLUSTER)"; then \ + echo "Deleting Kind GPU cluster '$(GPU_KIND_CLUSTER)'..."; \ + $(KIND) delete cluster --name $(GPU_KIND_CLUSTER); \ + else \ + echo "Kind GPU cluster '$(GPU_KIND_CLUSTER)' does not exist; nothing to delete."; \ + fi + +# Run the GPU smoke tier against an already-configured GPU cluster. +# Runs the no-GPU specs PLUS the e2e_gpu specs. Caller is responsible +# for: a GPU node labeled nvidia.com/gpu.present=true with the nvidia +# RuntimeClass; KUBECONFIG pointing at it; pushing the operator image +# (pass IMG=); and any cluster cleanup. Knobs: VLLM_MODEL, +# VLLM_IMAGE, SKIP_VLLM_INTEGRATION (see vllm_integration_smoke_test.go). +# 60m timeout absorbs cold pulls of the ~10GB vllm-openai image. +.PHONY: test-e2e-gpu-cluster +test-e2e-gpu-cluster: _require-img manifests generate fmt vet ## Run GPU smoke against an existing GPU cluster. + @echo "Running GPU smoke against $$(kubectl config current-context) using IMG=$(IMG)" + IMG=$(IMG) SMOKE_SKIP_IMAGE_LOAD=true \ + go test -tags=e2e,e2e_gpu ./test/e2e/ -v -ginkgo.v -timeout 60m diff --git a/operator/make/e2e.mk b/operator/make/e2e.mk new file mode 100644 index 00000000000..117dd30d33a --- /dev/null +++ b/operator/make/e2e.mk @@ -0,0 +1,64 @@ +##@ E2E (no-GPU) + +# Both *-cluster targets need a registry-reachable IMG and refuse the +# scaffold default (controller:latest) which is meaningless to a remote +# cluster's image puller. Used as a prereq by every cluster-targeting +# recipe in both e2e.mk and e2e-gpu.mk. +.PHONY: _require-img +_require-img: + @if [ -z "$(IMG)" ] || [ "$(IMG)" = "controller:latest" ]; then \ + echo "ERROR: pass IMG= pointing at an image the cluster can pull"; \ + exit 1; \ + fi + +# Unique Kind cluster name per `make test-e2e-kind` invocation so +# concurrent runs never collide and cleanup only touches THIS run's +# cluster. Set KIND_CLUSTER explicitly to target an existing cluster +# (you're responsible for cleanup in that case). +ifndef KIND_CLUSTER +KIND_CLUSTER := operator-test-e2e-$(shell date +%s%N | tail -c 9) +endif + +.PHONY: setup-test-e2e-kind +setup-test-e2e-kind: ## Create a fresh, uniquely-named Kind cluster for e2e tests + @command -v $(KIND) >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @echo "Creating Kind cluster '$(KIND_CLUSTER)'..." + @$(KIND) create cluster --name $(KIND_CLUSTER) + +# setup-test-e2e-kind is invoked from inside the trapped shell rather +# than as a Make prerequisite, so a setup failure still triggers +# cleanup (prerequisites run BEFORE the recipe's shell installs the +# trap). KIND_CLUSTER is passed to both sub-makes so they target THIS +# run's cluster, not a newly-computed unique name. SIGKILL is +# uncatchable; on that rare path the cluster leaks under its unique +# name (`kind delete cluster --name ...` to clean up manually). +.PHONY: test-e2e-kind +test-e2e-kind: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. + @echo "==> Using Kind cluster: $(KIND_CLUSTER) (auto-deleted after the run)" + @trap '$(MAKE) --no-print-directory cleanup-test-e2e-kind KIND_CLUSTER=$(KIND_CLUSTER)' EXIT INT TERM; \ + $(MAKE) --no-print-directory setup-test-e2e-kind KIND_CLUSTER=$(KIND_CLUSTER) && \ + KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v -timeout 30m + +.PHONY: cleanup-test-e2e-kind +cleanup-test-e2e-kind: ## Delete the Kind cluster used for e2e tests (idempotent) + @if $(KIND) get clusters 2>/dev/null | grep -qx "$(KIND_CLUSTER)"; then \ + echo "Deleting Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) delete cluster --name $(KIND_CLUSTER); \ + else \ + echo "Kind cluster '$(KIND_CLUSTER)' does not exist; nothing to delete."; \ + fi + +# Run the smoke suite against an already-configured cluster (no Kind setup). +# Caller is responsible for: KUBECONFIG pointing at the cluster, the +# operator image being pushed to a registry the cluster can pull from, +# and any cluster cleanup. The suite will install CRDs, deploy the +# controller, run every spec, then undeploy + uninstall; the cluster +# is left intact. +.PHONY: test-e2e-cluster +test-e2e-cluster: _require-img manifests generate fmt vet ## Run smoke against the currently-configured cluster (no Kind). + @echo "Running smoke suite against $$(kubectl config current-context) using IMG=$(IMG)" + IMG=$(IMG) SMOKE_SKIP_IMAGE_LOAD=true \ + go test -tags=e2e ./test/e2e/ -v -ginkgo.v -timeout 30m diff --git a/operator/make/lint.mk b/operator/make/lint.mk new file mode 100644 index 00000000000..563d3e634ea --- /dev/null +++ b/operator/make/lint.mk @@ -0,0 +1,22 @@ +##@ Lint + +.PHONY: install-hooks +install-hooks: golangci-lint ## Install git pre-commit hooks (requires pre-commit: pip install pre-commit). + @command -v pre-commit >/dev/null 2>&1 || { \ + echo "pre-commit not found. Install with: pip install pre-commit"; \ + exit 1; \ + } + cd "$(shell git rev-parse --show-toplevel)" && pre-commit install -c operator/.pre-commit-config.yaml + @echo "Pre-commit hooks installed (operator)." + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter + "$(GOLANGCI_LINT)" run + +.PHONY: lint-fix +lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes + "$(GOLANGCI_LINT)" run --fix + +.PHONY: lint-config +lint-config: golangci-lint ## Verify golangci-lint linter configuration + "$(GOLANGCI_LINT)" config verify diff --git a/operator/make/tools.mk b/operator/make/tools.mk new file mode 100644 index 00000000000..3577481bee7 --- /dev/null +++ b/operator/make/tools.mk @@ -0,0 +1,83 @@ +##@ Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p "$(LOCALBIN)" + +## Tool Binaries +KUBECTL ?= kubectl +KIND ?= kind +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint + +## Tool Versions +KUSTOMIZE_VERSION ?= v5.8.1 +CONTROLLER_TOOLS_VERSION ?= v0.20.1 + +#ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) +ENVTEST_VERSION ?= $(shell v='$(call gomodver,sigs.k8s.io/controller-runtime)'; \ + [ -n "$$v" ] || { echo "Set ENVTEST_VERSION manually (controller-runtime replace has no tag)" >&2; exit 1; }; \ + printf '%s\n' "$$v" | sed -E 's/^v?([0-9]+)\.([0-9]+).*/release-\1.\2/') + +#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) +ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \ + [ -n "$$v" ] || { echo "Set ENVTEST_K8S_VERSION manually (k8s.io/api replace has no tag)" >&2; exit 1; }; \ + printf '%s\n' "$$v" | sed -E 's/^v?[0-9]+\.([0-9]+).*/1.\1/') + +GOLANGCI_LINT_VERSION ?= v2.8.0 + +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) + +.PHONY: setup-envtest +setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. + @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." + @"$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path || { \ + echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ + exit 1; \ + } + +.PHONY: envtest +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. +$(ENVTEST): $(LOCALBIN) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) + @test -f .custom-gcl.yml && { \ + echo "Building custom golangci-lint with plugins..." && \ + $(GOLANGCI_LINT) custom --destination $(LOCALBIN) --name golangci-lint-custom && \ + mv -f $(LOCALBIN)/golangci-lint-custom $(GOLANGCI_LINT); \ + } || true + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f "$(1)-$(3)" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)" ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +rm -f "$(1)" ;\ +GOBIN="$(LOCALBIN)" go install $${package} ;\ +mv "$(LOCALBIN)/$$(basename "$(1)")" "$(1)-$(3)" ;\ +} ;\ +ln -sf "$$(realpath "$(1)-$(3)")" "$(1)" +endef + +define gomodver +$(shell go list -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' $(1) 2>/dev/null) +endef diff --git a/operator/test/e2e/auth_smoke_test.go b/operator/test/e2e/auth_smoke_test.go new file mode 100644 index 00000000000..8ecef7c6a2f --- /dev/null +++ b/operator/test/e2e/auth_smoke_test.go @@ -0,0 +1,155 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/LMCache/LMCache/test/utils" +) + +// Cross-namespace authSecretRef. +// +// The operator mirrors a Secret referenced by spec.l2Backend.resp. +// authSecretRef into the LMCacheEngine's namespace, then injects +// LMCACHE_RESP_USERNAME / LMCACHE_RESP_PASSWORD into the DaemonSet +// container as env vars (NOT as CLI args). This spec asserts: +// +// 1. A managed copy of the source Secret appears in the engine's +// namespace at -resp-auth. +// 2. The pod template references the local copy via env-var +// SecretKeyRef — never via container args. +// 3. The literal Secret values never leak into container args +// (i.e. they wouldn't show up in `kubectl describe pod`). +var _ = Describe("LMCacheEngine cross-namespace auth smoke (no-GPU)", Ordered, func() { + var ( + ctx context.Context + nsName string + sourceNSName string + expectedUser = "tm-test-user" + expectedPass = "tm-test-password" // pragma: allowlist secret + ) + + BeforeEach(func() { + ctx = context.Background() + nsName = createTestNamespace(ctx) + // Source namespace lives separately so we test true cross-NS + // mirroring rather than same-NS direct reference. + sourceNSName = createTestNamespace(ctx) + }) + + AfterEach(func() { + recordOnFailure(nsName) + }) + + It("mirrors a cross-namespace authSecretRef and uses env-var injection", func() { + By("creating the source Secret in the upstream namespace") + src := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "redis-auth", + Namespace: sourceNSName, + }, + Data: map[string][]byte{ + "username": []byte(expectedUser), + "password": []byte(expectedPass), + }, + } + Expect(k8sClient.Create(ctx, src)).To(Succeed()) + + By("loading the fixture and pointing it at the source namespace") + lmc, err := utils.NewLMCFromFixture("lmc_with_redis_l2_authsecret.yaml", nsName, "") + Expect(err).NotTo(HaveOccurred()) + Expect(lmc.Spec.L2Backend).NotTo(BeNil()) + Expect(lmc.Spec.L2Backend.RESP).NotTo(BeNil()) + Expect(lmc.Spec.L2Backend.RESP.AuthSecretRef).NotTo(BeNil()) + lmc.Spec.L2Backend.RESP.AuthSecretRef.Namespace = sourceNSName + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + + key := engineKey(nsName, lmc.Name) + Expect(utils.WaitLMCReconciled(ctx, k8sClient, key, 60*time.Second)).To(Succeed()) + + managedSecretName := lmc.Name + "-resp-auth" + + By("eventually a managed copy of the Secret appears in the engine namespace") + Eventually(func(g Gomega) { + got := &corev1.Secret{} + g.Expect(k8sClient.Get(ctx, + types.NamespacedName{Namespace: nsName, Name: managedSecretName}, + got, + )).To(Succeed()) + g.Expect(got.Data).To(HaveKeyWithValue("password", []byte(expectedPass))) + g.Expect(got.Data).To(HaveKeyWithValue("username", []byte(expectedUser))) + }, 30*time.Second, time.Second).Should(Succeed()) + + By("checking the DaemonSet wires creds through env-var SecretKeyRef") + ds := &appsv1.DaemonSet{} + Expect(k8sClient.Get(ctx, types.NamespacedName(key), ds)).To(Succeed()) + Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) + c := ds.Spec.Template.Spec.Containers[0] + + passwordEnv := findEnvVar(c.Env, "LMCACHE_RESP_PASSWORD") + Expect(passwordEnv).NotTo(BeNil(), "expected LMCACHE_RESP_PASSWORD env var") + Expect(passwordEnv.Value).To(BeEmpty(), + "LMCACHE_RESP_PASSWORD must be sourced from a Secret, not a literal value") + Expect(passwordEnv.ValueFrom).NotTo(BeNil()) + Expect(passwordEnv.ValueFrom.SecretKeyRef).NotTo(BeNil()) + Expect(passwordEnv.ValueFrom.SecretKeyRef.Name).To(Equal(managedSecretName), + "password env var must reference the local managed Secret, not the source namespace") + Expect(passwordEnv.ValueFrom.SecretKeyRef.Key).To(Equal("password")) + + usernameEnv := findEnvVar(c.Env, "LMCACHE_RESP_USERNAME") + Expect(usernameEnv).NotTo(BeNil(), "expected LMCACHE_RESP_USERNAME env var") + Expect(usernameEnv.ValueFrom).NotTo(BeNil()) + Expect(usernameEnv.ValueFrom.SecretKeyRef).NotTo(BeNil()) + Expect(usernameEnv.ValueFrom.SecretKeyRef.Name).To(Equal(managedSecretName)) + + By("ensuring the literal Secret values do NOT appear in container args") + // "kubectl describe pod" prints the args verbatim; if the + // operator inlined credentials the user/pass would leak there. + // Env-var injection (the contract) keeps them out of the args. + joinedArgs := strings.Join(c.Args, " ") + Expect(joinedArgs).NotTo(ContainSubstring(expectedUser)) + Expect(joinedArgs).NotTo(ContainSubstring(expectedPass)) + }) +}) + +// findEnvVar returns a pointer to the named env var (so callers can +// inspect ValueFrom) or nil if absent. We return *corev1.EnvVar rather +// than the value type so a missing entry is unambiguously nil instead +// of the zero struct, which would be indistinguishable from an empty +// env var Value="". +func findEnvVar(envs []corev1.EnvVar, name string) *corev1.EnvVar { + for i := range envs { + if envs[i].Name == name { + return &envs[i] + } + } + return nil +} diff --git a/operator/test/e2e/crd_smoke_test.go b/operator/test/e2e/crd_smoke_test.go new file mode 100644 index 00000000000..ea8bba011b1 --- /dev/null +++ b/operator/test/e2e/crd_smoke_test.go @@ -0,0 +1,249 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/LMCache/LMCache/test/utils" +) + +// Validate that applying an LMCacheEngine produces the documented K8s +// objects with the documented shape. Two fixtures exercise the same +// assertions: lmc_minimal (defaults) and lmc_custom_port (port=6555, +// chunkSize=128). The minimal fixture additionally diffs the connection +// ConfigMap against a checked-in golden file so schema drift in +// kv-transfer-config.json fails loudly. +// +// The first It block is a harness sanity check — kept intentionally +// cheap so a broken helper fails before the richer reconciliation +// specs run. +var _ = Describe("LMCacheEngine smoke (no-GPU)", Ordered, func() { + var ( + ctx context.Context + nsName string + ) + + BeforeEach(func() { + ctx = context.Background() + nsName = createTestNamespace(ctx) + }) + + AfterEach(func() { + recordOnFailure(nsName) + }) + + It("reconciles a minimal CR and cleans up on delete (harness check)", func() { + lmc, err := utils.NewLMCFromFixture("lmc_minimal.yaml", nsName, "smoke-harness") + Expect(err).NotTo(HaveOccurred()) + + By("applying the minimal LMCacheEngine") + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + + By("waiting for the operator to observe the spec and validate the config") + Expect(utils.WaitLMCReconciled(ctx, k8sClient, + engineKey(nsName, "smoke-harness"), + 60*time.Second, + )).To(Succeed()) + + By("ensuring the connection ConfigMap has been produced") + cfg, err := utils.GetConnectionConfig(ctx, k8sClient, engineKey(nsName, "smoke-harness")) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.KVConnector).To(Equal("LMCacheMPConnector")) + + By("deleting the CR and waiting for owned objects to be garbage-collected") + Expect(utils.DeleteLMCAndWaitGC(ctx, k8sClient, + engineKey(nsName, "smoke-harness"), + 60*time.Second, + )).To(Succeed()) + }) + + It("reconciles a minimal CR into the documented K8s artifacts", func() { + // Use the fixture's own metadata.name so the golden file's + // hostname matches verbatim ("smoke-minimal..svc..."). + lmc, err := utils.NewLMCFromFixture("lmc_minimal.yaml", nsName, "") + Expect(err).NotTo(HaveOccurred()) + Expect(lmc.Name).To(Equal("smoke-minimal")) + + By("applying the minimal LMCacheEngine") + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + + key := engineKey(nsName, lmc.Name) + + By("waiting for the operator to observe the spec") + Expect(utils.WaitLMCReconciled(ctx, k8sClient, key, 60*time.Second)).To(Succeed()) + + By("validating the DaemonSet pod template shape") + ds := &appsv1.DaemonSet{} + Expect(k8sClient.Get(ctx, types.NamespacedName(key), ds)).To(Succeed()) + assertDaemonSetShape(ds, 5555 /* default server port */) + + By("validating the lookup Service shape") + svc := &corev1.Service{} + Expect(k8sClient.Get(ctx, types.NamespacedName(key), svc)).To(Succeed()) + assertLookupServiceShape(svc, 5555) + + By("validating the connection ConfigMap matches the documented contract") + cfg, err := utils.GetConnectionConfig(ctx, k8sClient, key) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.KVConnector).To(Equal("LMCacheMPConnector")) + Expect(cfg.KVRole).To(Equal("kv_both")) + Expect(cfg.KVConnectorExtraConfig.Host).To(Equal( + fmt.Sprintf("tcp://smoke-minimal.%s.svc.cluster.local", nsName))) + Expect(cfg.KVConnectorExtraConfig.Port).To(Equal("5555")) + + By("diffing the raw kv-transfer-config.json against the golden snapshot") + assertGoldenKvTransferConfig(ctx, key, "kv_transfer_config_minimal.json", nsName) + }) + + It("propagates server.port and server.chunkSize into args, Service, and ConfigMap", func() { + lmc, err := utils.NewLMCFromFixture("lmc_custom_port.yaml", nsName, "") + Expect(err).NotTo(HaveOccurred()) + + By("applying the custom-port LMCacheEngine") + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + + key := engineKey(nsName, lmc.Name) + + By("waiting for the operator to observe the spec") + Expect(utils.WaitLMCReconciled(ctx, k8sClient, key, 60*time.Second)).To(Succeed()) + + By("validating the DaemonSet pod template") + ds := &appsv1.DaemonSet{} + Expect(k8sClient.Get(ctx, types.NamespacedName(key), ds)).To(Succeed()) + assertDaemonSetShape(ds, 6555 /* spec.server.port */) + + args := containerArgs(ds) + Expect(argValue(args, "--port")).To(Equal("6555")) + Expect(argValue(args, "--chunk-size")).To(Equal("128")) + + By("validating the lookup Service uses the custom port") + svc := &corev1.Service{} + Expect(k8sClient.Get(ctx, types.NamespacedName(key), svc)).To(Succeed()) + assertLookupServiceShape(svc, 6555) + + By("validating the ConfigMap reflects the custom port") + cfg, err := utils.GetConnectionConfig(ctx, k8sClient, key) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.KVConnectorExtraConfig.Port).To(Equal("6555")) + }) +}) + +// assertDaemonSetShape verifies the operator's auto-injected pod-level +// settings: hostIPC, runtimeClassName=nvidia, container privileged +// security context, --host 0.0.0.0 always present in container args, +// and the absence of any /dev/shm volume mount that would shadow the +// host's /dev/shm and break CUDA IPC. +func assertDaemonSetShape(ds *appsv1.DaemonSet, expectedServerPort int32) { + GinkgoHelper() + pod := ds.Spec.Template.Spec + Expect(pod.HostIPC).To(BeTrue(), "pod.hostIPC must be true") + Expect(pod.RuntimeClassName).NotTo(BeNil(), "pod.runtimeClassName must be set") + Expect(*pod.RuntimeClassName).To(Equal("nvidia")) + Expect(pod.Containers).To(HaveLen(1)) + + c := pod.Containers[0] + Expect(c.SecurityContext).NotTo(BeNil(), "container.securityContext must be set") + Expect(c.SecurityContext.Privileged).NotTo(BeNil()) + Expect(*c.SecurityContext.Privileged).To(BeTrue(), "container.privileged must be true") + Expect(c.Args).To(ContainElements("--host", "0.0.0.0")) + Expect(argValue(c.Args, "--port")).To(Equal(fmt.Sprintf("%d", expectedServerPort))) + + // /dev/shm is left to the host via hostIPC=true. An emptyDir mount + // would shadow it and break cudaIpcOpenMemHandle between LMCache + // and vLLM pods. Verify both sides: no volume mount AND no volume. + for _, vm := range c.VolumeMounts { + Expect(vm.MountPath).NotTo(Equal("/dev/shm"), + "unexpected /dev/shm volumeMount on lmcache container") + } + for _, v := range pod.Volumes { + // Some helm charts call this volume "shm" or "dshm"; rather + // than enumerate names we check for any volume mounted at + // /dev/shm via the loop above. This loop catches a different + // mistake: an emptyDir/Volume named after /dev/shm with no + // matching mount, which is harmless but unexpected. + if v.EmptyDir != nil { + Expect(v.Name).NotTo(Or(Equal("shm"), Equal("dshm"), Equal("dev-shm")), + "unexpected /dev/shm-style emptyDir volume present") + } + } +} + +// assertLookupServiceShape checks the node-local discovery Service has +// internalTrafficPolicy=Local (so kube-proxy routes only to the on-node +// LMCache pod) and exposes the spec'd server port. +func assertLookupServiceShape(svc *corev1.Service, expectedServerPort int32) { + GinkgoHelper() + Expect(svc.Spec.InternalTrafficPolicy).NotTo(BeNil()) + Expect(*svc.Spec.InternalTrafficPolicy).To(Equal(corev1.ServiceInternalTrafficPolicyLocal)) + var found bool + for _, p := range svc.Spec.Ports { + if p.Name == "server" { + found = true + Expect(p.Port).To(Equal(expectedServerPort)) + } + } + Expect(found).To(BeTrue(), "lookup Service must expose a port named 'server'") +} + +// assertGoldenKvTransferConfig compares the on-cluster ConfigMap data +// to the embedded golden snapshot. The placeholder __NAMESPACE__ in the +// golden file is substituted with the live test namespace before +// comparison so the diff sees a single reproducible string. +func assertGoldenKvTransferConfig(ctx context.Context, key types.NamespacedName, goldenName, namespace string) { + GinkgoHelper() + golden, err := utils.LoadGolden(goldenName) + Expect(err).NotTo(HaveOccurred(), "load golden %s", goldenName) + expected := strings.ReplaceAll(string(golden), "__NAMESPACE__", namespace) + + cm := &corev1.ConfigMap{} + cmKey := types.NamespacedName{Namespace: key.Namespace, Name: key.Name + "-connection"} + Expect(k8sClient.Get(ctx, cmKey, cm)).To(Succeed()) + actual, ok := cm.Data["kv-transfer-config.json"] + Expect(ok).To(BeTrue(), "ConfigMap missing kv-transfer-config.json") + + // Normalize both sides through json.Unmarshal+Marshal to absorb any + // trailing-newline / line-ending differences while still catching + // genuine schema drift. We compare on the canonical re-encoding. + Expect(canonicalJSON(actual)).To(Equal(canonicalJSON(expected)), + "kv-transfer-config.json drift vs %s\nactual:\n%s\nexpected:\n%s", + goldenName, actual, expected) +} + +func canonicalJSON(s string) string { + var v any + if err := json.Unmarshal([]byte(s), &v); err != nil { + // Return as-is so the caller's diff reveals the malformed input. + return s + } + out, _ := json.MarshalIndent(v, "", " ") + return string(out) +} diff --git a/operator/test/e2e/e2e_suite_test.go b/operator/test/e2e/e2e_suite_test.go index 7cc35853728..216b81b2f30 100644 --- a/operator/test/e2e/e2e_suite_test.go +++ b/operator/test/e2e/e2e_suite_test.go @@ -20,28 +20,53 @@ limitations under the License. package e2e import ( + "context" "fmt" "os" "os/exec" "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + lmcachev1alpha1 "github.com/LMCache/LMCache/api/v1alpha1" "github.com/LMCache/LMCache/test/utils" ) var ( - // managerImage is the manager image to be built and loaded for testing. - managerImage = "example.com/operator:v0.0.1" - // shouldCleanupCertManager tracks whether CertManager was installed by this suite. - shouldCleanupCertManager = false + // managerImage is the manager image used by build / load / deploy. + // Honors the IMG env var so test-e2e-cluster can point at an image + // already pushed to a registry the target cluster pulls from. + managerImage = envDefault("IMG", "example.com/operator:v0.0.1") + // skipImageLoad disables the Kind-only docker-build + kind-load + // steps. Set SMOKE_SKIP_IMAGE_LOAD=true when running against an + // existing cluster (OpenShift, EKS, k3s, etc.) where the image has + // already been pushed to a reachable registry. + skipImageLoad = os.Getenv("SMOKE_SKIP_IMAGE_LOAD") == "true" + // k8sClient is the typed controller-runtime client used by smoke specs. + // Initialised once in BeforeSuite after CRDs are installed; spec files + // read it directly without rebuilding their own client. + k8sClient client.Client ) +// envDefault returns os.Getenv(key) or def if the env var is unset/empty. +// Pulled out as a helper so package-level var initialisers stay readable. +func envDefault(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + // TestE2E runs the e2e test suite to validate the solution in an isolated environment. -// The default setup requires Kind and CertManager. -// -// To skip CertManager installation, set: CERT_MANAGER_INSTALL_SKIP=true +// Requires Kind (or an existing cluster reachable via the current kubeconfig). func TestE2E(t *testing.T) { RegisterFailHandler(Fail) _, _ = fmt.Fprintf(GinkgoWriter, "Starting operator e2e test suite\n") @@ -49,53 +74,88 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { - By("building the manager image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", managerImage)) - _, err := utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager image") - - // TODO(user): If you want to change the e2e test vendor from Kind, - // ensure the image is built and available, then remove the following block. - By("loading the manager image on Kind") - err = utils.LoadImageToKindClusterWithName(managerImage) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager image into Kind") - - setupCertManager() -}) + _, _ = fmt.Fprintf(GinkgoWriter, "manager image: %s (skipImageLoad=%v)\n", + managerImage, skipImageLoad) + + if skipImageLoad { + // Existing-cluster path: the user pushed the image to a + // registry before invoking the suite. We can't sideload, and + // we don't rebuild because rebuilds wouldn't propagate to the + // pushed copy anyway. The deploy step below uses managerImage + // as the registry URL. + By("skipping docker-build + kind-load (SMOKE_SKIP_IMAGE_LOAD=true)") + } else { + By("building the manager image") + _, err := utils.RunMake("docker-build", fmt.Sprintf("IMG=%s", managerImage)) + Expect(err).NotTo(HaveOccurred(), "Failed to build the manager image") + + By("loading the manager image on Kind") + err = utils.LoadImageToKindClusterWithName(managerImage) + Expect(err).NotTo(HaveOccurred(), "Failed to load the manager image into Kind") + } -var _ = AfterSuite(func() { - teardownCertManager() + By("installing CRDs") + _, err := utils.RunMake("install") + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + _, err = utils.RunMake("deploy", fmt.Sprintf("IMG=%s", managerImage)) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + + By("labeling the operator namespace with the restricted Pod Security profile") + labelCmd := exec.Command("kubectl", "label", "--overwrite", "ns", + "lmcache-operator-system", + "pod-security.kubernetes.io/enforce=restricted", + ) + _, err = labelCmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "Failed to label operator namespace") + + By("registering custom types in the scheme") + Expect(lmcachev1alpha1.AddToScheme(scheme.Scheme)).To(Succeed()) + Expect(monitoringv1.AddToScheme(scheme.Scheme)).To(Succeed()) + + By("building the typed Kubernetes client") + cfg, err := ctrl.GetConfig() + Expect(err).NotTo(HaveOccurred(), "Failed to load kubeconfig") + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred(), "Failed to construct typed client") + + By("waiting for the controller-manager Deployment to become Available") + Expect(waitDeploymentAvailable(context.Background(), + "lmcache-operator-system", + "lmcache-operator-controller-manager", + 3*time.Minute, + )).To(Succeed(), "Controller-manager Deployment did not become Available") }) -// setupCertManager installs CertManager if needed for webhook tests. -// Skips installation if CERT_MANAGER_INSTALL_SKIP=true or if already present. -func setupCertManager() { - if os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" { - _, _ = fmt.Fprintf(GinkgoWriter, "Skipping CertManager installation (CERT_MANAGER_INSTALL_SKIP=true)\n") - return +var _ = AfterSuite(func() { + By("undeploying the controller-manager") + if _, err := utils.RunMake("undeploy", "ignore-not-found=true"); err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "warning: undeploy failed: %v\n", err) } - By("checking if CertManager is already installed") - if utils.IsCertManagerCRDsInstalled() { - _, _ = fmt.Fprintf(GinkgoWriter, "CertManager is already installed. Skipping installation.\n") - return - } - - // Mark for cleanup before installation to handle interruptions and partial installs. - shouldCleanupCertManager = true - - By("installing CertManager") - Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") -} - -// teardownCertManager uninstalls CertManager if it was installed by setupCertManager. -// This ensures we only remove what we installed. -func teardownCertManager() { - if !shouldCleanupCertManager { - _, _ = fmt.Fprintf(GinkgoWriter, "Skipping CertManager cleanup (not installed by this suite)\n") - return + By("uninstalling CRDs") + if _, err := utils.RunMake("uninstall", "ignore-not-found=true"); err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "warning: uninstall failed: %v\n", err) } +}) - By("uninstalling CertManager") - utils.UninstallCertManager() +// waitDeploymentAvailable polls a Deployment's status until the +// Available condition is True, or until ctx is cancelled / timeout +// elapses. Uses kubectl rather than the typed client because the typed +// client only knows the schema we registered, not Deployments. +func waitDeploymentAvailable(ctx context.Context, namespace, name string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + // kubectl exits 0 with empty output if the condition isn't met + // yet, and 0 with "True" when it is — so we run it and inspect + // stdout rather than relying on exit codes. + cmd := exec.CommandContext(ctx, "kubectl", "get", "deployment", name, + "-n", namespace, + "-o", "jsonpath={.status.conditions[?(@.type=='Available')].status}") + out, err := cmd.Output() + if err != nil { + return false, nil + } + return string(out) == "True", nil + }) } diff --git a/operator/test/e2e/e2e_test.go b/operator/test/e2e/e2e_test.go index 0a1bc17fb76..0d586209e92 100644 --- a/operator/test/e2e/e2e_test.go +++ b/operator/test/e2e/e2e_test.go @@ -51,47 +51,10 @@ const metricsRoleBindingName = "operator-metrics-binding" var _ = Describe("Manager", Ordered, func() { var controllerPodName string - // Before running the tests, set up the environment by creating the namespace, - // enforce the restricted security policy to the namespace, installing CRDs, - // and deploying the controller. - BeforeAll(func() { - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") - - By("labeling the namespace to enforce the restricted security policy") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, - "pod-security.kubernetes.io/enforce=restricted") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") - - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") - - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", managerImage)) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") - }) - - // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, - // and deleting the namespace. - AfterAll(func() { - By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") - _, _ = utils.Run(cmd) - - By("uninstalling CRDs") - cmd = exec.Command("make", "uninstall") - _, _ = utils.Run(cmd) - - By("removing manager namespace") - cmd = exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) - }) + // Suite-level BeforeSuite (in integration_suite_test.go) handles building/loading + // the manager image, installing CRDs, deploying the controller, and + // labeling the operator namespace. This Describe block focuses on the + // metrics-endpoint contract; per-test setup runs in AfterEach. // After each test, check for failures and collect logs, events, // and pod descriptions for debugging. diff --git a/operator/test/e2e/field_coverage_smoke_test.go b/operator/test/e2e/field_coverage_smoke_test.go new file mode 100644 index 00000000000..33089aff09e --- /dev/null +++ b/operator/test/e2e/field_coverage_smoke_test.go @@ -0,0 +1,133 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + + "github.com/LMCache/LMCache/test/utils" +) + +// Field coverage smoke checks that non-default spec fields flow +// through to the reconciled K8s objects. +// +// - ServiceMonitor: enabling prometheus.serviceMonitor produces a +// ServiceMonitor CR with the configured labels. Auto-skipped when +// the Prometheus Operator CRDs are absent. +// - extraArgs: the user's --max-workers 4 wins over the +// operator's auto-generated --max-workers 1 (extraArgs are +// appended LAST by contract). +// - resourceOverrides: an explicit resourceOverrides block fully +// replaces the auto-computed memory request. +var _ = Describe("LMCacheEngine field coverage smoke (no-GPU)", Ordered, func() { + var ( + ctx context.Context + nsName string + ) + + BeforeEach(func() { + ctx = context.Background() + nsName = createTestNamespace(ctx) + }) + + AfterEach(func() { + recordOnFailure(nsName) + }) + + It("creates a ServiceMonitor with the configured labels", func() { + if !serviceMonitorCRDInstalled() { + Skip("monitoring.coreos.com ServiceMonitor CRD not installed; skipping ServiceMonitor spec") + } + + lmc, err := utils.NewLMCFromFixture("lmc_servicemonitor.yaml", nsName, "") + Expect(err).NotTo(HaveOccurred()) + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + + key := engineKey(nsName, lmc.Name) + Expect(utils.WaitLMCReconciled(ctx, k8sClient, key, 60*time.Second)).To(Succeed()) + + By("eventually a ServiceMonitor exists with the configured labels") + Eventually(func(g Gomega) { + sm := &monitoringv1.ServiceMonitor{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName(key), sm)).To(Succeed()) + g.Expect(sm.Labels).To(HaveKeyWithValue("release", "kube-prometheus-stack")) + g.Expect(sm.Spec.Endpoints).NotTo(BeEmpty()) + g.Expect(sm.Spec.Endpoints[0].Port).To(Equal("metrics")) + g.Expect(string(sm.Spec.Endpoints[0].Interval)).To(Equal("30s")) + }, 30*time.Second, time.Second).Should(Succeed()) + }) + + It("lets spec.extraArgs override the auto-generated --max-workers", func() { + lmc, err := utils.NewLMCFromFixture("lmc_minimal.yaml", nsName, "fields-extraargs") + Expect(err).NotTo(HaveOccurred()) + lmc.Spec.ExtraArgs = []string{"--max-workers", "4"} + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + + key := engineKey(nsName, lmc.Name) + Expect(utils.WaitLMCReconciled(ctx, k8sClient, key, 60*time.Second)).To(Succeed()) + + By("checking the DaemonSet container args show --max-workers=4 wins") + Eventually(func(g Gomega) { + ds := &appsv1.DaemonSet{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName(key), ds)).To(Succeed()) + args := containerArgs(ds) + // Both occurrences should be present — the auto-generated 1 + // and the override 4 — so we can prove the override is + // strictly the LAST one rather than a clobber of the first. + g.Expect(args).To(ContainElements("--max-workers", "1", "--max-workers", "4")) + g.Expect(argValueLast(args, "--max-workers")).To(Equal("4")) + }, 30*time.Second, time.Second).Should(Succeed()) + }) + + It("lets spec.resourceOverrides replace the auto-computed memory", func() { + lmc, err := utils.NewLMCFromFixture("lmc_minimal.yaml", nsName, "fields-resourceoverride") + Expect(err).NotTo(HaveOccurred()) + lmc.Spec.ResourceOverrides = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("70Gi"), + }, + } + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + + key := engineKey(nsName, lmc.Name) + Expect(utils.WaitLMCReconciled(ctx, k8sClient, key, 60*time.Second)).To(Succeed()) + + By("checking the DaemonSet container resources reflect the override") + Eventually(func(g Gomega) { + ds := &appsv1.DaemonSet{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName(key), ds)).To(Succeed()) + g.Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) + req := ds.Spec.Template.Spec.Containers[0].Resources.Requests + g.Expect(req).To(HaveKey(corev1.ResourceMemory)) + memQty := req[corev1.ResourceMemory] + g.Expect(memQty.String()).To(Equal("70Gi")) + }, 30*time.Second, time.Second).Should(Succeed()) + }) +}) diff --git a/operator/test/e2e/lifecycle_smoke_test.go b/operator/test/e2e/lifecycle_smoke_test.go new file mode 100644 index 00000000000..46f119d9307 --- /dev/null +++ b/operator/test/e2e/lifecycle_smoke_test.go @@ -0,0 +1,146 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + lmcachev1alpha1 "github.com/LMCache/LMCache/api/v1alpha1" + "github.com/LMCache/LMCache/test/utils" +) + +// Lifecycle smoke covers the three CR-edit scenarios: +// +// - Update propagation: patching spec.server.port flows into the +// ConfigMap data and DaemonSet container args. +// - Finalizer cleanup: deleting the CR removes every owned object +// within 60s; anything longer signals a stuck finalizer (a real +// bug we want to catch). +// - Invalid spec: l1.sizeGB=-1 is rejected by the API server +// at admission time; the controller never sees it and no +// DaemonSet/Service is created. +var _ = Describe("LMCacheEngine lifecycle smoke (no-GPU)", Ordered, func() { + var ( + ctx context.Context + nsName string + ) + + BeforeEach(func() { + ctx = context.Background() + nsName = createTestNamespace(ctx) + }) + + AfterEach(func() { + recordOnFailure(nsName) + }) + + It("patches spec.server.port and the new value flows to ConfigMap and DaemonSet", func() { + lmc, err := utils.NewLMCFromFixture("lmc_minimal.yaml", nsName, "lifecycle-update") + Expect(err).NotTo(HaveOccurred()) + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + + key := engineKey(nsName, lmc.Name) + Expect(utils.WaitLMCReconciled(ctx, k8sClient, key, 60*time.Second)).To(Succeed()) + + By("baseline: ConfigMap and DaemonSet show the default port 5555") + baseCfg, err := utils.GetConnectionConfig(ctx, k8sClient, key) + Expect(err).NotTo(HaveOccurred()) + Expect(baseCfg.KVConnectorExtraConfig.Port).To(Equal("5555")) + + By("patching spec.server.port from 5555 to 6555") + Expect(utils.PatchLMCSpec(ctx, k8sClient, types.NamespacedName(key), + func(spec *lmcachev1alpha1.LMCacheEngineSpec) { + if spec.Server == nil { + spec.Server = &lmcachev1alpha1.ServerSpec{} + } + newPort := int32(6555) + spec.Server.Port = &newPort + }, + )).To(Succeed()) + + By("eventually the ConfigMap reflects the new port") + Eventually(func(g Gomega) { + cfg, err := utils.GetConnectionConfig(ctx, k8sClient, key) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cfg.KVConnectorExtraConfig.Port).To(Equal("6555")) + }, 60*time.Second, time.Second).Should(Succeed()) + + By("eventually the DaemonSet container args contain the new port") + Eventually(func(g Gomega) { + ds := &appsv1.DaemonSet{} + g.Expect(k8sClient.Get(ctx, types.NamespacedName(key), ds)).To(Succeed()) + g.Expect(argValue(containerArgs(ds), "--port")).To(Equal("6555")) + }, 60*time.Second, time.Second).Should(Succeed()) + }) + + It("deletes the CR and garbage-collects DaemonSet, Service, and ConfigMap within 60s", func() { + lmc, err := utils.NewLMCFromFixture("lmc_minimal.yaml", nsName, "lifecycle-delete") + Expect(err).NotTo(HaveOccurred()) + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + + key := engineKey(nsName, lmc.Name) + Expect(utils.WaitLMCReconciled(ctx, k8sClient, key, 60*time.Second)).To(Succeed()) + + By("ensuring the owned objects are present before deletion") + Expect(k8sClient.Get(ctx, types.NamespacedName(key), &appsv1.DaemonSet{})).To(Succeed()) + Expect(k8sClient.Get(ctx, types.NamespacedName(key), &corev1.Service{})).To(Succeed()) + Expect(k8sClient.Get(ctx, + types.NamespacedName{Namespace: key.Namespace, Name: key.Name + "-connection"}, + &corev1.ConfigMap{}, + )).To(Succeed()) + + By("deleting the CR — finalizer + owner refs must clean up everything in 60s") + Expect(utils.DeleteLMCAndWaitGC(ctx, k8sClient, types.NamespacedName(key), 60*time.Second)).To(Succeed()) + }) + + It("rejects l1.sizeGB=-1 at admission and creates no owned objects", func() { + lmc, err := utils.NewLMCFromFixture("lmc_minimal.yaml", nsName, "lifecycle-invalid") + Expect(err).NotTo(HaveOccurred()) + lmc.Spec.L1.SizeGB = -1 + + By("attempting to create the CR — API server must reject with 422 Invalid") + applyErr := utils.ApplyLMC(ctx, k8sClient, lmc) + Expect(applyErr).To(HaveOccurred(), "expected API server to reject sizeGB=-1") + Expect(apierrors.IsInvalid(applyErr)).To(BeTrue(), + "expected an Invalid (422) status error, got: %v", applyErr) + + key := engineKey(nsName, "lifecycle-invalid") + + By("ensuring no LMCacheEngine, DaemonSet, or Service was created") + Expect(apierrors.IsNotFound( + k8sClient.Get(ctx, types.NamespacedName(key), &lmcachev1alpha1.LMCacheEngine{}), + )).To(BeTrue(), "rejected CR must not appear in the API") + Expect(apierrors.IsNotFound( + k8sClient.Get(ctx, types.NamespacedName(key), &appsv1.DaemonSet{}), + )).To(BeTrue(), "no DaemonSet may exist for a rejected CR") + Expect(apierrors.IsNotFound( + k8sClient.Get(ctx, types.NamespacedName(key), &corev1.Service{}), + )).To(BeTrue(), "no Service may exist for a rejected CR") + }) +}) diff --git a/operator/test/e2e/runtime_smoke_test.go b/operator/test/e2e/runtime_smoke_test.go new file mode 100644 index 00000000000..5b51d9610df --- /dev/null +++ b/operator/test/e2e/runtime_smoke_test.go @@ -0,0 +1,117 @@ +//go:build e2e && e2e_gpu +// +build e2e,e2e_gpu + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/LMCache/LMCache/test/utils" +) + +// Runtime smoke (e2e_gpu): verifies that the values declared on the CR +// actually take effect inside the LMCache server process — not just on +// the K8s objects. The M1 (no-GPU) suite stops at the K8s artifact +// shape; this spec runs the pod and probes its HTTP frontend so a +// regression that, say, swallows --chunk-size in arg-parsing is caught +// even when the DaemonSet container args look correct. +// +// We probe `/status` because it returns is_healthy, chunk_size, and +// hash_algorithm in one payload — enough to catch arg-parsing +// regressions on those fields. The other CR fields (port, max_workers, +// http_port) remain covered by the M1 K8s-side specs that diff the +// DaemonSet container args. +var _ = Describe("LMCacheEngine runtime smoke (GPU)", Ordered, func() { + var ( + ctx context.Context + nsName string + ) + + BeforeEach(func() { + ctx = context.Background() + nsName = createTestNamespace(ctx) + }) + + AfterEach(func() { + recordOnFailure(nsName) + }) + + It("GET /status reflects the CR's chunk_size + hash_algorithm end-to-end", func() { + lmc, err := utils.NewLMCFromFixture("lmc_runtime.yaml", nsName, "") + Expect(err).NotTo(HaveOccurred()) + + // Capture the spec values we'll assert against /status below. + // Reading via derefInt32-style accessors keeps the assertion + // independent of the operator's defaulting logic — we test + // "what the CR + defaults imply" vs "what the live server reports." + expectedChunkSize := int32(128) + expectedHashAlgorithm := "blake3" + // Used for port-forward target; matches lmc_runtime.yaml. + const httpPort int32 = 8080 + + By("applying the runtime LMCacheEngine") + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + key := engineKey(nsName, lmc.Name) + Expect(utils.WaitLMCReconciled(ctx, k8sClient, key, 60*time.Second)).To(Succeed()) + + By("waiting for the LMCache DaemonSet pod to become Ready") + // 8 min is intentional: this includes image pull on a cold node. + // Container image is lmcache/vllm-openai:latest (~10GB). + podName, err := utils.WaitDaemonSetPodReady(ctx, k8sClient, key, 8*time.Minute) + Expect(err).NotTo(HaveOccurred(), "LMCache pod did not become Ready") + + By(fmt.Sprintf("port-forwarding to pod %s:%d", podName, httpPort)) + closer, baseURL, err := utils.PortForward( + utils.PortForwardSpec{Namespace: nsName, Target: "pod/" + podName}, + fmt.Sprintf("0:%d", httpPort), + ) + Expect(err).NotTo(HaveOccurred()) + defer closer() + + By("waiting for /healthcheck to return 200") + Expect(utils.WaitHTTP200(ctx, baseURL+"/healthcheck", 2*time.Minute)).To(Succeed(), + "LMCache HTTP frontend did not become healthy") + + By("fetching /status and asserting CR fields propagated to the live server") + status := &statusPayload{} + Expect(utils.HTTPGetJSON(ctx, baseURL+"/status", status)).To(Succeed()) + Expect(status.IsHealthy).To(BeTrue(), + "status.is_healthy must be true — engine did not initialize cleanly") + Expect(status.ChunkSize).To(Equal(expectedChunkSize), + "status.chunk_size: live server disagrees with spec.server.chunkSize") + Expect(status.HashAlgorithm).To(Equal(expectedHashAlgorithm), + "status.hash_algorithm: live server disagrees with spec.server.hashAlgorithm") + }) +}) + +// statusPayload is the strict subset of /status that the runtime +// spec asserts against. Fields we don't assert on are intentionally +// omitted — the server adds operational fields (registered_gpu_ids, +// active_sessions, storage_manager) that change every run and would +// just be noise here. +type statusPayload struct { + IsHealthy bool `json:"is_healthy"` + ChunkSize int32 `json:"chunk_size"` + HashAlgorithm string `json:"hash_algorithm"` +} diff --git a/operator/test/e2e/smoke_helpers_test.go b/operator/test/e2e/smoke_helpers_test.go new file mode 100644 index 00000000000..b901bb7c978 --- /dev/null +++ b/operator/test/e2e/smoke_helpers_test.go @@ -0,0 +1,261 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os/exec" + "strings" + "sync/atomic" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint:revive,staticcheck + . "github.com/onsi/gomega" //nolint:revive,staticcheck + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// nsCounter is a process-wide monotonic counter appended to every +// generated namespace name. CurrentSpecReport().StartTime alone is not +// enough — specs that call createTestNamespace twice in a single +// BeforeEach (e.g. the cross-namespace auth spec, which needs both an +// engine namespace and a source-Secret namespace) would otherwise compute the same name +// twice and collide on AlreadyExists. +var nsCounter atomic.Int64 + +// createTestNamespace creates a fresh namespace whose name embeds the +// running spec's text to make failing-spec dumps easier to attribute. +// The deferred AfterEach is wired up by the caller via DeferCleanup so +// each spec gets per-test isolation without relying on package state. +// +// The namespace is pre-labeled with the `privileged` PodSecurity profile +// because the LMCache DaemonSet's pod template sets hostIPC=true and +// privileged=true. On clusters that enforce PodSecurity admission (OCP +// in particular), a `restricted` namespace would reject the DaemonSet at +// creation time, which the smokes need to succeed even though they never +// wait for pods to schedule. The label is a no-op on clusters that don't +// enforce PodSecurity. +func createTestNamespace(ctx context.Context) string { + GinkgoHelper() + name := uniqueNamespaceName() + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + Expect(k8sClient.Create(ctx, ns)).To(Succeed(), "Failed to create test namespace %q", name) + DeferCleanup(func(ctx SpecContext) { + deleteNamespace(ctx, name) + }) + return name +} + +// uniqueNamespaceName produces a DNS-1123-safe namespace name that's +// unique within the test process. The time component disambiguates +// across runs / Kind-cluster restarts; the counter disambiguates +// multiple calls inside a single spec. +func uniqueNamespaceName() string { + n := nsCounter.Add(1) + hash := fmt.Sprintf("%x", time.Now().UnixNano()) + if len(hash) > 8 { + hash = hash[len(hash)-8:] + } + return fmt.Sprintf("lmc-smoke-%s-%d", hash, n) +} + +// deleteNamespace tears down a test namespace. We tolerate NotFound so +// repeated cleanup calls don't fail; the namespace's contents are GC'd +// asynchronously by Kubernetes. +func deleteNamespace(ctx context.Context, name string) { + GinkgoHelper() + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}} + if err := k8sClient.Delete(ctx, ns); err != nil && !apierrors.IsNotFound(err) { + _, _ = fmt.Fprintf(GinkgoWriter, "warning: delete namespace %q failed: %v\n", name, err) + } +} + +// dumpControllerLogs writes the last lines of every +// controller-manager pod's log to GinkgoWriter. Used by AfterEach +// failure handlers to make CI dumps useful without paging the entire +// reconcile history. +func dumpControllerLogs(tail int) { + GinkgoHelper() + podName, err := controllerPodNameOnce() + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "warning: could not locate controller pod: %v\n", err) + return + } + cmd := exec.Command("kubectl", "logs", podName, + "-n", "lmcache-operator-system", + fmt.Sprintf("--tail=%d", tail), + ) + out, _ := cmd.CombinedOutput() + _, _ = fmt.Fprintf(GinkgoWriter, "controller-manager logs (tail=%d):\n%s\n", tail, string(out)) +} + +// dumpNamespace writes events, pods, and the LMCacheEngine CR yaml from +// the given namespace to GinkgoWriter. Designed to be cheap to call +// from AfterEach without flooding output on success. +func dumpNamespace(ns string) { + GinkgoHelper() + for _, args := range [][]string{ + {"get", "events", "-n", ns, "--sort-by=.lastTimestamp"}, + {"get", "lmcacheengines", "-n", ns, "-o", "yaml"}, + {"get", "pods", "-n", ns, "-o", "wide"}, + {"describe", "pods", "-n", ns}, + } { + out, _ := exec.Command("kubectl", args...).CombinedOutput() + _, _ = fmt.Fprintf(GinkgoWriter, "$ kubectl %v\n%s\n", args, string(out)) + } + + // Per-pod logs (last 200 lines). The vLLM and LMCache pods produce + // most of the diagnostic signal on failure; without their logs, + // `describe pod` only tells us readiness flapped, not why. The + // `--previous` fetches the prior-instantiation logs when the + // container has crashed and restarted — that's where mid-inference + // CUDA / Python tracebacks live. + podNames := exec.Command("kubectl", "get", "pods", "-n", ns, + "-o", "jsonpath={range .items[*]}{.metadata.name} {end}") + out, err := podNames.Output() + if err != nil { + return + } + for _, podName := range strings.Fields(string(out)) { + for _, args := range [][]string{ + {"logs", "-n", ns, podName, "--tail=200", "--all-containers"}, + {"logs", "-n", ns, podName, "--tail=200", "--all-containers", "--previous"}, + } { + cmd := exec.Command("kubectl", args...) + cmd.Stderr = nil // mute "previous terminated container not found" noise + out, _ := cmd.Output() + if len(out) == 0 { + continue + } + _, _ = fmt.Fprintf(GinkgoWriter, "$ kubectl %v\n%s\n", args, string(out)) + } + } +} + +// controllerPodNameOnce returns the first running controller-manager +// pod name. We resolve fresh each call rather than caching because the +// pod can be evicted/restarted between specs. +func controllerPodNameOnce() (string, error) { + cmd := exec.Command("kubectl", "get", "pods", + "-n", "lmcache-operator-system", + "-l", "control-plane=controller-manager", + "-o", "jsonpath={.items[0].metadata.name}", + ) + out, err := cmd.Output() + if err != nil { + return "", err + } + if len(out) == 0 { + return "", fmt.Errorf("no controller-manager pod found") + } + return string(out), nil +} + +// recordOnFailure runs the standard dump bundle iff the current spec +// failed. Use as the body of an AfterEach so passing specs stay quiet. +func recordOnFailure(ns string) { + if !CurrentSpecReport().Failed() { + return + } + dumpControllerLogs(200) + dumpNamespace(ns) +} + +// engineKey is shorthand for the typed namespaced name pair used throughout +// the smoke specs. +func engineKey(ns, name string) client.ObjectKey { + return client.ObjectKey{Namespace: ns, Name: name} +} + +// containerArgs returns the args of the first (and only) container in +// the DaemonSet pod template — the smoke contract requires exactly one. +func containerArgs(ds *appsv1.DaemonSet) []string { + if len(ds.Spec.Template.Spec.Containers) == 0 { + return nil + } + return ds.Spec.Template.Spec.Containers[0].Args +} + +// argValue returns the value following the first occurrence of flag in +// args, or "" if flag is not present or has no value. Args are flat +// ["--flag", "val", ...] pairs emitted by BuildContainerArgs. +func argValue(args []string, flag string) string { + for i := range args { + if args[i] == flag && i+1 < len(args) { + return args[i+1] + } + } + return "" +} + +// argValueLast returns the value following the LAST occurrence of flag. +// argparse-style flag handling lets a later --flag override an earlier +// one, so the operator's contract for spec.extraArgs is "appended last, +// can override any auto-generated flag." This helper mirrors that +// runtime semantics from the operator's perspective: which value would +// the server actually use? +func argValueLast(args []string, flag string) string { + last := "" + for i := range args { + if args[i] == flag && i+1 < len(args) { + last = args[i+1] + } + } + return last +} + +// serviceMonitorCRDInstalled reports whether the Prometheus Operator's +// ServiceMonitor CRD is registered on the target cluster. Specs that +// only make sense in that environment (the ServiceMonitor spec) call this in +// BeforeEach and Skip() when it returns false, so the same suite can +// run unchanged on bare Kind clusters and on clusters with the +// kube-prometheus-stack pre-installed. Errors other than NoMatch +// fail loudly so a transient API hiccup is never silently masked. +func serviceMonitorCRDInstalled() bool { + GinkgoHelper() + _, err := k8sClient.RESTMapper().RESTMapping(schema.GroupKind{ + Group: "monitoring.coreos.com", + Kind: "ServiceMonitor", + }, "v1") + if err == nil { + return true + } + if meta.IsNoMatchError(err) { + return false + } + Fail(fmt.Sprintf("unexpected error querying ServiceMonitor RESTMapping: %v", err)) + return false // unreachable +} diff --git a/operator/test/e2e/vllm_integration_smoke_test.go b/operator/test/e2e/vllm_integration_smoke_test.go new file mode 100644 index 00000000000..6c379511668 --- /dev/null +++ b/operator/test/e2e/vllm_integration_smoke_test.go @@ -0,0 +1,293 @@ +//go:build e2e && e2e_gpu +// +build e2e,e2e_gpu + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "os/exec" + "regexp" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/yaml" + + "github.com/LMCache/LMCache/test/utils" +) + +// vLLM + LMCache round-trip (e2e_gpu): verifies that an external vLLM +// pod, when pointed at the operator-managed LMCacheEngine via the +// -connection ConfigMap and run with vLLM's APC OFF, actually +// stores KV on request 1 and retrieves it on request 2. +// +// The assertion grep's the LMCache server's stdout for its own +// "Stored N tokens" / "Retrieved N tokens" log lines (server.py logs +// these at INFO level on every store and retrieve). This is more +// direct than scraping Prometheus counters — those are emitted by +// the vLLM-side connector and depend on OTel export cadence; the +// server log is written synchronously and is the same signal an +// on-call would look at when debugging cache behaviour by hand. +// +// Knobs: +// - VLLM_MODEL Hugging Face model id. Default: Qwen/Qwen2.5-0.5B +// (small enough to load on a 16GB GPU, big enough that one chunk +// covers ≥256 tokens of the test prompt). Only the model config +// and tokenizer are downloaded; weights are dummy (see fixture). +// - VLLM_IMAGE container image. Default: lmcache/vllm-openai:latest +// (same image the operator pins for the LMCache DaemonSet, which +// means a fresh node only pulls once). +// - SKIP_VLLM_INTEGRATION when set to "true", skip this spec. +// Useful when the cluster has GPUs but no internet egress to HF +// even for config/tokenizer files. +var _ = Describe("vLLM + LMCacheEngine integration smoke (GPU)", Ordered, func() { + var ( + ctx context.Context + nsName string + ) + + BeforeEach(func() { + if os.Getenv("SKIP_VLLM_INTEGRATION") == "true" { + Skip("SKIP_VLLM_INTEGRATION=true; skipping vLLM round-trip spec") + } + ctx = context.Background() + nsName = createTestNamespace(ctx) + }) + + AfterEach(func() { + recordOnFailure(nsName) + // vLLM Deployment is owned by the spec, not by the LMCacheEngine, + // so namespace deletion (via DeferCleanup in createTestNamespace) + // is what cleans it up. No explicit teardown needed here. + }) + + It("stores KV on first request and retrieves it on a second identical request", func() { + model := envDefault("VLLM_MODEL", "Qwen/Qwen2.5-0.5B") + vllmImage := envDefault("VLLM_IMAGE", "lmcache/vllm-openai:latest") + + By("applying the runtime LMCacheEngine") + lmc, err := utils.NewLMCFromFixture("lmc_runtime.yaml", nsName, "vllm-integration") + Expect(err).NotTo(HaveOccurred()) + Expect(utils.ApplyLMC(ctx, k8sClient, lmc)).To(Succeed()) + + key := engineKey(nsName, lmc.Name) + Expect(utils.WaitLMCReconciled(ctx, k8sClient, key, 60*time.Second)).To(Succeed()) + + By("waiting for the LMCache DaemonSet pod to become Ready") + lmcPodName, err := utils.WaitDaemonSetPodReady(ctx, k8sClient, key, 8*time.Minute) + Expect(err).NotTo(HaveOccurred(), "LMCache pod did not become Ready") + + By("applying the vLLM Deployment, parameterised against this engine") + vllmRaw, err := utils.LoadFixture("vllm_deployment.yaml") + Expect(err).NotTo(HaveOccurred()) + vllmYAML := substituteVLLMPlaceholders(string(vllmRaw), nsName, lmc.Name, model, vllmImage) + vllmDeploy := &appsv1.Deployment{} + Expect(yaml.Unmarshal([]byte(vllmYAML), vllmDeploy)).To(Succeed()) + Expect(k8sClient.Create(ctx, vllmDeploy)).To(Succeed()) + + By("waiting for the vLLM Deployment to become Available (model load + connector handshake)") + // 15 min — covers cold-image pull (~10GB), HF model download, + // and vLLM startup. The readinessProbe gates on /v1/models so + // the deployment is only Available once vLLM has fully loaded. + Expect(utils.WaitDeploymentAvailable( + ctx, k8sClient, + types.NamespacedName{Namespace: nsName, Name: vllmDeploy.Name}, + 15*time.Minute, + )).To(Succeed(), "vLLM did not become Available — check pod events / logs") + + By("port-forwarding to vLLM service for completion requests") + vllmCloser, vllmBaseURL, err := utils.PortForward( + utils.PortForwardSpec{Namespace: nsName, Target: "deployment/" + vllmDeploy.Name}, + "0:8000", + ) + Expect(err).NotTo(HaveOccurred()) + defer vllmCloser() + Expect(utils.WaitHTTP200(ctx, vllmBaseURL+"/v1/models", 60*time.Second)).To(Succeed()) + + // Build a prompt long enough to cross at least one LMCache + // chunk boundary (chunkSize=128 tokens for the runtime + // fixture). The repeated paragraph generates well over a + // thousand tokens so the round-trip stores at least 8 chunks. + prompt := buildLongPrompt() + + By("sending the first /v1/completions request (cold — LMCache should STORE)") + Expect(postCompletion(ctx, vllmBaseURL, model, prompt)).To(Succeed()) + + // Server-side flushes are synchronous up to the log line, but + // kubelet's log streaming buffers a beat behind. A couple of + // seconds is enough to absorb that without padding the suite. + By("waiting 2s for the LMCache server log to flush") + time.Sleep(2 * time.Second) + + By("verifying LMCache logged a Stored line after the first request") + Eventually(func() int { + return countLMCacheStored(ctx, nsName, lmcPodName) + }, 30*time.Second, 2*time.Second).Should(BeNumerically(">=", 1), + "LMCache did not log any 'Stored N tokens' line after request 1") + storedBefore := countLMCacheStored(ctx, nsName, lmcPodName) + retrievedBefore := countLMCacheRetrieved(ctx, nsName, lmcPodName) + Expect(retrievedBefore).To(Equal(0), + "LMCache logged a 'Retrieved' before any cache hit was possible (got %d)", retrievedBefore) + + By("sending the second /v1/completions request (same prompt — LMCache should HIT)") + Expect(postCompletion(ctx, vllmBaseURL, model, prompt)).To(Succeed()) + + By("verifying LMCache logged a Retrieved line after the second request") + // Eventually: log buffering between server.py and kubectl logs + // usually settles in <2s, but the storage controller is async + // in its accounting so we poll up to 30s. Hitting the floor of + // "retrieved-after >= 1 AND stored-after unchanged" proves the + // second request rode the cache rather than re-storing. + Eventually(func(g Gomega) { + retrievedAfter := countLMCacheRetrieved(ctx, nsName, lmcPodName) + storedAfter := countLMCacheStored(ctx, nsName, lmcPodName) + g.Expect(retrievedAfter).To(BeNumerically(">=", 1), + "LMCache did not log any 'Retrieved' line after request 2 (got %d)", + retrievedAfter) + g.Expect(storedAfter).To(Equal(storedBefore), + "LMCache logged additional 'Stored' lines on the repeat request "+ + "(before=%d, after=%d) — the cache hit didn't short-circuit the store path", + storedBefore, storedAfter) + }, 30*time.Second, 2*time.Second).Should(Succeed()) + + _, _ = fmt.Fprintf(GinkgoWriter, + "cache round-trip assertion satisfied (stored=%d, retrieved=%d)\n", + countLMCacheStored(ctx, nsName, lmcPodName), + countLMCacheRetrieved(ctx, nsName, lmcPodName), + ) + }) +}) + +// substituteVLLMPlaceholders renders the vllm Deployment fixture for a +// concrete run. Kept as a tiny function so the template stays readable +// and so we don't pull in a heavy templating dependency for four substitutions. +func substituteVLLMPlaceholders(yamlText, ns, engineName, model, image string) string { + r := strings.NewReplacer( + "__NAMESPACE__", ns, + "__ENGINE_NAME__", engineName, + "__MODEL__", model, + "__VLLM_IMAGE__", image, + ) + return r.Replace(yamlText) +} + +// completionRequest is the minimal /v1/completions payload — we don't +// need streaming, sampling, or beam parameters since we're measuring +// cache behaviour, not generation quality. +type completionRequest struct { + Model string `json:"model"` + Prompt string `json:"prompt"` + MaxTokens int `json:"max_tokens"` + Temperature float64 `json:"temperature"` +} + +// postCompletion sends a single /v1/completions request and discards +// the result. Failures (non-2xx, network errors, JSON parse errors) +// propagate to the spec so the assertion failure shows the underlying +// HTTP context, not just "cache miss." +func postCompletion(ctx context.Context, baseURL, model, prompt string) error { + body := completionRequest{ + Model: model, + Prompt: prompt, + MaxTokens: 32, + Temperature: 0, + } + // vllm inference can be slow on first request (kernel cache cold), + // so the request gets its own longer timeout than the default + // httpJSONClient. We use HTTPPostJSON with a derived context. + reqCtx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + return utils.HTTPPostJSON(reqCtx, baseURL+"/v1/completions", body, nil) +} + +// lmcacheStoredLine matches the LMCache MP server's stdout INFO line +// emitted on every successful STORE, e.g. +// +// [2026-05-13 23:21:02,989] LMCache INFO: Stored 1152 tokens in 0.003 seconds +// +// The leading bracketed timestamp + log-level prefix may be coloured +// with ANSI escapes in TTY mode; we don't bother stripping those +// because `kubectl logs` returns the raw line and the regex only +// requires the literal "Stored " substring. +var lmcacheStoredLine = regexp.MustCompile(`LMCache INFO:.*\bStored \d+ tokens\b`) + +// lmcacheRetrievedLine mirrors lmcacheStoredLine for the RETRIEVE path, +// emitted from server.py:531 on every cache hit. +var lmcacheRetrievedLine = regexp.MustCompile(`LMCache INFO:.*\bRetrieved \d+ tokens\b`) + +// countLMCacheStored runs `kubectl logs` on the LMCache MP server pod +// and returns the number of matching "Stored N tokens" lines seen so +// far. Returns 0 on any kubectl error (caller is expected to wrap the +// call in Eventually, which already retries on transient failures). +func countLMCacheStored(ctx context.Context, ns, podName string) int { + return countLogLines(ctx, ns, podName, lmcacheStoredLine) +} + +// countLMCacheRetrieved is the RETRIEVE counterpart of countLMCacheStored. +func countLMCacheRetrieved(ctx context.Context, ns, podName string) int { + return countLogLines(ctx, ns, podName, lmcacheRetrievedLine) +} + +// countLogLines is the implementation behind countLMCacheStored / +// countLMCacheRetrieved. We shell out to kubectl rather than building +// a typed Pods/log client because the rest of the suite already +// depends on kubectl being on PATH for port-forward, and adding a +// streaming-log client just for this would be heavier than the +// problem warrants. +func countLogLines(ctx context.Context, ns, podName string, pattern *regexp.Regexp) int { + cmd := exec.CommandContext(ctx, "kubectl", "logs", + "-n", ns, podName, + "--tail=10000", // generous: a few k tokens of LMCache INFO output + ) + out, err := cmd.Output() + if err != nil { + return 0 + } + return len(pattern.FindAllIndex(out, -1)) +} + +// buildLongPrompt assembles ~1.2k tokens of prose so the prompt spans +// at least 8 chunks at chunkSize=128. We use a repeated history +// passage rather than random tokens so the prompt is human-readable +// in dump output (helps debugging when the test fails) and so vLLM's +// tokeniser segments it consistently across runs. +func buildLongPrompt() string { + const para = "The history and significance of the Roman empire spans more than a thousand years " + + "and profoundly shaped Western civilization. Its legal, architectural, linguistic, and " + + "political legacies persist to this day, influencing modern governments, languages, art, " + + "engineering, and law. The empire's trajectory from the founding of Rome through the " + + "Republic, the transition to the Principate under Augustus, the Pax Romana, the crisis of " + + "the third century, the Dominate under Diocletian, the adoption of Christianity under " + + "Constantine, the splitting into Western and Eastern halves, and the eventual collapse of " + + "the West is one of history's great narratives. Key figures include Julius Caesar, " + + "Augustus, Marcus Aurelius, Diocletian, Constantine, Justinian, and many others. " + var b strings.Builder + for range 8 { + b.WriteString(para) + } + b.WriteString("Tell me a long, detailed story about the rise, peak, and eventual fall of Rome, " + + "naming important figures and events.") + return b.String() +} diff --git a/operator/test/utils/fixtures.go b/operator/test/utils/fixtures.go new file mode 100644 index 00000000000..8f4e53d273a --- /dev/null +++ b/operator/test/utils/fixtures.go @@ -0,0 +1,82 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "embed" + "fmt" + "path" + + "sigs.k8s.io/yaml" + + lmcachev1alpha1 "github.com/LMCache/LMCache/api/v1alpha1" +) + +// fixturesFS embeds the fixtures tree shipped with the smoke harness: +// CR YAMLs at fixtures/*.yaml and golden JSON snapshots at +// fixtures/golden/*.json. Embedding sidesteps the working-directory +// hack in utils.GetProjectDir: fixtures are compiled into the test +// binary and resolved by relative path. +// +//go:embed fixtures/*.yaml fixtures/golden/*.json +var fixturesFS embed.FS + +// LoadFixture returns the raw bytes of a fixture file by name (no path). +func LoadFixture(name string) ([]byte, error) { + data, err := fixturesFS.ReadFile(path.Join("fixtures", name)) + if err != nil { + return nil, fmt.Errorf("load fixture %q: %w", name, err) + } + return data, nil +} + +// LoadGolden returns the raw bytes of a golden snapshot by name (no path). +// Golden files live under fixtures/golden/ and represent expected JSON +// payloads emitted by the operator. Tests typically substitute namespace +// placeholders into the result before comparing. +func LoadGolden(name string) ([]byte, error) { + data, err := fixturesFS.ReadFile(path.Join("fixtures", "golden", name)) + if err != nil { + return nil, fmt.Errorf("load golden %q: %w", name, err) + } + return data, nil +} + +// NewLMCFromFixture loads a fixture YAML, decodes it as an LMCacheEngine, +// and overrides metadata.name and metadata.namespace with the supplied +// values. Callers can mutate the returned object further before applying. +func NewLMCFromFixture(fixtureName, namespace, name string) (*lmcachev1alpha1.LMCacheEngine, error) { + data, err := LoadFixture(fixtureName) + if err != nil { + return nil, err + } + lmc := &lmcachev1alpha1.LMCacheEngine{} + if err := yaml.Unmarshal(data, lmc); err != nil { + return nil, fmt.Errorf("decode fixture %q: %w", fixtureName, err) + } + if name != "" { + lmc.Name = name + } + if namespace != "" { + lmc.Namespace = namespace + } + // Strip any server-side metadata that may have leaked through. + lmc.ResourceVersion = "" + lmc.UID = "" + lmc.Generation = 0 + return lmc, nil +} diff --git a/operator/test/utils/fixtures/golden/kv_transfer_config_minimal.json b/operator/test/utils/fixtures/golden/kv_transfer_config_minimal.json new file mode 100644 index 00000000000..5f74c8e6eee --- /dev/null +++ b/operator/test/utils/fixtures/golden/kv_transfer_config_minimal.json @@ -0,0 +1,8 @@ +{ + "kv_connector": "LMCacheMPConnector", + "kv_connector_extra_config": { + "lmcache.mp.host": "tcp://smoke-minimal.__NAMESPACE__.svc.cluster.local", + "lmcache.mp.port": "5555" + }, + "kv_role": "kv_both" +} \ No newline at end of file diff --git a/operator/test/utils/fixtures/lmc_custom_port.yaml b/operator/test/utils/fixtures/lmc_custom_port.yaml new file mode 100644 index 00000000000..b349f109c35 --- /dev/null +++ b/operator/test/utils/fixtures/lmc_custom_port.yaml @@ -0,0 +1,18 @@ +# Custom-port LMCacheEngine fixture for the custom-port smoke spec. +# +# Exercises the non-default code paths in BuildContainerArgs and +# BuildLookupService: the controller must propagate spec.server.port +# (6555) and spec.server.chunkSize (128) into both the DaemonSet +# container args and the lookup Service port. The golden assertion in +# crd_smoke_test.go pairs this fixture with the expected ConfigMap. +apiVersion: lmcache.lmcache.ai/v1alpha1 +kind: LMCacheEngine +metadata: + name: smoke-custom-port +spec: + nodeSelector: {} + server: + port: 6555 + chunkSize: 128 + l1: + sizeGB: 10 diff --git a/operator/test/utils/fixtures/lmc_minimal.yaml b/operator/test/utils/fixtures/lmc_minimal.yaml new file mode 100644 index 00000000000..97295e2eb4b --- /dev/null +++ b/operator/test/utils/fixtures/lmc_minimal.yaml @@ -0,0 +1,19 @@ +# Minimal LMCacheEngine fixture used by the smoke test harness. +# +# The fixture intentionally sets nodeSelector to {} (empty map) so the +# operator's SetDefaults does NOT fall back to nvidia.com/gpu.present. +# That keeps the DaemonSet schedulable on a no-GPU Kind cluster, which +# is required for the no-GPU smoke tier. The pods will not become Ready +# (no nvidia runtime / image), but the operator will still reconcile +# every K8s artifact it owns — which is what the smokes assert on. +# +# Tests that need a deterministic, non-default port should use +# lmc_custom_port.yaml or PatchLMCSpec instead. +apiVersion: lmcache.lmcache.ai/v1alpha1 +kind: LMCacheEngine +metadata: + name: smoke-minimal +spec: + nodeSelector: {} + l1: + sizeGB: 10 diff --git a/operator/test/utils/fixtures/lmc_runtime.yaml b/operator/test/utils/fixtures/lmc_runtime.yaml new file mode 100644 index 00000000000..aac797987ec --- /dev/null +++ b/operator/test/utils/fixtures/lmc_runtime.yaml @@ -0,0 +1,22 @@ +# Runtime fixture used by the e2e_gpu smoke tier. +# +# This fixture deliberately OMITS spec.nodeSelector so the operator's +# SetDefaults adds nvidia.com/gpu.present="true" and the DaemonSet pod +# lands only on GPU nodes. (Compare with lmc_minimal.yaml which sets +# nodeSelector: {} to allow scheduling on no-GPU Kind for the M1 tier.) +# +# Custom server.port + server.chunkSize values are non-defaults so the +# /conf round-trip spec can prove "what the CR said" was actually +# applied to the running LMCache server — not just to the K8s objects. +apiVersion: lmcache.lmcache.ai/v1alpha1 +kind: LMCacheEngine +metadata: + name: smoke-runtime +spec: + l1: + sizeGB: 10 + server: + port: 6555 + chunkSize: 128 + maxWorkers: 2 + hashAlgorithm: blake3 diff --git a/operator/test/utils/fixtures/lmc_servicemonitor.yaml b/operator/test/utils/fixtures/lmc_servicemonitor.yaml new file mode 100644 index 00000000000..798267e77d9 --- /dev/null +++ b/operator/test/utils/fixtures/lmc_servicemonitor.yaml @@ -0,0 +1,20 @@ +# ServiceMonitor smoke fixture. +# +# Enabling prometheus.serviceMonitor.enabled is the only way the operator +# creates a monitoring.coreos.com/v1 ServiceMonitor. We pin a release +# label here so the test can assert it round-trips into the CR. +apiVersion: lmcache.lmcache.ai/v1alpha1 +kind: LMCacheEngine +metadata: + name: smoke-servicemonitor +spec: + nodeSelector: {} + l1: + sizeGB: 10 + prometheus: + enabled: true + serviceMonitor: + enabled: true + interval: 30s + labels: + release: kube-prometheus-stack diff --git a/operator/test/utils/fixtures/lmc_with_redis_l2_authsecret.yaml b/operator/test/utils/fixtures/lmc_with_redis_l2_authsecret.yaml new file mode 100644 index 00000000000..1f434a97d85 --- /dev/null +++ b/operator/test/utils/fixtures/lmc_with_redis_l2_authsecret.yaml @@ -0,0 +1,22 @@ +# Cross-namespace authSecretRef fixture. +# +# The authSecretRef.namespace is intentionally a placeholder — the test +# overrides it with a per-spec source namespace at runtime so multiple +# specs can run in parallel without colliding on a shared "redis-src". +# The host/port don't need to be reachable; the operator only inspects +# the auth machinery for this smoke. +apiVersion: lmcache.lmcache.ai/v1alpha1 +kind: LMCacheEngine +metadata: + name: smoke-auth +spec: + nodeSelector: {} + l1: + sizeGB: 10 + l2Backend: + resp: + host: redis.example.svc.cluster.local + port: 6379 + authSecretRef: + name: redis-auth + namespace: __OVERRIDE_ME__ diff --git a/operator/test/utils/fixtures/vllm_deployment.yaml b/operator/test/utils/fixtures/vllm_deployment.yaml new file mode 100644 index 00000000000..aa9eab5765a --- /dev/null +++ b/operator/test/utils/fixtures/vllm_deployment.yaml @@ -0,0 +1,122 @@ +# vLLM Deployment used by the e2e_gpu integration spec. +# +# Renders a single-replica vLLM serve, configured to reach the LMCache +# DaemonSet via the operator-produced -connection ConfigMap. +# Placeholders are substituted by the spec at apply time so the test +# can target whichever namespace/engine/model the run was parameterised +# with: +# +# __NAMESPACE__ — engine namespace +# __ENGINE_NAME__ — LMCacheEngine metadata.name (also Service / ConfigMap name) +# __MODEL__ — Hugging Face model id (e.g., Qwen/Qwen2.5-0.5B) +# __VLLM_IMAGE__ — container image (defaults to lmcache/vllm-openai:latest, +# the bundled vLLM + LMCache build) +# +# Requirements baked in (matching the operator's LMCache pod template +# and the constraints of cudaIpcOpenMemHandle-based KV handoff): +# - hostIPC: true — share /dev/shm with the LMCache pod +# - runtimeClassName: nvidia — same as LMCache so they co-schedule +# on GPU nodes +# - nodeSelector: gpu.present — pin to a GPU node so the service's +# internalTrafficPolicy=Local routes +# to the on-node LMCache pod +# - nvidia.com/gpu: 1 — consume one GPU device +# - --no-enable-prefix-caching — REQUIRED: APC must be off so prompt +# re-runs miss vLLM's local KV and +# fall through to LMCache lookup +# - --kv-transfer-config — inlined from the operator-produced +# ConfigMap by the entrypoint wrapper +# (vllm serve takes a JSON string, +# not a path) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vllm-smoke + namespace: __NAMESPACE__ + labels: + app: vllm-smoke +spec: + replicas: 1 + selector: + matchLabels: + app: vllm-smoke + template: + metadata: + labels: + app: vllm-smoke + spec: + hostIPC: true + runtimeClassName: nvidia + nodeSelector: + nvidia.com/gpu.present: "true" + containers: + - name: vllm + image: __VLLM_IMAGE__ + imagePullPolicy: IfNotPresent + # vllm serve --kv-transfer-config takes the JSON connector + # config as a single argv token (string), so we read the + # mounted ConfigMap file inline via `cat`. Going through a + # shell wrapper keeps the JSON source of truth in the + # operator-produced ConfigMap rather than duplicated here. + # + # --load-format dummy: skip downloading the model weights + # from Hugging Face. vLLM still fetches the model's metadata + # (config.json, tokenizer.json) but generates random weights + # in-process. The test asserts on KV-cache hit counters, not + # on generation quality, so dummy weights are fine and we + # save several GB of download + ~5 min of test runtime. + command: ["/bin/sh", "-c"] + args: + - >- + exec vllm serve "__MODEL__" + --port 8000 + --no-enable-prefix-caching + --enforce-eager + --load-format dummy + --gpu-memory-utilization 0.5 + --kv-transfer-config "$(cat /etc/lmcache/kv-transfer-config.json)" + env: + # MP transport requires single-process scheduler so the + # connector lives in the same proc as the engine core. + - name: VLLM_ENABLE_V1_MULTIPROCESSING + value: "0" + # Determinism: same prompt -> same chunk hashes across runs + # so the cache lookup on the second request actually hits. + - name: PYTHONHASHSEED + value: "0" + - name: NVIDIA_VISIBLE_DEVICES + value: "all" + - name: NVIDIA_DRIVER_CAPABILITIES + value: "all" + ports: + - name: http + containerPort: 8000 + protocol: TCP + resources: + limits: + nvidia.com/gpu: 1 + readinessProbe: + httpGet: + path: /v1/models + port: 8000 + # /v1/models is ready only after the model is loaded onto + # the GPU — a strong signal that the LMCache connector + # also negotiated successfully (otherwise vllm would have + # crashed during init). + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 60 + volumeMounts: + - name: lmcache-connection + mountPath: /etc/lmcache + readOnly: true + volumes: + - name: lmcache-connection + configMap: + # __ENGINE_NAME__-connection is owned by the LMCacheEngine + # so its lifecycle is tied to the CR — the test asserts on + # GC of this same ConfigMap in lifecycle_smoke_test.go. + name: __ENGINE_NAME__-connection + items: + - key: kv-transfer-config.json + path: kv-transfer-config.json diff --git a/operator/test/utils/http.go b/operator/test/utils/http.go new file mode 100644 index 00000000000..53c589f8aac --- /dev/null +++ b/operator/test/utils/http.go @@ -0,0 +1,157 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "k8s.io/apimachinery/pkg/util/wait" +) + +// httpJSONClient is the shared client used by HTTP helpers. We do NOT +// set Client.Timeout because that cap would override the caller's +// context deadline — and callers like the vLLM-integration spec need +// minutes-long deadlines for cold-cache completions. Every helper here +// uses NewRequestWithContext, so cancellation is governed entirely by +// the caller's ctx. +// +// DisableKeepAlives is on purpose: every test request goes through a +// `kubectl port-forward` proxy that quietly drops idle TCP connections +// after a few seconds. With keep-alive on, the second HTTP call in a +// spec ends up trying to reuse a dead connection and gets EOF before +// the request even reaches the upstream. Re-establishing the TCP +// connection per request costs a few ms in tests; the alternative is +// flaky failures that look like upstream crashes when they aren't. +var httpJSONClient = &http.Client{ + Transport: &http.Transport{DisableKeepAlives: true}, +} + +// HTTPGetJSON issues a GET against url and decodes the response body +// into out. Returns an error if the request fails, the status is not +// 2xx, or the body is not valid JSON. The caller owns out's lifetime. +func HTTPGetJSON(ctx context.Context, url string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("build GET %s: %w", url, err) + } + resp, err := httpJSONClient.Do(req) + if err != nil { + return fmt.Errorf("GET %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read GET %s body: %w", url, err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("GET %s: status %d body=%s", url, resp.StatusCode, string(body)) + } + if out == nil { + return nil + } + if err := json.Unmarshal(body, out); err != nil { + return fmt.Errorf("decode JSON from GET %s: %w\nbody=%s", url, err, string(body)) + } + return nil +} + +// HTTPGetText issues a GET against url and returns the raw response +// body as a string. Used for endpoints that don't speak JSON +// (Prometheus text exposition at /metrics). +func HTTPGetText(ctx context.Context, url string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", fmt.Errorf("build GET %s: %w", url, err) + } + resp, err := httpJSONClient.Do(req) + if err != nil { + return "", fmt.Errorf("GET %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("read GET %s body: %w", url, err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return "", fmt.Errorf("GET %s: status %d body=%s", url, resp.StatusCode, string(body)) + } + return string(body), nil +} + +// HTTPPostJSON issues a POST with a JSON-encoded body and decodes the +// 2xx response body into out (when non-nil). Returns the parsed JSON +// status code so callers that care about distinguishing 200 vs 201 can +// check it themselves. +func HTTPPostJSON(ctx context.Context, url string, body, out any) error { + buf, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("encode POST %s body: %w", url, err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(buf)) + if err != nil { + return fmt.Errorf("build POST %s: %w", url, err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := httpJSONClient.Do(req) + if err != nil { + return fmt.Errorf("POST %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read POST %s body: %w", url, err) + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("POST %s: status %d body=%s", url, resp.StatusCode, string(respBody)) + } + if out == nil { + return nil + } + if err := json.Unmarshal(respBody, out); err != nil { + return fmt.Errorf("decode JSON from POST %s: %w\nbody=%s", url, err, string(respBody)) + } + return nil +} + +// WaitHTTP200 polls url with GET until it returns 2xx or the timeout +// elapses. Tighter than HTTPGetJSON because it tolerates transient +// connection-refused / status-503 noise during pod startup: the LMCache +// HTTP frontend is reachable on the readiness-probed port (TCP-readiness +// just confirms the ZMQ socket is bound), and the FastAPI app may not +// have finished initializing yet. The poll absorbs that race so the +// caller's assertion runs against a fully-ready server. +func WaitHTTP200(ctx context.Context, url string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return false, err + } + resp, err := httpJSONClient.Do(req) + if err != nil { + return false, nil + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + return resp.StatusCode >= 200 && resp.StatusCode < 300, nil + }) +} diff --git a/operator/test/utils/lmc.go b/operator/test/utils/lmc.go new file mode 100644 index 00000000000..5a070368d02 --- /dev/null +++ b/operator/test/utils/lmc.go @@ -0,0 +1,209 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "encoding/json" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + + lmcachev1alpha1 "github.com/LMCache/LMCache/api/v1alpha1" +) + +// KvTransferConfig is the typed schema of the kv-transfer-config.json +// document produced by the operator into the -connection ConfigMap. +// Field tags mirror the JSON keys that vLLM's --kv-transfer-config expects. +type KvTransferConfig struct { + KVConnector string `json:"kv_connector"` + KVRole string `json:"kv_role"` + KVConnectorExtraConfig KvConnectorExtraConfig `json:"kv_connector_extra_config"` +} + +// KvConnectorExtraConfig holds the LMCache MP-mode connection string the +// vLLM connector uses to reach the LMCache server. The keys contain dots +// — vLLM treats them as flat strings, not nested paths. +type KvConnectorExtraConfig struct { + Host string `json:"lmcache.mp.host"` + Port string `json:"lmcache.mp.port"` +} + +// ApplyLMC creates the LMCacheEngine if absent, or patches it (spec only) +// if it already exists. The status subresource is left untouched. +func ApplyLMC(ctx context.Context, c client.Client, lmc *lmcachev1alpha1.LMCacheEngine) error { + existing := &lmcachev1alpha1.LMCacheEngine{} + err := c.Get(ctx, client.ObjectKeyFromObject(lmc), existing) + if apierrors.IsNotFound(err) { + return c.Create(ctx, lmc) + } + if err != nil { + return fmt.Errorf("get LMCacheEngine %s/%s: %w", lmc.Namespace, lmc.Name, err) + } + patch := client.MergeFrom(existing.DeepCopy()) + existing.Spec = lmc.Spec + existing.Labels = lmc.Labels + existing.Annotations = lmc.Annotations + if err := c.Patch(ctx, existing, patch); err != nil { + return fmt.Errorf("patch LMCacheEngine %s/%s: %w", lmc.Namespace, lmc.Name, err) + } + *lmc = *existing + return nil +} + +// WaitLMCReconciled polls until status.observedGeneration matches the +// CR's metadata.generation AND the ConfigValid condition is True. This +// is the readiness signal for the no-GPU smoke tier — pods do not need +// to be Running for the operator to have produced its K8s artifacts. +func WaitLMCReconciled(ctx context.Context, c client.Client, key types.NamespacedName, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { + lmc := &lmcachev1alpha1.LMCacheEngine{} + if err := c.Get(ctx, key, lmc); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + if lmc.Generation == 0 || lmc.Status.ObservedGeneration != lmc.Generation { + return false, nil + } + cond := meta.FindStatusCondition(lmc.Status.Conditions, lmcachev1alpha1.ConditionConfigValid) + return cond != nil && cond.Status == "True", nil + }) +} + +// WaitLMCPhase polls until status.phase equals the requested phase value +// (e.g. "Running", "Pending"). Use WaitLMCReconciled instead when the +// observation contract is "controller saw the spec," not "pods are up." +func WaitLMCPhase( + ctx context.Context, + c client.Client, + key types.NamespacedName, + phase string, + timeout time.Duration, +) error { + return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { + lmc := &lmcachev1alpha1.LMCacheEngine{} + if err := c.Get(ctx, key, lmc); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return lmc.Status.Phase == phase, nil + }) +} + +// WaitLMCReady is a thin alias for WaitLMCPhase("Running"). It exists +// because the GPU tier (M3) reads more naturally as "wait until ready" +// — the no-GPU tier should not call this. +func WaitLMCReady(ctx context.Context, c client.Client, key types.NamespacedName, timeout time.Duration) error { + return WaitLMCPhase(ctx, c, key, lmcachev1alpha1.PhaseRunning, timeout) +} + +// GetConnectionConfig fetches the -connection ConfigMap and parses +// kv-transfer-config.json into a typed struct. Returns an error if the +// ConfigMap is missing, the data key is missing, or the JSON is invalid. +func GetConnectionConfig(ctx context.Context, c client.Client, key types.NamespacedName) (*KvTransferConfig, error) { + cm := &corev1.ConfigMap{} + cmKey := types.NamespacedName{Namespace: key.Namespace, Name: key.Name + "-connection"} + if err := c.Get(ctx, cmKey, cm); err != nil { + return nil, fmt.Errorf("get ConfigMap %s: %w", cmKey, err) + } + raw, ok := cm.Data["kv-transfer-config.json"] + if !ok { + return nil, fmt.Errorf("ConfigMap %s missing key kv-transfer-config.json", cmKey) + } + cfg := &KvTransferConfig{} + if err := json.Unmarshal([]byte(raw), cfg); err != nil { + return nil, fmt.Errorf("decode kv-transfer-config.json: %w", err) + } + return cfg, nil +} + +// PatchLMCSpec re-fetches the CR, applies mutate to the spec, and submits +// a merge patch. The mutate callback receives a pointer to the spec and +// must mutate in place. Use this for tests like the port-update spec +// that need to model "user kubectl-patches the spec." +func PatchLMCSpec( + ctx context.Context, + c client.Client, + key types.NamespacedName, + mutate func(*lmcachev1alpha1.LMCacheEngineSpec), +) error { + lmc := &lmcachev1alpha1.LMCacheEngine{} + if err := c.Get(ctx, key, lmc); err != nil { + return fmt.Errorf("get LMCacheEngine %s: %w", key, err) + } + patch := client.MergeFrom(lmc.DeepCopy()) + mutate(&lmc.Spec) + return c.Patch(ctx, lmc, patch) +} + +// DeleteLMCAndWaitGC issues a foreground delete on the CR and waits until +// both the CR and its primary owned resources (DaemonSet, lookup Service, +// connection ConfigMap) are gone. Times out if a finalizer is stuck. +func DeleteLMCAndWaitGC(ctx context.Context, c client.Client, key types.NamespacedName, timeout time.Duration) error { + lmc := &lmcachev1alpha1.LMCacheEngine{} + if err := c.Get(ctx, key, lmc); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("get LMCacheEngine %s: %w", key, err) + } + if err := c.Delete(ctx, lmc); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete LMCacheEngine %s: %w", key, err) + } + + cmKey := types.NamespacedName{Namespace: key.Namespace, Name: key.Name + "-connection"} + return wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { + if exists, err := objectExists(ctx, c, key, &lmcachev1alpha1.LMCacheEngine{}); err != nil || exists { + return false, err + } + if exists, err := objectExists(ctx, c, key, &appsv1.DaemonSet{}); err != nil || exists { + return false, err + } + if exists, err := objectExists(ctx, c, key, &corev1.Service{}); err != nil || exists { + return false, err + } + if exists, err := objectExists(ctx, c, cmKey, &corev1.ConfigMap{}); err != nil || exists { + return false, err + } + return true, nil + }) +} + +// objectExists returns true if the object at key still exists in the API. +// It treats NotFound as the only "absent" signal; any other error is +// surfaced so the caller can distinguish transient API failures. +func objectExists(ctx context.Context, c client.Client, key types.NamespacedName, obj client.Object) (bool, error) { + err := c.Get(ctx, key, obj) + if err == nil { + return true, nil + } + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err +} diff --git a/operator/test/utils/portforward.go b/operator/test/utils/portforward.go new file mode 100644 index 00000000000..184cc328e91 --- /dev/null +++ b/operator/test/utils/portforward.go @@ -0,0 +1,173 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "net" + "os/exec" + "strconv" + "strings" + "time" +) + +// PortForwardSpec identifies what kubectl port-forward should target. +// Target follows the kubectl convention (e.g. "svc/my-cache", +// "pod/my-cache-abc", "deployment/my-cache"). +type PortForwardSpec struct { + Namespace string + Target string +} + +// PortForward starts a `kubectl port-forward` subprocess and waits until +// the first local port begins accepting TCP connections. Each port arg +// follows kubectl syntax: "LOCAL:REMOTE" or just "PORT" (where the local +// and remote ports are equal). A LOCAL of "0" is replaced with a +// kernel-picked free port chosen before kubectl starts, so concurrent +// specs in the same run never collide on a fixed local port. +// +// Returns: +// - closer: must be called to terminate the subprocess and free ports. +// Safe to call multiple times. +// - localBase: "http://127.0.0.1:" using the first port mapping. +// +// Namespace is passed via the spec struct rather than encoded into +// target because kubectl requires namespace as a separate -n flag +// and silently ignores prefixes embedded in the target string. +func PortForward(spec PortForwardSpec, ports ...string) (func(), string, error) { + if len(ports) == 0 { + return nil, "", fmt.Errorf("PortForward: at least one port mapping is required") + } + + // Substitute "0:REMOTE" and "0" with a concrete kernel-picked port + // in every mapping. We resolve the port BEFORE invoking kubectl + // because kubectl's "LOCAL=0 => pick one" mode writes the chosen + // port to stdout asynchronously, which is racy to scrape — and + // because the first mapping's local port is what waitForLocalPort + // + the returned localBase URL refer to. The race between Close+ + // kubectl-bind is acceptable: we don't run concurrent forwards in + // the same spec, and other processes binding ephemerals during that + // microsecond is highly unlikely. + resolved := make([]string, len(ports)) + for i, p := range ports { + r, err := resolvePortMapping(p) + if err != nil { + return nil, "", err + } + resolved[i] = r + } + localPort, err := localPortFromMapping(resolved[0]) + if err != nil { + return nil, "", err + } + + args := []string{"port-forward"} + if spec.Namespace != "" { + args = append(args, "-n", spec.Namespace) + } + args = append(args, spec.Target) + args = append(args, resolved...) + + cmd := exec.Command("kubectl", args...) + if err := cmd.Start(); err != nil { + return nil, "", fmt.Errorf("start kubectl port-forward: %w", err) + } + + closer := func() { + // Killing the process is sufficient — kubectl port-forward + // closes the listener on SIGKILL, freeing the local port. + // We drop the Wait error because once we kill, the typical + // exit status is "signal: killed" which is expected. + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + } + + if err := waitForLocalPort(localPort, 30*time.Second); err != nil { + closer() + return nil, "", fmt.Errorf("port-forward to %s/%s did not become ready: %w", + spec.Namespace, spec.Target, err) + } + return closer, fmt.Sprintf("http://127.0.0.1:%d", localPort), nil +} + +// resolvePortMapping replaces a "0:REMOTE" mapping (or bare "0") with +// ":REMOTE" using a kernel-allocated ephemeral port. Mappings +// that already specify a concrete LOCAL pass through unchanged. +func resolvePortMapping(mapping string) (string, error) { + parts := strings.SplitN(mapping, ":", 2) + local := parts[0] + if local != "0" { + return mapping, nil + } + picked, err := pickEphemeralPort() + if err != nil { + return "", fmt.Errorf("pick ephemeral local port: %w", err) + } + if len(parts) == 1 { + // "0" alone is ambiguous — there's no remote to forward to. + return "", fmt.Errorf("invalid port mapping %q: LOCAL=0 requires an explicit :REMOTE", mapping) + } + return fmt.Sprintf("%d:%s", picked, parts[1]), nil +} + +// pickEphemeralPort asks the kernel for a free TCP port on 127.0.0.1 +// and immediately releases it. There is a small race window between +// release and kubectl re-binding; tests don't run forwards concurrently +// inside a single spec, and parallel host workloads rarely steal the +// exact port in that microsecond. +func pickEphemeralPort() (int, error) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + port := l.Addr().(*net.TCPAddr).Port + if err := l.Close(); err != nil { + return 0, err + } + return port, nil +} + +// localPortFromMapping extracts the LOCAL port from a kubectl mapping +// of the form "LOCAL:REMOTE" or just "PORT" (where the local and remote +// ports are equal). Expects LOCAL to already be a concrete integer — +// the "0" case is handled by resolvePortMapping upstream. +func localPortFromMapping(mapping string) (int, error) { + parts := strings.SplitN(mapping, ":", 2) + p, err := strconv.Atoi(parts[0]) + if err != nil { + return 0, fmt.Errorf("invalid port mapping %q: %w", mapping, err) + } + return p, nil +} + +// waitForLocalPort polls 127.0.0.1:port until a TCP connection succeeds +// or timeout elapses. kubectl port-forward briefly accepts connections +// and immediately closes them once before the upstream is wired, so we +// also require the connection to stay open long enough to write to. +func waitForLocalPort(port int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + addr := fmt.Sprintf("127.0.0.1:%d", port) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", addr, 500*time.Millisecond) + if err == nil { + _ = conn.Close() + return nil + } + time.Sleep(200 * time.Millisecond) + } + return fmt.Errorf("local port %d not reachable after %s", port, timeout) +} diff --git a/operator/test/utils/runner.go b/operator/test/utils/runner.go new file mode 100644 index 00000000000..d0898daaa9e --- /dev/null +++ b/operator/test/utils/runner.go @@ -0,0 +1,65 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +// operatorRoot resolves to the absolute path of the operator/ directory, +// derived at compile time from this file's location. Unlike GetProjectDir, +// this does not depend on the current working directory and never calls +// os.Chdir, so tests can run from any working directory. +var operatorRoot = func() string { + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + // Should never happen — runtime.Caller(0) only fails if the + // stack has fewer frames than requested. + panic("utils.operatorRoot: runtime.Caller failed") + } + // thisFile = /operator/test/utils/runner.go + return filepath.Clean(filepath.Join(filepath.Dir(thisFile), "..", "..")) +}() + +// OperatorRoot returns the absolute path of the operator/ directory. +func OperatorRoot() string { + return operatorRoot +} + +// RunFromOperator runs an exec.Cmd with cmd.Dir set to the operator/ root. +// It does not modify the process working directory. Output is captured +// and returned as a single string; on non-zero exit, the error wraps +// the combined output. +func RunFromOperator(cmd *exec.Cmd) (string, error) { + cmd.Dir = operatorRoot + out, err := cmd.CombinedOutput() + if err != nil { + return string(out), fmt.Errorf("%s failed: %w\n%s", strings.Join(cmd.Args, " "), err, string(out)) + } + return string(out), nil +} + +// RunMake runs `make ` from the operator/ directory and returns +// the combined output. It is the no-Chdir replacement for the working- +// directory hack in utils.Run. +func RunMake(args ...string) (string, error) { + return RunFromOperator(exec.Command("make", args...)) +} diff --git a/operator/test/utils/utils.go b/operator/test/utils/utils.go index ced92c74745..8346a8529f1 100644 --- a/operator/test/utils/utils.go +++ b/operator/test/utils/utils.go @@ -28,17 +28,10 @@ import ( ) const ( - certmanagerVersion = "v1.19.4" - certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" - defaultKindBinary = "kind" defaultKindCluster = "kind" ) -func warnError(err error) { - _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) -} - // Run executes the provided command within this context func Run(cmd *exec.Cmd) (string, error) { dir, _ := GetProjectDir() @@ -59,80 +52,6 @@ func Run(cmd *exec.Cmd) (string, error) { return string(output), nil } -// UninstallCertManager uninstalls the cert manager -func UninstallCertManager() { - url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { - warnError(err) - } - - // Delete leftover leases in kube-system (not cleaned by default) - kubeSystemLeases := []string{ - "cert-manager-cainjector-leader-election", - "cert-manager-controller", - } - for _, lease := range kubeSystemLeases { - cmd = exec.Command("kubectl", "delete", "lease", lease, - "-n", "kube-system", "--ignore-not-found", "--force", "--grace-period=0") - if _, err := Run(cmd); err != nil { - warnError(err) - } - } -} - -// InstallCertManager installs the cert manager bundle. -func InstallCertManager() error { - url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "apply", "-f", url) - if _, err := Run(cmd); err != nil { - return err - } - // Wait for cert-manager-webhook to be ready, which can take time if cert-manager - // was re-installed after uninstalling on a cluster. - cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", - "--for", "condition=Available", - "--namespace", "cert-manager", - "--timeout", "5m", - ) - - _, err := Run(cmd) - return err -} - -// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed -// by verifying the existence of key CRDs related to Cert Manager. -func IsCertManagerCRDsInstalled() bool { - // List of common Cert Manager CRDs - certManagerCRDs := []string{ - "certificates.cert-manager.io", - "issuers.cert-manager.io", - "clusterissuers.cert-manager.io", - "certificaterequests.cert-manager.io", - "orders.acme.cert-manager.io", - "challenges.acme.cert-manager.io", - } - - // Execute the kubectl command to get all CRDs - cmd := exec.Command("kubectl", "get", "crds") - output, err := Run(cmd) - if err != nil { - return false - } - - // Check if any of the Cert Manager CRDs are present - crdList := GetNonEmptyLines(output) - for _, crd := range certManagerCRDs { - for _, line := range crdList { - if strings.Contains(line, crd) { - return true - } - } - } - - return false -} - // LoadImageToKindClusterWithName loads a local docker image to the kind cluster func LoadImageToKindClusterWithName(name string) error { cluster := defaultKindCluster diff --git a/operator/test/utils/wait.go b/operator/test/utils/wait.go new file mode 100644 index 00000000000..863083217cc --- /dev/null +++ b/operator/test/utils/wait.go @@ -0,0 +1,113 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// WaitDaemonSetPodReady waits until at least one pod selected by the +// DaemonSet's pod selector reports a Ready=True condition. Returns the +// name of the first Ready pod so the caller can target it with +// port-forward (kubectl port-forward needs a stable pod name; selecting +// the DaemonSet directly is flaky when multiple pods exist on a +// multi-node cluster). The DaemonSet's TCP readiness probe binds the +// LMCache server, so once the pod is Ready the HTTP frontend is also +// (very nearly) up — the caller should still retry HTTP calls briefly. +func WaitDaemonSetPodReady( + ctx context.Context, + c client.Client, + key types.NamespacedName, + timeout time.Duration, +) (string, error) { + var podName string + err := wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + ds := &appsv1.DaemonSet{} + if err := c.Get(ctx, key, ds); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + pods := &corev1.PodList{} + if err := c.List(ctx, pods, + client.InNamespace(key.Namespace), + client.MatchingLabels(ds.Spec.Selector.MatchLabels), + ); err != nil { + return false, err + } + for i := range pods.Items { + p := &pods.Items[i] + if p.DeletionTimestamp != nil { + continue + } + if p.Status.Phase != corev1.PodRunning { + continue + } + for _, cond := range p.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + podName = p.Name + return true, nil + } + } + } + return false, nil + }) + if err != nil { + return "", fmt.Errorf("wait DaemonSet %s pod Ready: %w", key, err) + } + return podName, nil +} + +// WaitDeploymentAvailable polls a Deployment until its Available +// condition is True. Used to gate test logic on vLLM (or any other +// auxiliary workload) being fully scheduled and accepting traffic. +// We intentionally key off the Available condition rather than +// ReadyReplicas because Available encodes the rolling-update completion +// semantics — ReadyReplicas can flap during a fresh rollout even on a +// clean cluster. +func WaitDeploymentAvailable( + ctx context.Context, + c client.Client, + key types.NamespacedName, + timeout time.Duration, +) error { + return wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + dep := &appsv1.Deployment{} + if err := c.Get(ctx, key, dep); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + for _, cond := range dep.Status.Conditions { + if cond.Type == appsv1.DeploymentAvailable && cond.Status == corev1.ConditionTrue { + return true, nil + } + } + return false, nil + }) +} From 84667435bb508074bf2936694632d7c13451869b Mon Sep 17 00:00:00 2001 From: Jiayue Chen <96101996+glbyktjys@users.noreply.github.com> Date: Fri, 15 May 2026 19:22:13 -0700 Subject: [PATCH 20/69] add Chinese translation pipeline (#3265) * add Chinese translation pipeline Signed-off-by: Jiayue Chen --- .github/workflows/build_doc.yml | 9 +- .github/workflows/translate_doc_zh.yml | 81 +++++ .gitignore | 3 + docs/source/_static/custom.css | 40 +++ docs/source/_static/custom.js | 105 +++++++ docs/source/conf.py | 3 + docs/source/locale/README.md | 27 ++ requirements/docs.txt | 1 + tools/translate_docs_zh.py | 420 +++++++++++++++++++++++++ 9 files changed, 688 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/translate_doc_zh.yml create mode 100644 docs/source/locale/README.md create mode 100644 tools/translate_docs_zh.py diff --git a/.github/workflows/build_doc.yml b/.github/workflows/build_doc.yml index 64247a757e8..90f45507b6e 100644 --- a/.github/workflows/build_doc.yml +++ b/.github/workflows/build_doc.yml @@ -42,15 +42,22 @@ jobs: - name: Build Versioned Sphinx Documentation run: | sphinx-multiversion docs/source output + SPHINX_LANGUAGE=zh_CN sphinx-multiversion docs/source output-zh continue-on-error: false - - name: Setup root docs and .nojekyll + - name: Setup root docs, .nojekyll, and Chinese docs run: | touch output/.nojekyll # Copy dev docs to root (latest docs at root URL) cp -r output/dev/* output/ # Remove dev directory to avoid duplication rm -rf output/dev + # Copy Chinese dev docs to /zh_CN/ and versioned docs under /zh_CN// + mkdir -p output/zh_CN + cp -r output-zh/dev/* output/zh_CN/ + rm -rf output-zh/dev + cp -r output-zh/. output/zh_CN/ + rm -rf output-zh - name: Upload doc artifacts to GHA uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 diff --git a/.github/workflows/translate_doc_zh.yml b/.github/workflows/translate_doc_zh.yml new file mode 100644 index 00000000000..9f9a62f2c3c --- /dev/null +++ b/.github/workflows/translate_doc_zh.yml @@ -0,0 +1,81 @@ +name: Update Chinese Documentation Translations + +on: + schedule: + - cron: "0 9 * * 1" + workflow_dispatch: + inputs: + max_entries: + description: "Maximum entries to translate in this run" + required: false + default: "10" + +concurrency: + group: update-chinese-docs + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +jobs: + translate-docs: + name: Update Chinese docs translations + runs-on: ubuntu-latest + steps: + - name: Harden Runner + uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 + with: + egress-policy: audit + + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + ref: dev + + - name: Setup Python 3.13 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: "3.13" + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements/docs.txt + + - name: Update gettext catalogs + run: | + sphinx-build -b gettext docs/source docs/build/gettext + sphinx-intl update -p docs/build/gettext -l zh_CN -d docs/source/locale -j 1 + + - name: Translate missing Chinese entries + env: + TRANSLATION_API_BASE_URL: ${{ secrets.TRANSLATION_API_BASE_URL }} + TRANSLATION_API_KEY: ${{ secrets.TRANSLATION_API_KEY }} + TRANSLATION_MODEL: ${{ secrets.TRANSLATION_MODEL }} + TRANSLATION_MAX_ENTRIES: ${{ inputs.max_entries || '500' }} + run: | + python tools/translate_docs_zh.py + + - name: Validate Chinese docs build + run: | + SPHINX_LANGUAGE=zh_CN sphinx-build -b html docs/source docs/build/html/zh_CN + + - name: Open or update translation PR + uses: peter-evans/create-pull-request@v8 + with: + token: ${{ github.token }} + branch: automation/update-chinese-docs + base: dev + delete-branch: true + add-paths: | + docs/source/locale/zh_CN/LC_MESSAGES/**/*.po + commit-message: "Update Chinese documentation translations" + title: "Update Chinese documentation translations" + body: | + Automated weekly update for Chinese documentation translations. + + Generated by `.github/workflows/translate_doc_zh.yml`. + labels: | + documentation + automation diff --git a/.gitignore b/.gitignore index 1e31eb8fde9..a9b91514fa5 100644 --- a/.gitignore +++ b/.gitignore @@ -143,3 +143,6 @@ operator/cover.out operator/coverage.html operator/Dockerfile.cross *.spv + +# Compiled gettext catalogs (built from .po at HTML build time) +docs/source/locale/**/*.mo diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index 0d550e684a6..f649ddda57a 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -68,6 +68,46 @@ div.highlight button.copybtn + button.copybtn { font-size: 10px; } +.lmcache-language-switcher { + position: fixed; + top: 10px; + right: 72px; + z-index: 60; + display: inline-flex; + align-items: center; + gap: 8px; + min-height: 36px; + padding: 0 8px; + white-space: nowrap; + font-size: 18px; + font-weight: 700; + line-height: 1; + color: var(--foreground); +} + +@media (max-width: 640px) { + .lmcache-language-switcher { + right: 56px; + font-size: 15px; + } +} + +.lmcache-language-switcher a { + color: inherit; + opacity: 0.72; + text-decoration: none; +} + +.lmcache-language-switcher a:hover, +.lmcache-language-switcher a:focus-visible, +.lmcache-language-switcher a[aria-current="page"] { + opacity: 1; +} + +.lmcache-language-switcher__divider { + opacity: 0.38; +} + .lmcache-goblin { position: fixed; z-index: 25; diff --git a/docs/source/_static/custom.js b/docs/source/_static/custom.js index 63800b19efa..06e83d200f3 100644 --- a/docs/source/_static/custom.js +++ b/docs/source/_static/custom.js @@ -52,6 +52,110 @@ function addRunLlmWidget() { document.head.appendChild(script); } +/** + * Return true when the current page is rendered under the Chinese docs prefix. + * + * @returns {boolean} Whether the current path is a Chinese documentation page. + */ +function isChineseDocsPage() { + return window.location.pathname.split("/").includes("zh_CN"); +} + +/** + * Build the target language URL for the current page. + * + * @param {"en" | "zh_CN"} language The target language. + * @returns {string} The target URL. + */ +function buildLanguageUrl(language) { + var pathParts = window.location.pathname.split("/"); + var zhIndex = pathParts.indexOf("zh_CN"); + + if (language === "zh_CN" && zhIndex === -1) { + pathParts.splice(1, 0, "zh_CN"); + } else if (language === "en" && zhIndex !== -1) { + pathParts.splice(zhIndex, 1); + } + + var nextPath = pathParts.join("/") || "/"; + return nextPath + window.location.search + window.location.hash; +} + +/** + * Build the home URL for a target language. + * + * @param {"en" | "zh_CN"} language The target language. + * @returns {string} The target language home URL. + */ +function buildLanguageHomeUrl(language) { + return language === "zh_CN" ? "/zh_CN/" : "/"; +} + +/** + * Fall back to the target language home page if the equivalent page is absent. + * + * @param {HTMLAnchorElement} link The language link to validate. + * @param {"en" | "zh_CN"} language The target language. + * @returns {void} + */ +function fallbackMissingLanguagePage(link, language) { + window + .fetch(link.href, { method: "HEAD" }) + .then(function (response) { + if (!response.ok) { + link.href = buildLanguageHomeUrl(language); + } + }) + .catch(function () { + link.href = buildLanguageHomeUrl(language); + }); +} + +/** + * Add a compact language switcher to the docs header. + * + * @returns {void} + */ +function addLanguageSwitcher() { + if (document.querySelector(".lmcache-language-switcher")) { + return; + } + + var switcher = document.createElement("div"); + var chineseLink = document.createElement("a"); + var divider = document.createElement("span"); + var englishLink = document.createElement("a"); + var isChinesePage = isChineseDocsPage(); + + switcher.className = "lmcache-language-switcher"; + switcher.setAttribute("aria-label", "Documentation language"); + + chineseLink.href = buildLanguageUrl("zh_CN"); + chineseLink.textContent = "中文"; + chineseLink.setAttribute("aria-label", "Switch to Chinese"); + + divider.className = "lmcache-language-switcher__divider"; + divider.textContent = "|"; + + englishLink.href = buildLanguageUrl("en"); + englishLink.textContent = "Eng"; + englishLink.setAttribute("aria-label", "Switch to English"); + + if (isChinesePage) { + chineseLink.setAttribute("aria-current", "page"); + } else { + englishLink.setAttribute("aria-current", "page"); + } + + switcher.appendChild(chineseLink); + switcher.appendChild(divider); + switcher.appendChild(englishLink); + document.body.appendChild(switcher); + + fallbackMissingLanguagePage(chineseLink, "zh_CN"); + fallbackMissingLanguagePage(englishLink, "en"); +} + /** * Remove all goblin easter egg elements from the current page. * @@ -391,6 +495,7 @@ function addGoblinEasterEgg() { * @returns {void} */ function initializeDocsWidgets() { + addLanguageSwitcher(); addRunLlmWidget(); addGoblinEasterEgg(); } diff --git a/docs/source/conf.py b/docs/source/conf.py index 05f4530ec62..2536a3727ab 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -66,6 +66,9 @@ def add_line(self, line: str, source: str, *lineno: int) -> None: templates_path = ["_templates"] exclude_patterns: list[Any] = [] add_module_names = False +language = os.environ.get("SPHINX_LANGUAGE", "en") +locale_dirs = ["locale/"] +gettext_compact = False # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/docs/source/locale/README.md b/docs/source/locale/README.md new file mode 100644 index 00000000000..3346e9adbf1 --- /dev/null +++ b/docs/source/locale/README.md @@ -0,0 +1,27 @@ +# Documentation Translations + +This directory stores Sphinx/gettext translation catalogs. English `.rst` files +under `docs/source/` remain the source of truth. + +## Directory Layout + +- `zh_CN/`: Simplified Chinese translations. +- `LC_MESSAGES/`: Standard gettext catalog directory. +- `.po`: Reviewable translation files generated/updated by `sphinx-intl`. + +## Translation Flow + +1. `sphinx-build -b gettext` extracts English strings into `.pot` templates (throwaway). +2. `sphinx-intl update` writes new strings as empty entries and changed strings as `fuzzy` in the committed `.po` files. +3. `tools/translate_docs_zh.py` calls the model to fill in the empty and fuzzy entries. +4. A PR is opened on `automation/update-chinese-docs` for human review. +5. After merge, `build_doc.yml` rebuilds the site at `/zh_CN/`. + +Steps 1–4 run weekly in `translate_doc_zh.yml` (Monday 09:00 UTC, or manual +dispatch). Step 5 runs automatically on push to `dev`. + +To change the translation prompt, edit the system message in +`tools/translate_docs_zh.py`. + +This README is for maintainers only. It is not linked from the Sphinx toctree, +so it should not appear on the public documentation website. diff --git a/requirements/docs.txt b/requirements/docs.txt index c77ef894fc1..65589f42849 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -11,6 +11,7 @@ sphinxawesome_theme==5.3.2 sphinx-copybutton==0.5.2 sphinxcontrib-mermaid==1.2.2 sphinx-multiversion==0.2.4 +sphinx-intl==2.3.1 sphinxcontrib-images msgspec sphinx-design diff --git a/tools/translate_docs_zh.py b/tools/translate_docs_zh.py new file mode 100644 index 00000000000..4ebc9d8bb98 --- /dev/null +++ b/tools/translate_docs_zh.py @@ -0,0 +1,420 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Translate missing Chinese Sphinx gettext entries with a model endpoint.""" + +# Future +from __future__ import annotations + +# Standard +from pathlib import Path +from urllib import request +import json +import os +import sys + +LOCALE_DIR = Path("docs/source/locale/zh_CN/LC_MESSAGES") +MAX_ENTRIES = int(os.environ.get("TRANSLATION_MAX_ENTRIES", "500")) + +SYSTEM_PROMPT = ( + "You translate Sphinx gettext entries from English to Simplified Chinese. " + "Each request contains one focal entry to translate, optionally preceded " + "by a 'Source file:' line and surrounded by 'Previous entry' and " + "'Next entry' blocks for context. Translate ONLY the focal msgid under " + "'Translate this entry'. Return the Chinese translation as plain text " + "with no labels, no surrounding quotes, no other entries, and no " + "explanation. " + "Use the neighboring entries only to infer the section topic and to " + "keep terminology consistent; never include their text in your output. " + "If the focal msgid is a code identifier (snake_case, " + "SCREAMING_SNAKE_CASE, a config key, environment variable, file path, " + "URL, or product name), return it unchanged. " + "Do not add notes, explanations, summaries, or comments. " + "Preserve all reStructuredText/Sphinx formatting exactly, including " + "headings, indentation, directives, roles, labels, anchors, references, " + "admonitions, lists, tables, and cross-references. " + "Do not translate or change code blocks, inline code, shell commands, " + "configuration snippets, Python/JSON/YAML/TOML, API signatures, " + "environment variables, file paths, URLs, placeholders, package names, " + "class names, function names, method names, parameter names, CLI flags, " + "model names, feature names, product names, or values inside backticks. " + "Preserve Sphinx roles exactly, such as :ref:`...`, :doc:`...`, " + ":class:`...`, :func:`...`, :meth:`...`, :mod:`...`, :attr:`...`, " + ":obj:`...`, and :term:`...`. " + "Translate only normal explanatory prose. " + "Use natural Simplified Chinese for developer documentation. " + "Use these terms consistently: " + "KV cache -> KV Cache; KV Cache -> KV Cache; " + "inference -> 推理; serving -> 服务; " + "prefill -> Prefill; decode -> 解码; decoding -> 解码; " + "offload -> 卸载; offloading -> 卸载; " + "prefetch -> 预取; prefetch module -> 预取模块; " + "evict -> 逐出; eviction -> 逐出; " + "recompute -> 重计算; recomputation -> 重计算; " + "throughput -> 吞吐量; latency -> 延迟; " + "GPU memory -> 显存; VRAM -> 显存; CPU memory -> CPU 内存; " + "backend -> 后端; connector -> 连接器; storage -> 存储; " + "cache hit -> 缓存命中; cache miss -> 缓存未命中; lookup -> 查找; " + "chunk -> 块; layerwise -> 逐层; compression -> 压缩; " + "quantization -> 量化; " + "serialization -> 序列化; deserialization -> 反序列化; " + "disaggregated prefill -> 分离式 Prefill; multi-tenant -> 多租户. " + "Keep these terms unchanged: LMCache, vLLM, SGLang, TensorRT-LLM, NIXL, " + "CUDA, ROCm, Redis, S3, GDS, POSIX, Hugging Face, CacheBlend, cache_salt, " + "TTFT, LLM, GPU, CPU, API, token, prompt, system prompt, Attention. " + "If unsure whether something is syntax, code, an identifier, or a " + "product name, keep it unchanged." +) + + +class PoEntry: + """One translatable message read from a ``.po`` translation file. + + A ``.po`` file is a list of messages. Each message pairs the original + English text with its Chinese translation, plus some status markers. + + Attributes: + start: Line number where this message begins in the file. + end: Line number just after this message ends in the file. + flags: Status markers on the message. ``"fuzzy"`` means the English + text changed since the last translation, so the Chinese is stale + and should be redone. + msgid: The original English text. + msgstr: The current Chinese translation (empty string if not yet + translated). + """ + + def __init__( + self, + start: int, + end: int, + flags: set[str], + msgid: str, + msgstr: str, + ) -> None: + self.start = start + self.end = end + self.flags = flags + self.msgid = msgid + self.msgstr = msgstr + + +def decode_po_string(value: str) -> str: + """Unwrap one quoted ``.po`` line into a plain Python string.""" + return json.loads(value) + + +def encode_po_string(value: str) -> list[str]: + """Convert a Python string into one or more quoted ``.po`` lines.""" + if "\n" not in value: + return [json.dumps(value, ensure_ascii=False)] + + lines = ['""'] + parts = value.splitlines(keepends=True) + for part in parts: + lines.append(json.dumps(part, ensure_ascii=False)) + return lines + + +def collect_field(lines: list[str], index: int) -> tuple[str, int]: + """Read one ``msgid``/``msgstr`` value plus any continuation lines. + + Returns: + The joined string value, and the next line number to read. + """ + _, raw_value = lines[index].split(" ", 1) + values = [decode_po_string(raw_value.strip())] + index += 1 + + while index < len(lines) and lines[index].startswith('"'): + values.append(decode_po_string(lines[index].strip())) + index += 1 + + return "".join(values), index + + +def parse_entries(lines: list[str]) -> list[PoEntry]: + """Parse a ``.po`` file into one :class:`PoEntry` per message.""" + entries: list[PoEntry] = [] + index = 0 + + while index < len(lines): + start = index + flags: set[str] = set() + + while index < len(lines) and not lines[index].startswith("msgid "): + if lines[index].startswith("#,"): + flags.update(flag.strip() for flag in lines[index][2:].split(",")) + index += 1 + + if index >= len(lines): + break + + msgid, index = collect_field(lines, index) + + while index < len(lines) and not lines[index].startswith("msgstr "): + index += 1 + + if index >= len(lines): + break + + msgstr, index = collect_field(lines, index) + + while index < len(lines) and lines[index].strip(): + index += 1 + + entries.append(PoEntry(start, index, flags, msgid, msgstr)) + + return entries + + +def replace_msgstr(lines: list[str], entry: PoEntry, translation: str) -> None: + """Replace one entry's ``msgstr`` in ``lines``, in place.""" + msgstr_index = entry.start + while msgstr_index < entry.end and not lines[msgstr_index].startswith("msgstr "): + msgstr_index += 1 + + msgstr_end = msgstr_index + 1 + while msgstr_end < len(lines) and lines[msgstr_end].startswith('"'): + msgstr_end += 1 + + encoded = encode_po_string(translation) + lines[msgstr_index:msgstr_end] = ["msgstr " + encoded[0], *encoded[1:]] + + +def clean_fuzzy_flag(lines: list[str], entry: PoEntry) -> None: + """Remove the ``"fuzzy"`` marker from a message, modifying ``lines`` in place.""" + for index in range(entry.start, entry.end): + if not lines[index].startswith("#,"): + continue + flags = [flag.strip() for flag in lines[index][2:].split(",")] + flags = [flag for flag in flags if flag != "fuzzy"] + if flags: + lines[index] = "#, " + ", ".join(flags) + else: + lines.pop(index) + return + + +def should_translate(entry: PoEntry) -> bool: + """Return True if the entry has English content and is empty or fuzzy.""" + return bool(entry.msgid.strip()) and ( + not entry.msgstr.strip() or "fuzzy" in entry.flags + ) + + +def build_user_message( + target: PoEntry, + previous: PoEntry | None, + next_: PoEntry | None, + source_file: str | None = None, +) -> str: + """Build the chat-completions user message for one translation call. + + Wraps the focal entry in labeled blocks and prepends the neighboring + entries (when present) so the model can disambiguate identifiers from + prose and reuse established terminology. + + Args: + target: The entry being translated. + previous: The entry immediately before ``target``, or None at the + start of the file or when the neighbor has an empty msgid (the + gettext metadata header). + next_: The entry immediately after ``target``, or None at the end. + source_file: Optional ``.rst`` path the entries came from (e.g., + ``"api_reference/configurations.rst"``). + + Returns: + A multi-line string suitable for the ``content`` field of a user + message. + """ + parts: list[str] = [] + + if source_file: + parts.append(f"Source file: {source_file}") + parts.append("") + + if previous is not None and previous.msgid.strip(): + parts.append("Previous entry (context only, do not translate):") + parts.append(f" msgid: {json.dumps(previous.msgid, ensure_ascii=False)}") + if previous.msgstr.strip(): + parts.append(f" msgstr: {json.dumps(previous.msgstr, ensure_ascii=False)}") + parts.append("") + + parts.append("Translate this entry:") + parts.append(f" msgid: {json.dumps(target.msgid, ensure_ascii=False)}") + + if next_ is not None and next_.msgid.strip(): + parts.append("") + parts.append("Next entry (context only, do not translate):") + parts.append(f" msgid: {json.dumps(next_.msgid, ensure_ascii=False)}") + if next_.msgstr.strip(): + parts.append(f" msgstr: {json.dumps(next_.msgstr, ensure_ascii=False)}") + + return "\n".join(parts) + + +def clean_translation_response(text: str) -> str: + """Strip incidental formatting the model occasionally adds to the output. + + Trims whitespace, removes a leading ``msgstr:`` / ``Translation:`` label + if present, and unwraps a single pair of surrounding double quotes. + + Args: + text: Raw ``message.content`` string returned by the endpoint. + + Returns: + The Chinese translation only, ready to be written into ``msgstr``. + """ + cleaned = text.strip() + + for prefix in ("msgstr:", "msgstr =", "translation:"): + if cleaned.lower().startswith(prefix): + cleaned = cleaned[len(prefix) :].lstrip() + break + + if len(cleaned) >= 2 and cleaned[0] == '"' and cleaned[-1] == '"': + try: + cleaned = json.loads(cleaned) + except json.JSONDecodeError: + cleaned = cleaned[1:-1] + + return cleaned + + +def endpoint_url() -> str: + """Build the chat-completions URL from ``TRANSLATION_API_BASE_URL``.""" + base_url = os.environ["TRANSLATION_API_BASE_URL"].rstrip("/") + if base_url.endswith("/chat/completions"): + return base_url + return base_url + "/chat/completions" + + +def translate_text(user_message: str) -> str: + """Send a pre-built user message to the endpoint and return the cleaned translation. + + Raises: + RuntimeError: If the response has no message content. + """ + payload = { + "model": os.environ["TRANSLATION_MODEL"], + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": user_message}, + ], + "temperature": 0.2, + } + body = json.dumps(payload).encode("utf-8") + http_request = request.Request( + endpoint_url(), + data=body, + headers={ + "Authorization": "Bearer " + os.environ["TRANSLATION_API_KEY"], + "Content-Type": "application/json", + }, + method="POST", + ) + + with request.urlopen(http_request, timeout=120) as response: + data = json.loads(response.read().decode("utf-8")) + + try: + raw = data["choices"][0]["message"]["content"] + except (KeyError, IndexError) as exc: + raise RuntimeError("Translation endpoint returned no message content") from exc + return clean_translation_response(raw) + + +def validate_environment() -> None: + """Exit with an error if any required API env var is unset. + + Raises: + SystemExit: If ``TRANSLATION_API_BASE_URL``, ``TRANSLATION_API_KEY``, + or ``TRANSLATION_MODEL`` is missing. + """ + missing = [ + name + for name in ( + "TRANSLATION_API_BASE_URL", + "TRANSLATION_API_KEY", + "TRANSLATION_MODEL", + ) + if not os.environ.get(name) + ] + if missing: + joined = ", ".join(missing) + raise SystemExit(f"Missing required translation secret(s): {joined}") + + +def update_file(path: Path, remaining_budget: int) -> int: + """Translate up to ``remaining_budget`` empty/fuzzy messages in one ``.po`` file. + + For each entry needing translation, sends the focal msgid along with its + immediate neighbors as contextual hints to the model. + + Returns: + Number of messages actually translated. + """ + lines = path.read_text(encoding="utf-8").splitlines() + entries = parse_entries(lines) + translated = 0 + + try: + source_file = str(path.relative_to(LOCALE_DIR).with_suffix(".rst")) + except ValueError: + source_file = None + + for i in reversed(range(len(entries))): + if translated >= remaining_budget: + break + entry = entries[i] + if not should_translate(entry): + continue + + previous = entries[i - 1] if i > 0 else None + next_ = entries[i + 1] if i + 1 < len(entries) else None + + user_message = build_user_message(entry, previous, next_, source_file) + translation = translate_text(user_message) + + replace_msgstr(lines, entry, translation) + if "fuzzy" in entry.flags: + clean_fuzzy_flag(lines, entry) + translated += 1 + + if translated: + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + return translated + + +def main() -> int: + """Translate up to ``MAX_ENTRIES`` messages across all Chinese ``.po`` files.""" + po_files = sorted(LOCALE_DIR.glob("**/*.po")) + if not po_files: + print("No Chinese PO files found.") + return 0 + + needs_translation = False + for path in po_files: + entries = parse_entries(path.read_text(encoding="utf-8").splitlines()) + if any(should_translate(entry) for entry in entries): + needs_translation = True + break + + if not needs_translation: + print("No missing or fuzzy Chinese translations found.") + return 0 + + validate_environment() + + translated = 0 + for path in po_files: + if translated >= MAX_ENTRIES: + break + translated += update_file(path, MAX_ENTRIES - translated) + + print(f"Translated {translated} Chinese documentation entries.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From d37cfa16ba49b90918516e48c90e3d64e8102603 Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Sat, 16 May 2026 01:38:27 -0700 Subject: [PATCH 21/69] [CI] pin vLLM CUDA wheel variant (#3298) * pin vllm wheel cuda version Signed-off-by: deng451e <838677410@qq.com> --- .buildkite/k3_harness/setup-blend-env.sh | 25 +++++++++--------------- docker/Dockerfile | 12 +++++++----- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/.buildkite/k3_harness/setup-blend-env.sh b/.buildkite/k3_harness/setup-blend-env.sh index 1f8e6b47168..8b2d0b62caf 100755 --- a/.buildkite/k3_harness/setup-blend-env.sh +++ b/.buildkite/k3_harness/setup-blend-env.sh @@ -53,26 +53,17 @@ TEST_VENV_BIN="/workspace/.venv/bin" # When flashinfer and flashinfer-cubin resolve to different patch versions, skip strict check. export FLASHINFER_DISABLE_VERSION_CHECK=1 -# vLLM: pinned to a known-good cu129 nightly wheel as a temporary workaround. -# Background: vLLM's PyPI stable is now built against CUDA 13 (libcudart.so.13) -# which this CUDA-12 container can't load. The cu129 nightly channel auto-rolls -# on every upstream commit and resolver-based installs have proven fragile -# during the cu12/cu13 split (unsafe-best-match picks PyPI stable; first-index -# with a version cap backtracked all the way to vllm 0.2.5). Pinning a wheel -# URL bypasses resolution for vLLM; transitive deps still resolve from PyPI + -# PyTorch cu129. -# TODO: bump the URL when this pinned build goes stale, or revert to dynamic -# selection once the cu12/cu13 transition settles. -VLLM_WHEEL_URL="https://wheels.vllm.ai/8189a15914ca48461acf106f126c58ef7e41c9ee/vllm-0.20.2rc1.dev112%2Bg8189a1591.cu129-cp38-abi3-manylinux_2_34_x86_64.whl" -echo "--- :python: Installing vLLM (pinned wheel: ${VLLM_WHEEL_URL})" +# vLLM: force cu12.9 variant; PyPI default is cu13 (libcudart.so.13), unloadable in this cu12 container. +echo "--- :python: Installing vLLM (nightly cu129)" # --index-strategy unsafe-best-match is needed so uv considers PyPI for vLLM's # transitive deps (setuptools>=77 specifically — the PyTorch cu129 index only # carries setuptools<=70.2.0, and uv's default first-index would refuse to -# look at PyPI for it). Safe here because vLLM itself is pinned by URL, so -# resolver strategy can't influence which vLLM gets installed. -"${UV_BIN}" pip install -p "${TEST_VENV_BIN}/python" -U "${VLLM_WHEEL_URL}" \ - --extra-index-url "https://download.pytorch.org/whl/cu129" \ +# look at PyPI for it). VLLM_PRECOMPILED_WHEEL_VARIANT pins the vLLM CUDA +# variant, so the resolver strategy can't pull a mismatched build. +VLLM_PRECOMPILED_WHEEL_VARIANT=cu129 "${UV_BIN}" pip install -p "${TEST_VENV_BIN}/python" -U vllm --pre \ + --extra-index-url https://wheels.vllm.ai/nightly/cu129 \ + --extra-index-url https://download.pytorch.org/whl/cu129 \ --index-strategy unsafe-best-match @@ -103,6 +94,8 @@ echo "--- :python: Installing LMCache from source" # Skip setuptools_scm git describe; the repo carries non-PEP-440 tags # (nightly, nightly-cu13) that crash the newer vcs_versioning backend. export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE="${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE:-0.0.0+ci}" +# Select cu12 nixl wheel +export LMCACHE_CUDA_MAJOR=12 "${UV_BIN}" pip install -p "${DEFAULT_VENV_BIN}/python" -e . --no-build-isolation "${UV_BIN}" pip install -p "${TEST_VENV_BIN}/python" -e . --no-build-isolation # Work around openai_harmony vocab download/load issues for GPT-OSS (vLLM recipes troubleshooting). diff --git a/docker/Dockerfile b/docker/Dockerfile index 3540d2ac8c6..b3aec2348ac 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -81,13 +81,13 @@ RUN --mount=type=cache,target=/root/.cache/ccache,id=ccache \ CUDA_TAG=cu$(echo ${CUDA_VERSION} | tr -d '.') && \ export LMCACHE_CUDA_MAJOR=$(echo ${CUDA_VERSION} | cut -d. -f1) && \ if [ "$VLLM_VERSION" = "nightly" ]; then \ - uv pip install --prerelease=allow \ + VLLM_PRECOMPILED_WHEEL_VARIANT=${CUDA_TAG} uv pip install --prerelease=allow \ 'vllm[runai,tensorizer,flashinfer]' \ --extra-index-url https://wheels.vllm.ai/nightly/${CUDA_TAG} \ --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG} \ --index-strategy unsafe-best-match ; \ else \ - uv pip install --prerelease=allow \ + VLLM_PRECOMPILED_WHEEL_VARIANT=${CUDA_TAG} uv pip install --prerelease=allow \ "vllm[runai,tensorizer,flashinfer]==${VLLM_VERSION}" ; \ fi && \ python3 -c 'import torch; print("TORCH=", torch.__version__)' && \ @@ -108,7 +108,7 @@ ARG CUDA_VERSION RUN . /opt/venv/bin/activate && \ CUDA_TAG=cu$(echo ${CUDA_VERSION} | tr -d '.') && \ - uv pip install --prerelease=allow \ + VLLM_PRECOMPILED_WHEEL_VARIANT=${CUDA_TAG} uv pip install --prerelease=allow \ vllm[runai,tensorizer,flashinfer] \ --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG} \ --index-strategy unsafe-best-match && \ @@ -120,7 +120,8 @@ WORKDIR /workspace ENTRYPOINT ["/opt/venv/bin/vllm", "serve"] #################### vLLM IMAGE & LMCache (Release, cu129) ###################### -# Installs stable vLLM with cu129 index and lmcache from the v{tag}-cu129 GitHub Release. +# Installs nightly cu129 vLLM (no stable cu129 channel exists upstream) and +# lmcache from the v{tag}-cu129 GitHub Release. FROM base AS image-release-cu129 @@ -132,8 +133,9 @@ ARG LMCACHE_VERSION RUN . /opt/venv/bin/activate && \ CUDA_TAG=cu$(echo ${CUDA_VERSION} | tr -d '.') && \ VER=$(echo ${LMCACHE_VERSION} | sed 's/^v//') && \ - uv pip install --prerelease=allow \ + VLLM_PRECOMPILED_WHEEL_VARIANT=${CUDA_TAG} uv pip install --prerelease=allow \ vllm[runai,tensorizer,flashinfer] \ + --extra-index-url https://wheels.vllm.ai/nightly/${CUDA_TAG} \ --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG} \ --index-strategy unsafe-best-match && \ uv pip install lmcache==${VER} \ From 4658d69e3c3be1fa055beb7eaa6047800d3ee16c Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Sun, 17 May 2026 09:44:14 +0800 Subject: [PATCH 22/69] Defer test_cache runtime imports so lmcache-cli loads without torch (#3294) * Defer test_cache runtime imports so lmcache-cli loads without torch Signed-off-by: Tony Lin --- lmcache/cli/commands/bench/__init__.py | 25 +++++++++++-------------- lmcache/cli/commands/test_cache.py | 7 +++++-- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lmcache/cli/commands/bench/__init__.py b/lmcache/cli/commands/bench/__init__.py index 79fb1429b15..0055411167b 100644 --- a/lmcache/cli/commands/bench/__init__.py +++ b/lmcache/cli/commands/bench/__init__.py @@ -8,6 +8,7 @@ import sys # First Party +from lmcache.cli.commands import test_cache as _test_cache_mod from lmcache.cli.commands.base import BaseCommand from lmcache.cli.commands.bench.engine_bench.config import ( EngineBenchConfig, @@ -26,17 +27,9 @@ StatsCollector, ) from lmcache.cli.commands.bench.engine_bench.workloads import create_workload +from lmcache.cli.commands.test_cache import TestCacheCommand from lmcache.logging import init_logger -# Gated for slim install — TestCacheCommand pulls torch/MP runtime. -_TEST_CACHE_IMPORT_ERROR: ImportError | None = None -try: - # First Party - from lmcache.cli.commands.test_cache import TestCacheCommand -except ImportError as _exc: - _TEST_CACHE_IMPORT_ERROR = _exc - TestCacheCommand = None # type: ignore[assignment,misc] - logger = init_logger(__name__) @@ -47,7 +40,7 @@ def __init__(self) -> None: super().__init__() # None on slim install; _register_kvcache registers a stub instead. self._kvcache_delegate = ( - TestCacheCommand() if TestCacheCommand is not None else None + TestCacheCommand() if _test_cache_mod._IMPORT_ERROR is None else None ) def name(self) -> str: @@ -340,7 +333,7 @@ def _register_kvcache( """Register ``lmcache bench kvcache``. Delegates to :class:`TestCacheCommand`, or registers a stub on slim install. """ - if self._kvcache_delegate is None: + if _test_cache_mod._IMPORT_ERROR is not None: subparsers.add_parser( "kvcache", help="(requires full lmcache install)", @@ -351,6 +344,7 @@ def _register_kvcache( ), ).set_defaults(func=self.execute) return + assert self._kvcache_delegate is not None parser = subparsers.add_parser( "kvcache", help=self._kvcache_delegate.help(), @@ -360,21 +354,24 @@ def _register_kvcache( "and verifies KV cache checksums." ), ) + assert self._kvcache_delegate is not None self._kvcache_delegate.add_arguments(parser) parser.set_defaults(func=self.execute) def _bench_kvcache(self, args: argparse.Namespace) -> None: """Dispatch ``lmcache bench kvcache`` to ``TestCacheCommand``.""" - if self._kvcache_delegate is None: + if _test_cache_mod._IMPORT_ERROR is not None: print( "ERROR: `lmcache bench kvcache` needs the full LMCache " - "package, but only the `lmcache-cli` shell is installed.\n" + "package (torch, zmq, MP runtime), but only the " + "`lmcache-cli` shell appears to be installed.\n" " Install the full package with `pip install lmcache` " "and try again.\n" - f" Original import error: {_TEST_CACHE_IMPORT_ERROR}", + f" Original import error: {_test_cache_mod._IMPORT_ERROR}", file=sys.stderr, ) sys.exit(1) + assert self._kvcache_delegate is not None self._kvcache_delegate.execute(args) def execute(self, args: argparse.Namespace) -> None: diff --git a/lmcache/cli/commands/test_cache.py b/lmcache/cli/commands/test_cache.py index 37177471c1d..21d1448cb54 100644 --- a/lmcache/cli/commands/test_cache.py +++ b/lmcache/cli/commands/test_cache.py @@ -46,7 +46,6 @@ # First Party from lmcache import torch_dev, torch_device_type from lmcache.cli.commands.base import BaseCommand -from lmcache.utils import check_interprocess_event_support # ``lmcache bench kvcache`` allocates real CUDA tensors and talks to # the MP server via ZMQ, both of which are absent from the thin @@ -62,7 +61,11 @@ import zmq # First Party - from lmcache.utils import EngineType, compress_slot_mapping + from lmcache.utils import ( + EngineType, + check_interprocess_event_support, + compress_slot_mapping, + ) from lmcache.v1.kv_layer_groups import ( DTYPE_MAP, KVLayerGroupInfo, From b4a2d7edb1c56e60b744ae40819096594f5a0f7c Mon Sep 17 00:00:00 2001 From: ikasas <144680962+ikasas@users.noreply.github.com> Date: Mon, 18 May 2026 10:52:59 +0900 Subject: [PATCH 23/69] [Doc] Fix a typo in p2p_init_ports parameter (#2282) Signed-off-by: ikasas Co-authored-by: Dongjoo Seo Co-authored-by: maobaolong --- docs/source/kv_cache_management/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/kv_cache_management/index.rst b/docs/source/kv_cache_management/index.rst index 713d86e7d8d..6ace02cf4e7 100644 --- a/docs/source/kv_cache_management/index.rst +++ b/docs/source/kv_cache_management/index.rst @@ -101,7 +101,7 @@ Expected output: # p2p configuration p2p_host: localhost - p2p_init_port: [11, 12, 13] + p2p_init_ports: [11, 12, 13] .. toctree:: :maxdepth: 1 From 516b498cb45764754acccae17c23e24e6235cf15 Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Mon, 18 May 2026 20:33:27 +0800 Subject: [PATCH 24/69] feat: add StubCPUDevice to allow import/startup fallback in CPU-only environments (#3280) * feat: add StubCPUDevice to enable CPU-only MP mode Introduce StubCPUDevice as a torch_dev drop-in for CPU-only environments, allowing the MP server to run without a GPU. Update _detect_device() to fall back to StubCPUDevice when no accelerator is available. Override KV cache layout to HND for CPU backend since vLLM's get_kv_cache_layout() is unavailable in MP mode and defaults to NHD. Verified with L1 adapter end-to-end. Signed-off-by: Tony Lin * fix path sharder ci in cpu-only path Signed-off-by: Tony Lin * add docstring Signed-off-by: Tony Lin * update comments Signed-off-by: Tony Lin * move platform from lmcache to lmcache/v1 Signed-off-by: Tony Lin --------- Signed-off-by: Tony Lin --- lmcache/__init__.py | 14 +- lmcache/v1/gpu_connector/utils.py | 10 +- lmcache/v1/platform/cpu/stub_cpu_device.py | 314 ++++++++++++++++++ tests/v1/storage_backend/test_path_sharder.py | 47 ++- 4 files changed, 369 insertions(+), 16 deletions(-) create mode 100644 lmcache/v1/platform/cpu/stub_cpu_device.py diff --git a/lmcache/__init__.py b/lmcache/__init__.py index 066f4d78151..992a1268fb3 100644 --- a/lmcache/__init__.py +++ b/lmcache/__init__.py @@ -33,8 +33,6 @@ def _detect_device() -> tuple[Any, str]: tuple[Any, str]: A tuple of (torch_device_module, device_type_string), e.g. ``(torch.cuda, "cuda")`` or ``(torch.xpu, "xpu")``. - Raises: - RuntimeError: If no supported accelerator is found (checked CUDA, XPU, HPU). """ try: # Third Party @@ -46,14 +44,20 @@ def _detect_device() -> tuple[Any, str]: return torch.xpu, "xpu" elif hasattr(torch, "hpu") and torch.hpu.is_available(): return torch.hpu, "hpu" - else: - # Fallback: always return torch.cuda for backward compatibility - # with existing tests and code paths that assume CUDA is the default. + elif torch.cuda.is_available(): return torch.cuda, "cuda" + else: + # First Party + from lmcache.v1.platform.cpu.stub_cpu_device import StubCPUDevice + + # Fallback: always return torch, cpu as stub + return StubCPUDevice("cpu"), "cpu" torch_dev, torch_device_type = _detect_device() +logger.info(" torch_dev=%s, torch_device_type=%s", torch_dev, torch_device_type) + # -------------------------- # Dynamic backend selection diff --git a/lmcache/v1/gpu_connector/utils.py b/lmcache/v1/gpu_connector/utils.py index 9fea963cf2e..7211d922e37 100644 --- a/lmcache/v1/gpu_connector/utils.py +++ b/lmcache/v1/gpu_connector/utils.py @@ -20,6 +20,7 @@ import torch # First Party +from lmcache import torch_device_type from lmcache.logging import init_logger from lmcache.utils import EngineType from lmcache.v1.config import LMCacheEngineConfig @@ -526,7 +527,14 @@ def normalize_kv_and_discover_format( detected_format = lmc_ops.GPUKVFormat.NB_NL_TWO_NH_BS_HS elif serving_engine == EngineType.VLLM: kv_layout = layout_hints.get("kv_layout") - if kv_layout is None: + # NOTE: vLLM's CPU attention backend stores KV cache in HND layout. + # however, get_kv_cache_layout from vllm.v1.attention.backends.utils + # does not return the right layout for CPU attention. + # Right fix should come from vllm side, but hardcode here as safeguard. + if torch_device_type == "cpu": + kv_layout = "HND" + logger.info("CPU backend detected, using HND KV cache layout") + elif kv_layout is None: logger.warning( "No KV Cache Layout hint provided when using vLLM, defaulting to NHD" ) diff --git a/lmcache/v1/platform/cpu/stub_cpu_device.py b/lmcache/v1/platform/cpu/stub_cpu_device.py new file mode 100644 index 00000000000..af54bf3a312 --- /dev/null +++ b/lmcache/v1/platform/cpu/stub_cpu_device.py @@ -0,0 +1,314 @@ +# SPDX-License-Identifier: Apache-2.0 +# Future +from __future__ import annotations + +# Standard +from contextlib import nullcontext +from typing import Any + + +class StubDeviceProperties: + """Stub for torch_dev.get_device_properties() return value.""" + + def __init__(self) -> None: + self.name = "StubCPU" + self.major = 0 + self.minor = 0 + self.total_memory = 0 + self.multi_processor_count = 0 + self.uuid = "stub-0000-0000-0000-000000000000" + + def __repr__(self) -> str: + return f"StubDeviceProperties(name={self.name!r})" + + +class StubEvent: + """Stub for a CUDA event, used in CPU-only test environments.""" + + def __init__( + self, + enable_timing: bool = False, + blocking: bool = False, + interprocess: bool = False, + ) -> None: + self.enable_timing = enable_timing + self.blocking = blocking + self.interprocess = interprocess + self._recorded = False + self._handle = b"stub_ipc_handle" + + def record(self, stream: Any = None) -> None: + """Mark this event as recorded on the given stream. + + Args: + stream: The stream to record on. If None, uses the + current stream. + """ + self._recorded = True + + def wait(self, stream: Any = None) -> None: + """Make the given stream wait until this event completes. + + Args: + stream: The stream that should wait. If None, uses the + current stream. + """ + return None + + def query(self) -> bool: + """Check whether the event has completed. + + Returns: + True always, since the stub has no real work. + """ + return True + + def synchronize(self) -> None: + """Block the host until this event completes. + + No-op in the stub implementation. + """ + return None + + def elapsed_time(self, end_event: "StubEvent") -> float: + """Return elapsed time in milliseconds between this event and *end_event*. + + Args: + end_event: The ending event to measure against. + + Returns: + Elapsed time in milliseconds. Always 0.0 for the stub. + """ + return 0.0 + + def ipc_handle(self) -> bytes: + """Return an IPC handle for cross-process sharing. + + Returns: + A bytes object representing the IPC handle. + """ + return self._handle + + @classmethod + def from_ipc_handle(cls, device: Any, handle: bytes) -> "StubEvent": + """Reconstruct a StubEvent from an IPC handle. + + Args: + device: The device to associate with the event. + handle: The IPC handle bytes obtained from + :meth:`ipc_handle`. + + Returns: + A new StubEvent with interprocess=True and the given handle. + """ + ev = cls(interprocess=True) + ev._handle = handle + return ev + + def __repr__(self) -> str: + return f"StubEvent(interprocess={self.interprocess}, recorded={self._recorded})" + + +class StubStream: + """Stub for a CUDA stream, used in CPU-only test environments.""" + + def __init__(self, device: Any = "cpu", priority: int = 0, **kwargs: Any) -> None: + self.device = device + self.priority = priority + self.cuda_stream = 0 + + def synchronize(self) -> None: + """Block the host until all kernels on this stream complete. + + No-op in the stub implementation. + """ + return None + + def wait_event(self, event: StubEvent) -> None: + """Make this stream wait until *event* completes. + + Args: + event: The event this stream should wait for. + """ + return None + + def wait_stream(self, stream: "StubStream") -> None: + """Make this stream wait until all kernels on *stream* complete. + + Args: + stream: The stream whose pending work must finish before + this stream continues. + """ + return None + + def record_event(self, event: StubEvent | None = None) -> StubEvent: + """Record an event on this stream and return it. + + Args: + event: An existing event to record. If None, a new + StubEvent is created. + + Returns: + The recorded StubEvent. + """ + event = event or StubEvent() + event.record(self) + return event + + def query(self) -> bool: + """Check whether all kernels on this stream have completed. + + Returns: + True always, since the stub has no real work. + """ + return True + + @staticmethod + def priority_range() -> tuple[int, int]: + """Return the range of stream priorities. + + Returns: + A tuple of (lowest_priority, highest_priority). Always + (0, 0) for the stub. + """ + return (0, 0) + + def __repr__(self) -> str: + return f"StubStream(device={self.device}, priority={self.priority})" + + +class StubCPUDevice: + """Stub stand-in for torch_dev in CPU-only test environments.""" + + def __init__(self, device_type: str = "cpu") -> None: + self._device_type = device_type + self._stream = StubStream(device=device_type) + + self.Event = StubEvent + self.Stream = StubStream + + def is_available(self) -> bool: + """Check whether the device backend is available. + + Returns: + False always, since this is a CPU-only stub. + """ + return False + + def init(self) -> None: + """Initialize the device backend. + + No-op in the stub implementation. + """ + return None + + def device(self, device: Any = None) -> Any: + """Return a context manager that sets the current device. + + Args: + device: The device to select. Ignored in the stub. + + Returns: + A no-op context manager. + """ + return nullcontext() + + def current_stream(self, device: Any = None) -> StubStream: + """Return the current stream for the given device. + + Args: + device: The device to query. Ignored in the stub. + + Returns: + The current StubStream. + """ + return self._stream + + def default_stream(self, device: Any = None) -> StubStream: + """Return the default stream for the given device. + + Args: + device: The device to query. Ignored in the stub. + + Returns: + The default StubStream. + """ + return self._stream + + def stream(self, stream: StubStream | None = None) -> Any: + """Return a context manager that sets the active stream. + + Args: + stream: The stream to activate. If None, uses the + current stream. + + Returns: + A context manager yielding the active StubStream. + """ + return nullcontext(stream or self._stream) + + def synchronize(self, device: Any = None) -> None: + """Wait for all streams on the given device to complete. + + Args: + device: The device to synchronize. Ignored in the stub. + """ + return None + + def set_stream(self, stream: StubStream) -> None: + """Set the current stream. + + Args: + stream: The stream to make current. + """ + self._stream = stream + + def device_count(self) -> int: + """Return the number of available devices. + + Returns: + 1 always for the stub. + """ + return 1 + + def current_device(self) -> int: + """Return the index of the currently selected device. + + Returns: + 0 always for the stub. + """ + return 0 + + def set_device(self, device: Any) -> None: + """Select the given device. + + Args: + device: The device index or identifier to select. + Ignored in the stub. + """ + return None + + def get_device_properties(self, device: Any = 0) -> StubDeviceProperties: + """Return device properties for the given device. + + Args: + device: The device index or identifier to query. + Ignored in the stub. + + Returns: + A StubDeviceProperties instance with default values. + """ + return StubDeviceProperties() + + def empty_cache(self) -> None: + """Release all unoccupied cached memory. + + No-op in the stub implementation. + """ + return None + + def __getattr__(self, name: str) -> Any: + raise AttributeError(f"StubCPUDevice does not implement '{name}'") + + def __repr__(self) -> str: + return f"StubCPUDevice(device_type={self._device_type})" diff --git a/tests/v1/storage_backend/test_path_sharder.py b/tests/v1/storage_backend/test_path_sharder.py index 7718d2e2396..70da60620ed 100644 --- a/tests/v1/storage_backend/test_path_sharder.py +++ b/tests/v1/storage_backend/test_path_sharder.py @@ -95,7 +95,10 @@ def test_strips_whitespace(self): for d in dirs: shutil.rmtree(d, ignore_errors=True) - @patch("torch.cuda.is_available", return_value=False) + @patch( + "lmcache.v1.storage_backend.path_sharder.torch_dev.is_available", + return_value=False, + ) def test_cpu_device_selects_first_path(self, _avail): dirs = [tempfile.mkdtemp() for _ in range(2)] try: @@ -118,10 +121,16 @@ def test_all_paths_returns_copy(self): # -- device-resolution edge cases (exercised via public API) ----------- - @patch("torch.cuda.is_available", return_value=True) - @patch("torch.cuda.current_device", return_value=1) + @patch( + "lmcache.v1.storage_backend.path_sharder.torch_dev.is_available", + return_value=True, + ) + @patch( + "lmcache.v1.storage_backend.path_sharder.torch_dev.current_device", + return_value=1, + ) def test_bare_cuda_uses_current_device(self, _cur, _avail): - """Bare 'cuda' resolves to torch.cuda.current_device().""" + """Bare 'cuda' resolves to torch_dev.current_device().""" dirs = [tempfile.mkdtemp() for _ in range(3)] try: csv = ",".join(dirs) @@ -131,7 +140,10 @@ def test_bare_cuda_uses_current_device(self, _cur, _avail): for d in dirs: shutil.rmtree(d, ignore_errors=True) - @patch("torch.cuda.is_available", return_value=False) + @patch( + "lmcache.v1.storage_backend.path_sharder.torch_dev.is_available", + return_value=False, + ) def test_bare_cuda_no_gpu_selects_first(self, _avail): """Bare 'cuda' with no GPU falls back to device 0.""" dirs = [tempfile.mkdtemp() for _ in range(3)] @@ -143,8 +155,14 @@ def test_bare_cuda_no_gpu_selects_first(self, _avail): for d in dirs: shutil.rmtree(d, ignore_errors=True) - @patch("torch.cuda.is_available", return_value=True) - @patch("torch.cuda.current_device", return_value=2) + @patch( + "lmcache.v1.storage_backend.path_sharder.torch_dev.is_available", + return_value=True, + ) + @patch( + "lmcache.v1.storage_backend.path_sharder.torch_dev.current_device", + return_value=2, + ) def test_cpu_device_always_selects_first(self, _cur, _avail): """'cpu' always resolves to index 0, even when CUDA is available.""" dirs = [tempfile.mkdtemp() for _ in range(3)] @@ -156,8 +174,14 @@ def test_cpu_device_always_selects_first(self, _cur, _avail): for d in dirs: shutil.rmtree(d, ignore_errors=True) - @patch("torch.cuda.is_available", return_value=True) - @patch("torch.cuda.current_device", return_value=2) + @patch( + "lmcache.v1.storage_backend.path_sharder.torch_dev.is_available", + return_value=True, + ) + @patch( + "lmcache.v1.storage_backend.path_sharder.torch_dev.current_device", + return_value=2, + ) def test_malformed_device_empty_index_falls_back(self, _cur, _avail): """'cuda:' (no int) falls back to current_device.""" dirs = [tempfile.mkdtemp() for _ in range(3)] @@ -169,7 +193,10 @@ def test_malformed_device_empty_index_falls_back(self, _cur, _avail): for d in dirs: shutil.rmtree(d, ignore_errors=True) - @patch("torch.cuda.is_available", return_value=False) + @patch( + "lmcache.v1.storage_backend.path_sharder.torch_dev.is_available", + return_value=False, + ) def test_malformed_device_non_numeric_falls_back(self, _avail): """'cuda:foo' falls back to 0 when CUDA is unavailable.""" dirs = [tempfile.mkdtemp() for _ in range(3)] From c9cfa3af12422a513a0c3cf7c60d143957ee8514 Mon Sep 17 00:00:00 2001 From: kumaneko Date: Mon, 18 May 2026 23:50:09 +0800 Subject: [PATCH 25/69] Fix typos and misspellings across codebase (#3306) - 'non-blcocking' -> 'non-blocking' in abstract_backend.py - 'Memoryobjs' -> 'MemoryObj' in abstract_backend.py - 'pareant_allocator' -> 'parent_allocator' in memory_management.py - 'storage-manger' -> 'storage-manager' in storage_manager.py - 'trak' -> 'track' in executor.py (2 occurrences) - 'the the' -> 'the', 'instance' -> 'instances' in kv_controller.py Signed-off-by: rigginschen Co-authored-by: rigginschen --- lmcache/v1/cache_controller/controllers/kv_controller.py | 2 +- lmcache/v1/cache_controller/executor.py | 4 ++-- lmcache/v1/memory_management.py | 2 +- lmcache/v1/storage_backend/abstract_backend.py | 4 ++-- lmcache/v1/storage_backend/storage_manager.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lmcache/v1/cache_controller/controllers/kv_controller.py b/lmcache/v1/cache_controller/controllers/kv_controller.py index c6878ec0d48..80c7f3c7640 100644 --- a/lmcache/v1/cache_controller/controllers/kv_controller.py +++ b/lmcache/v1/cache_controller/controllers/kv_controller.py @@ -45,7 +45,7 @@ """ The kv controller use `(instance_id, worker_id)` -> [location -> set[chunk_hash]] -as kv_pool. When the the number of instance is small and stable, the time complexity +as kv_pool. When the number of instances is small and stable, the time complexity of `lookup` in kv controller is O(n). If the number of instance is large or unknown, the time complexity will degrade to O(n^2), and the ReverseIndexKVController is a better choice. diff --git a/lmcache/v1/cache_controller/executor.py b/lmcache/v1/cache_controller/executor.py index 1b54cd17282..a78d19710df 100644 --- a/lmcache/v1/cache_controller/executor.py +++ b/lmcache/v1/cache_controller/executor.py @@ -79,7 +79,7 @@ async def clear(self, msg: ClearMsg) -> Union[ClearRetMsg, ErrorMsg]: ) sockets.append(socket) - # TODO(Jiayi): Need a way to trak event_id -> worker_event_id mapping + # TODO(Jiayi): Need a way to track event_id -> worker_event_id mapping # Also, we need to track worker_event_id status worker_event_id = f"Worker{worker_id}{msg.event_id}" serialized_msg = msgspec.msgpack.encode( @@ -128,7 +128,7 @@ async def pin(self, msg: PinMsg) -> Union[PinRetMsg, ErrorMsg]: ) sockets.append(socket) - # TODO(Jiayi): Need a way to trak event_id -> worker_event_id mapping + # TODO(Jiayi): Need a way to track event_id -> worker_event_id mapping # Also, we need to track worker_event_id status worker_event_id = f"Worker{worker_id}{msg.event_id}" serialized_msg = msgspec.msgpack.encode( diff --git a/lmcache/v1/memory_management.py b/lmcache/v1/memory_management.py index 164cb058f6f..9e1a5182b3c 100644 --- a/lmcache/v1/memory_management.py +++ b/lmcache/v1/memory_management.py @@ -1706,7 +1706,7 @@ def batched_allocate( self.batched_free(allocated_blocks, update_stats=False) return None - # FIXME: think about whether pareant_allocator + # FIXME: think about whether parent_allocator # should be updated here. free_block.meta.shape = shapes[0] free_block.meta.dtype = dtypes[0] diff --git a/lmcache/v1/storage_backend/abstract_backend.py b/lmcache/v1/storage_backend/abstract_backend.py index f3d89e91419..1b7b15a4ea4 100644 --- a/lmcache/v1/storage_backend/abstract_backend.py +++ b/lmcache/v1/storage_backend/abstract_backend.py @@ -165,11 +165,11 @@ async def batched_get_non_blocking( transfer_spec: Any = None, ) -> list[MemoryObj]: """ - A non-blcocking function to get the kv cache from the storage backend. + A non-blocking function to get the kv cache from the storage backend. :param list[CacheEngineKey] keys: The keys of the list of MemoryObjs. - :return: a list of Memoryobjs. + :return: a list of MemoryObj. """ raise NotImplementedError diff --git a/lmcache/v1/storage_backend/storage_manager.py b/lmcache/v1/storage_backend/storage_manager.py index 6ce6156031d..156ba97094f 100644 --- a/lmcache/v1/storage_backend/storage_manager.py +++ b/lmcache/v1/storage_backend/storage_manager.py @@ -235,7 +235,7 @@ def __init__( self.thread = threading.Thread( target=start_loop_in_thread_with_exceptions, args=(self.loop,), - name="storage-manger-event-loop", + name="storage-manager-event-loop", ) self.thread.start() From 55bcefd9c5a5e025bca8790bbf13e1ef0d10e385 Mon Sep 17 00:00:00 2001 From: Ilya Yanok Date: Mon, 18 May 2026 18:22:23 +0200 Subject: [PATCH 26/69] nixl_storage: naive support for files + dynamic (#2936) * nixl_storage: naive support for files + dynamic static is not actually very usable with files: we bump into the OS limits on the open files very quickly, limiting the size of the cache. With tons of VRAM, having G3 of comparable size is not helping much, we really need it to be much bigger. This commit adds support for files in dynamic mode of nixl storage. There are some naive things done: 1. No support for sharing the cache storage. We assume the worker has exclusive access to the files and it is always safe to overwrite existing files. Note that with TP!=1 we will have several workers with the same target directory, that's why it's important to have a worker id as part of the key, so they don't affect each other. 2. No eviction support. As before, dynamic mode has no evicting support. This kinda made sense when it was only working with OBJ storages, but now that is something to be aware of. It's not hard to add eviction though. 3. Flat directory with all the cache files. There are going to be a lot of them, especially provided we don't have eviction. Most filesystems don't optimize for the case of a directory with millions of files, and we constantly open/close files there, that might be a source of additional latency. - There is an existing PR to support sharding the directory, we can merge it as a band aid. - As a more advanced solution, we can do multi-layer subdirectory structure, like gds backend does. 4. We switched directly from having _all_ files open at once, to open/close _every_ single file on access. We might explore having an LRU cache of open files instead. Signed-off-by: Ilya Yanok * Fix formatting Signed-off-by: Guy Ealey Morag * Fix PR Comments Signed-off-by: Guy Ealey Morag * Extract _build_descs Signed-off-by: Guy Ealey Morag * Fix test Signed-off-by: Guy Ealey Morag * Refactor to increase readability Signed-off-by: Guy Ealey Morag * Fix another leak issue Signed-off-by: Guy Ealey Morag * Fix release order Signed-off-by: Guy Ealey Morag * Treat file not found as a miss Signed-off-by: Guy Ealey Morag * Add docstring about file create mode 0o644 Signed-off-by: Guy Ealey Morag * Unlink files on failure Signed-off-by: Guy Ealey Morag * Add missing docs Signed-off-by: Guy Ealey Morag * Fix pre-commit issues Signed-off-by: Guy Ealey Morag * Fix mypy error Signed-off-by: Guy Ealey Morag * Fix test Signed-off-by: Guy Ealey Morag * Make tests pass Signed-off-by: Guy Ealey Morag * Fix formatting Signed-off-by: Guy Ealey Morag --------- Signed-off-by: Ilya Yanok Signed-off-by: Guy Ealey Morag Co-authored-by: Guy Ealey Morag --- .../storage_backend/nixl_storage_backend.py | 447 +++++++++++++----- tests/v1/test_nixl_storage.py | 268 ++++++++++- 2 files changed, 590 insertions(+), 125 deletions(-) diff --git a/lmcache/v1/storage_backend/nixl_storage_backend.py b/lmcache/v1/storage_backend/nixl_storage_backend.py index ae8a8bc9c90..856f351d8dc 100644 --- a/lmcache/v1/storage_backend/nixl_storage_backend.py +++ b/lmcache/v1/storage_backend/nixl_storage_backend.py @@ -16,7 +16,7 @@ # Standard from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Any, Callable, List, Optional, Sequence, Set, Union, cast +from typing import Any, Callable, List, Optional, Sequence, Set, Tuple, Union, cast from urllib.parse import quote as url_quote import asyncio import os @@ -70,6 +70,10 @@ logger = init_logger(__name__) +# POSIX permission mode for files created via ``os.open()`` with ``O_CREAT``. +# 0o644 = rw-r--r-- (owner read/write, group/others read-only). +DEFAULT_FILE_CREATE_MODE = 0o644 + # Max concurrency for parallel S3 HEAD requests in batched_contains(). _CONTAINS_BATCH_SIZE = 16 @@ -90,21 +94,14 @@ class NixlStorageConfig: sync_mode: Optional[Any] # nixl_thread_sync_t, None if unsupported @staticmethod - def validate_nixl_backend(dynamic_storage: bool, backend: str, device: str): - if dynamic_storage: # For now only supports OBJ and AZURE_BLOB backend - if backend in ("OBJ",): - return device == "cpu" or device == "cuda" - elif backend in ("AZURE_BLOB",): - return device == "cpu" - else: - return False + def validate_nixl_backend(backend: str, device: str) -> bool: + device = device.split(":", 1)[0] + if backend in ("GDS", "GDS_MT", "OBJ"): + return device == "cpu" or device == "cuda" + elif backend in ("POSIX", "HF3FS", "AZURE_BLOB"): + return device == "cpu" else: - if backend in ("GDS", "GDS_MT", "OBJ"): - return device == "cpu" or device == "cuda" - elif backend in ("POSIX", "HF3FS", "AZURE_BLOB"): - return device == "cpu" - else: - return False + return False @staticmethod def from_cache_engine_config( @@ -201,9 +198,12 @@ def from_cache_engine_config( config.nixl_buffer_size = buffer_size assert NixlStorageConfig.validate_nixl_backend( - dynamic_storage, backend, config.nixl_buffer_device + backend, config.nixl_buffer_device ), "Invalid NIXL backend & device combination" + if backend in ("GDS", "GDS_MT", "POSIX", "HF3FS"): + assert path is not None, f"nixl_path must be provided for {backend} backend" + return NixlStorageConfig( buffer_size=config.nixl_buffer_size, pool_size=pool_size, @@ -252,6 +252,7 @@ def __init__(self, size: int, path: str, use_direct_io: bool): self.fds: List[int] = [] assert path is not None + os.makedirs(path, exist_ok=True) flags = os.O_CREAT | os.O_RDWR if use_direct_io: @@ -265,7 +266,7 @@ def __init__(self, size: int, path: str, use_direct_io: bool): for i in reversed(range(size)): filename = f"obj_{i}_{uuid.uuid4().hex[0:4]}.bin" tmp_path = os.path.join(path, filename) - fd = os.open(tmp_path, flags) + fd = os.open(tmp_path, flags, DEFAULT_FILE_CREATE_MODE) self.fds.append(fd) def close(self): @@ -312,6 +313,32 @@ def contains(self, key: int) -> bool: class NixlDesc: device_id: int meta_info: str + path: Optional[str] = None + + +def _close_file_descs(descs: List[NixlDesc]) -> None: + """Best-effort close of the FDs in descs.""" + for d in descs: + try: + os.close(d.device_id) + except OSError: + pass + + +def _unlink_file_descs(descs: List[NixlDesc]) -> None: + """ + Best-effort unlink of every desc whose ``path`` is set. + + Called only on FILE-write failure paths to remove the (empty or + partially-written) file we created with ``O_CREAT``. + """ + for d in descs: + if d.path is None: + continue + try: + os.unlink(d.path) + except OSError: + pass @dataclass @@ -546,8 +573,7 @@ def __init__( if backend in ("OBJ", "AZURE_BLOB"): self.mem_type = "OBJ" else: - # Already validated in validate_nixl_backend - raise ValueError(f"unexpected backend: {backend}") + self.mem_type = "FILE" def create_batched_storage_handler(self, descs: list[NixlDesc], page_size: int): reg_list = [] @@ -569,12 +595,36 @@ def post_async(self, handle: NixlXferHandle): state = self.nixl_agent.transfer(handle) return state - def release_storage_handler(self, reg_descs, xfer_handler): - """Release storage handler resources""" + def release_storage_handler( + self, + reg_descs: nixlBind.nixlRegDList, + xfer_handler: NixlDlistHandle, + descs: List[NixlDesc], + ) -> None: + """ + Release storage handler resources. + + :param reg_descs: Memory descriptors to deregister. + :param xfer_handler: Transfer dlist handle to release. + :param descs: Descriptors used for this transfer. + """ self.nixl_agent.release_dlist_handle(xfer_handler) self.nixl_agent.deregister_memory(reg_descs) + if self.mem_type == "FILE": + _close_file_descs(descs) + + def nixl_desc_exists(self, meta_info: str, path: str) -> bool: + """ + Check whether a NIXL descriptor exists in storage. + + :param meta_info: Descriptor key (file basename for FILE backends, + object key for OBJ backends). + :param path: Directory for FILE backends; ignored for OBJ. + :return: ``True`` if present, ``False`` otherwise. + """ + if self.mem_type == "FILE": + return os.path.exists(os.path.join(path, meta_info)) - def nixl_desc_exists(self, meta_info: str) -> bool: reg_list = [(0, 0, 0, meta_info)] try: resp = self.nixl_agent.query_memory( @@ -1107,7 +1157,16 @@ def __init__( self.async_mode = nixl_config.enable_async_put self.enable_presence_cache = nixl_config.enable_presence_cache - + self.path = nixl_config.path + self.direct_io_flag = 0 + if nixl_config.use_direct_io: + if hasattr(os, "O_DIRECT"): + self.direct_io_flag = os.O_DIRECT + else: + logger.warning( + "use_direct_io is True, but O_DIRECT is not available on " + "this system. Falling back to buffered I/O." + ) # Presence cache to reduce remote contains checks self.hit_counter = 0 self.total_counter = 0 @@ -1213,36 +1272,123 @@ def _format_object_key(self, key: CacheEngineKey) -> str: Similar to s3_connector._format_safe_path() """ key_str = key.to_string() - # Replace slashes with underscores to make it safe for object storage + # Replace slashes with underscores to make it safe for object storage/FS flat_key_str = key_str.replace("/", "_").replace("@", "_") # URL encode for safety return url_quote(flat_key_str, safe="") - def key_exists(self, key: CacheEngineKey) -> bool: + def _build_descs( + self, keys: Sequence[CacheEngineKey], *, write: bool + ) -> List[NixlDesc]: + """ + Build NixlDescs for ``keys``. For FILE backends this opens one fd per + key; the caller owns FD lifetime once this method returns successfully. + On mid-loop failure, every already-opened fd is closed before the exception + is re-raised (for write paths, files are also unlinked). + + :param write: If True, opens with O_CREAT | O_RDWR (mem_to_storage). + If False, opens with O_RDONLY (storage_to_mem). + """ if self.agent.mem_type == "OBJ": - meta_info = self._format_object_key(key) - else: - # Already validated in validate_nixl_backend - raise ValueError(f"unexpected mem_type: {self.agent.mem_type}") + device_ids = self._alloc_device_ids(len(keys)) + return [ + NixlDesc(device_id=device_ids[i], meta_info=self._format_object_key(k)) + for i, k in enumerate(keys) + ] + if self.agent.mem_type == "FILE": + flags = (os.O_CREAT | os.O_RDWR) if write else os.O_RDONLY + flags |= self.direct_io_flag + mode_args = (DEFAULT_FILE_CREATE_MODE,) if write else () + descs: List[NixlDesc] = [] + try: + for k in keys: + path = os.path.join(self.path, self._format_object_key(k)) + fd = os.open(path, flags, *mode_args) + descs.append( + NixlDesc( + device_id=fd, + meta_info="", + path=path if write else None, + ) + ) + return descs + except OSError: + _close_file_descs(descs) + _unlink_file_descs(descs) + raise + # Already validated in validate_nixl_backend + raise ValueError(f"unexpected mem_type: {self.agent.mem_type}") - return self.agent.nixl_desc_exists(meta_info) + def _acquire_storage_handle( + self, + keys: Sequence[CacheEngineKey], + mem_indices: List[int], + storage_indices: Sequence[int], + page_size: int, + write: bool, + ) -> Tuple[ + List[NixlDesc], + nixlBind.nixlRegDList, + NixlDlistHandle, + NixlXferHandle, + ]: + """Open FDs, register the storage handler, and build the transfer handle. + + On any failure, releases everything already acquired (FDs, NIXL + state, and any FILE-write files created with ``O_CREAT``) before + re-raising, so the caller either gets a fully-acquired tuple or + an exception with nothing leaked. + """ + descs = self._build_descs(keys, write=write) + try: + reg_descs, xfer_handler = self.agent.create_batched_storage_handler( + descs, page_size + ) + except Exception: + if self.agent.mem_type == "FILE": + _close_file_descs(descs) + _unlink_file_descs(descs) + raise - def storage_to_mem( - self, keys: list[CacheEngineKey], pin: bool = False - ) -> list[Optional[MemoryObj]]: - obj_list: list[Optional[MemoryObj]] = [] - page_size = self.memory_allocator.align_bytes - start_time = time.time() - storage_indices = [] - mem_indices = [] - descs = [] + try: + if write: + handle = self.agent.get_mem_to_storage_handle( + mem_indices, xfer_handler, storage_indices + ) + else: + handle = self.agent.get_storage_to_mem_handle( + mem_indices, xfer_handler, storage_indices + ) + except Exception: + # release_storage_handler closes the FDs and releases the dlist. + self.agent.release_storage_handler(reg_descs, xfer_handler, descs) + if self.agent.mem_type == "FILE": + _unlink_file_descs(descs) + raise + + return descs, reg_descs, xfer_handler, handle + + def key_exists(self, key: CacheEngineKey) -> bool: + meta_info = self._format_object_key(key) + + return self.agent.nixl_desc_exists(meta_info, self.path) - # Prepare mem and storage indices + def _allocate_for_read( + self, keys: Sequence[CacheEngineKey] + ) -> Tuple[Optional[List[MemoryObj]], List[int], List[int]]: + """Allocate one MemoryObj per key. + + Returns ``(None, [], [])`` if any allocation fails, after freeing + what was already taken; otherwise returns the allocated objects + and the parallel mem/storage index lists. + """ + assert self.meta_shape is not None + assert self.meta_dtype is not None + assert self.meta_fmt is not None + obj_list: List[MemoryObj] = [] + mem_indices: List[int] = [] + storage_indices: List[int] = [] for idx in range(len(keys)): - # Allocate memory for the object - assert self.meta_shape is not None - assert self.meta_dtype is not None - assert self.meta_fmt is not None obj = self.memory_allocator.allocate( self.meta_shape, self.meta_dtype, self.meta_fmt ) @@ -1254,56 +1400,58 @@ def storage_to_mem( for obj in obj_list: if obj is not None: obj.ref_count_down() - return [None] * len(keys) - + return None, [], [] obj_list.append(obj) mem_indices.append(obj.meta.address) storage_indices.append(idx) + return obj_list, mem_indices, storage_indices - if self.agent.mem_type == "OBJ": - device_ids = self._alloc_device_ids(len(keys)) - for idx in range(len(keys)): - object_key = self._format_object_key(keys[idx]) - descs.append(NixlDesc(device_id=device_ids[idx], meta_info=object_key)) - else: - # Already validated in validate_nixl_backend - raise ValueError(f"unexpected mem_type: {self.agent.mem_type}") + def storage_to_mem( + self, keys: list[CacheEngineKey], pin: bool = False + ) -> list[Optional[MemoryObj]]: + page_size = self.memory_allocator.align_bytes + start_time = time.time() - # Create batched storage handler - storage_reg_descs, storage_xfer_handler = ( - self.agent.create_batched_storage_handler(descs, page_size) - ) - # Create transfer handle - handle = self.agent.get_storage_to_mem_handle( - mem_indices, storage_xfer_handler, storage_indices - ) + obj_list, mem_indices, storage_indices = self._allocate_for_read(keys) + if obj_list is None: + return [None] * len(keys) - # Try to read the object try: - self.agent.post_blocking(handle) - xfer_state = True - except nixlBind.nixlBackendError as exc: - logger.warning(f"Batch Transfer failed: {exc}") - # Treat "not found", timeout or other transfer failures as recoverable - # Do not raise exception to avoid terminating the program - xfer_state = False + descs, reg_descs, xfer_handler, handle = self._acquire_storage_handle( + keys, mem_indices, storage_indices, page_size, write=False + ) + except FileNotFoundError: + # FILE backend: at least one key's file does not exist, + # treat the whole batch as a miss. + logger.warning("storage_to_mem: missing file in FILE backend") + for obj in obj_list: + if obj is not None: + obj.ref_count_down() + for key in keys: + self._cache_discard(key.chunk_hash) + return [None] * len(keys) - self.agent.release_handle(handle) - self.agent.release_storage_handler(storage_reg_descs, storage_xfer_handler) + try: + try: + self.agent.post_blocking(handle) + xfer_state = True + except nixlBind.nixlBackendError as exc: + logger.warning(f"Batch Transfer failed: {exc}") + # Treat transfer failures (not found, timeout, etc.) as a + # miss rather than raising and terminating the program. + xfer_state = False + finally: + self.agent.release_handle(handle) + self.agent.release_storage_handler(reg_descs, xfer_handler, descs) + except Exception: + # Acquisition or transfer raised; return the allocated MemoryObj + # slots to the allocator so they aren't leaked. + for obj in obj_list: + if obj is not None: + obj.ref_count_down() + raise - if xfer_state: - for i in range(len(keys)): - key = keys[i] - self._cache_add(key.chunk_hash) - end_time = time.time() - duration = end_time - start_time - logger.debug( - f"storage_to_mem for {len(keys)} objects size {page_size * len(keys)} " - f"took {duration:.6f} seconds" - ) - return obj_list - else: - # Release the allocated memory and discard cache if transfer failed + if not xfer_state: for obj in obj_list: if obj is not None: obj.ref_count_down() @@ -1311,6 +1459,15 @@ def storage_to_mem( self._cache_discard(key.chunk_hash) return [None] * len(keys) + for key in keys: + self._cache_add(key.chunk_hash) + duration = time.time() - start_time + logger.debug( + f"storage_to_mem for {len(keys)} objects size " + f"{page_size * len(keys)} took {duration:.6f} seconds" + ) + return cast(list[Optional[MemoryObj]], obj_list) + async def _wait_for_transfer( self, handle: NixlXferHandle, @@ -1318,9 +1475,15 @@ async def _wait_for_transfer( keys: Sequence[CacheEngineKey], storage_reg_descs: nixlBind.nixlRegDList, storage_xfer_handler: NixlDlistHandle, + descs: List[NixlDesc], mem_objs: List[MemoryObj], ): - """Asynchronously wait for transfer to complete without blocking.""" + """Asynchronously wait for transfer to complete without blocking. + + On any non-DONE outcome (``ERR`` or earlier exception), unlink any + FILE-write files just created at the final key path so that + ``contains()`` does not observe them as bogus hits. + """ state = "" try: state = initial_state @@ -1333,13 +1496,17 @@ async def _wait_for_transfer( finally: # Release the handle after transfer completes (success or failure) self.agent.release_handle(handle) - self.agent.release_storage_handler(storage_reg_descs, storage_xfer_handler) + self.agent.release_storage_handler( + storage_reg_descs, storage_xfer_handler, descs + ) if state == "DONE": for key in keys: with self.progress_lock: self.progress_set.discard(key) self._cache_add(key.chunk_hash) + elif self.agent.mem_type == "FILE": + _unlink_file_descs(descs) for mem_obj in mem_objs: mem_obj.ref_count_down() @@ -1347,61 +1514,90 @@ async def _wait_for_transfer( async def mem_to_storage( self, keys: Sequence[CacheEngineKey], mem_objs: List[MemoryObj] ) -> None: - start_time = time.time() - if len(keys) == 0: + if not keys: return + page_size = self.memory_allocator.align_bytes storage_indices = range(len(keys)) mem_indices = [mem_obj.meta.address for mem_obj in mem_objs] - page_size = self.memory_allocator.align_bytes - descs = [] - if self.agent.mem_type == "OBJ": - device_ids = self._alloc_device_ids(len(keys)) - for idx in range(len(keys)): - object_key = self._format_object_key(keys[idx]) - descs.append(NixlDesc(device_id=device_ids[idx], meta_info=object_key)) - else: - # Already validated in validate_nixl_backend - raise ValueError(f"unexpected mem_type: {self.agent.mem_type}") - - storage_reg_descs, storage_xfer_handler = ( - self.agent.create_batched_storage_handler(descs, page_size) - ) - - handle = self.agent.get_mem_to_storage_handle( - mem_indices, storage_xfer_handler, storage_indices + descs, reg_descs, xfer_handler, handle = self._acquire_storage_handle( + keys, mem_indices, storage_indices, page_size, write=True ) if self.async_mode: + self._submit_async_mem_to_storage( + handle, keys, reg_descs, xfer_handler, descs, mem_objs + ) + else: + self._run_sync_mem_to_storage( + handle, keys, reg_descs, xfer_handler, descs, page_size + ) + + def _submit_async_mem_to_storage( + self, + handle: NixlXferHandle, + keys: Sequence[CacheEngineKey], + reg_descs: nixlBind.nixlRegDList, + xfer_handler: NixlDlistHandle, + descs: List[NixlDesc], + mem_objs: List[MemoryObj], + ) -> None: + """Post the transfer and hand cleanup off to a background task. + + On any failure before the task is scheduled, ownership has not + transferred, so we release everything ourselves -- including any + FILE-write files just created at the final key path. + """ + try: initial_state = self.agent.post_async(handle) - # Submit the async wait to the event loop and return immediately asyncio.create_task( self._wait_for_transfer( handle, initial_state, keys, - storage_reg_descs, - storage_xfer_handler, + reg_descs, + xfer_handler, + descs, mem_objs, ) ) - else: - self.agent.post_blocking(handle) + except Exception: self.agent.release_handle(handle) - self.agent.release_storage_handler(storage_reg_descs, storage_xfer_handler) + self.agent.release_storage_handler(reg_descs, xfer_handler, descs) + if self.agent.mem_type == "FILE": + _unlink_file_descs(descs) + raise - end_time = time.time() - duration = end_time - start_time - logger.debug( - f"mem_to_storage for {len(keys)} objects size {page_size * len(keys)} " - f"took {duration:.3f} seconds" - ) + def _run_sync_mem_to_storage( + self, + handle: NixlXferHandle, + keys: Sequence[CacheEngineKey], + reg_descs: nixlBind.nixlRegDList, + xfer_handler: NixlDlistHandle, + descs: List[NixlDesc], + page_size: int, + ) -> None: + start_time = time.time() + try: + self.agent.post_blocking(handle) + except Exception: + if self.agent.mem_type == "FILE": + _unlink_file_descs(descs) + raise + finally: + self.agent.release_handle(handle) + self.agent.release_storage_handler(reg_descs, xfer_handler, descs) - for key in keys: - with self.progress_lock: - self.progress_set.discard(key) - self._cache_add(key.chunk_hash) + duration = time.time() - start_time + logger.debug( + f"mem_to_storage for {len(keys)} objects size " + f"{page_size * len(keys)} took {duration:.3f} seconds" + ) + for key in keys: + with self.progress_lock: + self.progress_set.discard(key) + self._cache_add(key.chunk_hash) def exists_in_put_tasks(self, key: CacheEngineKey) -> bool: """ @@ -1635,8 +1831,15 @@ def remove(self, key: CacheEngineKey, force: bool = True) -> bool: :param key: The key to remove. :param force: Whether to force removal (not used in this implementation) + :return: True if the key is removed, False otherwise. """ self._cache_discard(key.chunk_hash) + if self.agent.mem_type == "FILE": + try: + os.unlink(os.path.join(self.path, self._format_object_key(key))) + except FileNotFoundError: + return False + return True def pin(self, key: CacheEngineKey) -> bool: diff --git a/tests/v1/test_nixl_storage.py b/tests/v1/test_nixl_storage.py index 740aa57a9a9..634c6484cb7 100644 --- a/tests/v1/test_nixl_storage.py +++ b/tests/v1/test_nixl_storage.py @@ -2,6 +2,8 @@ # Standard from pathlib import Path import asyncio +import os +import sys import threading # Third Party @@ -83,7 +85,7 @@ def run(config: LMCacheEngineConfig, shape, dtype): assert not nixl_backend.contains(key, False) assert not nixl_backend.exists_in_put_tasks(key) - obj = nixl_backend.memory_allocator.allocate(shape=shape, dtype=dtype) + obj = nixl_backend.memory_allocator.allocate(torch.Size(shape), dtype) assert obj is not None assert obj.tensor is not None objs.append(obj) @@ -110,7 +112,12 @@ def run(config: LMCacheEngineConfig, shape, dtype): assert returned_memory_obj.get_shape() == obj.get_shape() assert returned_memory_obj.get_dtype() == obj.get_dtype() assert returned_memory_obj.metadata.address != obj.metadata.address - assert torch.equal(returned_memory_obj.tensor, obj.tensor) + + returned_tensor = returned_memory_obj.tensor + obj_tensor = obj.tensor + assert returned_tensor is not None + assert obj_tensor is not None + assert torch.equal(returned_tensor, obj_tensor) obj_list = asyncio.run( nixl_backend.batched_get_non_blocking(lookup_id="test", keys=first_keys) @@ -123,7 +130,12 @@ def run(config: LMCacheEngineConfig, shape, dtype): assert returned_memory_obj.get_shape() == obj.get_shape() assert returned_memory_obj.get_dtype() == obj.get_dtype() assert returned_memory_obj.metadata.address != obj.metadata.address - assert torch.equal(returned_memory_obj.tensor, obj.tensor) + + returned_tensor = returned_memory_obj.tensor + obj_tensor = obj.tensor + assert returned_tensor is not None + assert obj_tensor is not None + assert torch.equal(returned_tensor, obj_tensor) def test_eviction(new_idx, old_idx): nixl_backend.batched_submit_put_task([keys[new_idx]], [objs[new_idx]]) @@ -305,3 +317,253 @@ def test_nixl_posix_backend(): config.extra_config["enable_cuda"] = False run(config, shape, dtype) + + +_DYNAMIC_KV_SHAPE = (4, 2, 256, 8, 128) + + +def _build_dynamic_file_backend(config, dtype): + """ + Build a NixlStorageBackend in dynamic-FILE mode and the surrounding + event-loop thread. Returns (backend, backends, thread_loop, thread, keys, + objs) so the caller can drive the test and tear everything down via + ``_teardown_dynamic_file_backend``. + """ + BACKEND_NAME = "NixlStorageBackend" + + keys = [ + create_key("e3229141e680fb413d2c5d3ebb416c4ad300d381e309fc9e417757b91406c157"), + create_key("e3229141e680fb413d2c5d3ebb416c4ad300d381e309fc9e417757b91406d268"), + ] + + thread_loop = asyncio.new_event_loop() + thread = threading.Thread(target=thread_loop.run_forever) + thread.start() + + metadata = LMCacheMetadata( + model_name="Llama-3.1-70B-Instruct", + world_size=1, + local_world_size=1, + worker_id=0, + local_worker_id=0, + kv_dtype=dtype, + kv_shape=_DYNAMIC_KV_SHAPE, + ) + + backends = CreateStorageBackends( + config, + metadata, + thread_loop, + dst_device=config.nixl_buffer_device, + ) + nixl_backend = backends[BACKEND_NAME] + assert isinstance(nixl_backend, NixlStorageBackend) + + # In dynamic mode the backend internally allocates with meta_shape + # (derived from kv_shape via init_chunk_meta), so allocate the test + # objects with the same shape so put/get round-trip shapes match. + obj_shape = nixl_backend.meta_shape + obj_dtype = nixl_backend.meta_dtype + assert obj_shape is not None + assert obj_dtype is not None + + obj_fmt = nixl_backend.meta_fmt + assert obj_fmt is not None + + objs = [] + for _ in keys: + obj = nixl_backend.memory_allocator.allocate(obj_shape, obj_dtype, obj_fmt) + assert obj is not None + assert obj.tensor is not None + objs.append(obj) + + objs[0].tensor.zero_() + objs[1].tensor.zero_() + objs[0].tensor[0, 0, 100, 200] = 1 + objs[1].tensor[1, 0, 50, 300] = 1 + + return nixl_backend, backends, thread_loop, thread, keys, objs + + +def _teardown_dynamic_file_backend(backends, thread_loop, thread, objs=()): + for obj in objs: + if obj is None: + continue + if obj.is_valid() and obj.get_ref_count() > 0: + obj.ref_count_down() + for backend in backends.values(): + backend.close() + if thread_loop and thread_loop.is_running(): + thread_loop.call_soon_threadsafe(thread_loop.stop) + if thread and thread.is_alive(): + thread.join() + + +def run_dynamic_file(config, dtype, tmp_path): + """ + Exercise the dynamic-FILE backend's new code paths: contains/key_exists, + put/get round-trip, and remove for both present and missing files. + """ + nixl_backend, backends, thread_loop, thread, keys, objs = ( + _build_dynamic_file_backend(config, dtype) + ) + + retained_objs = list(objs) + + try: + for key in keys: + assert not nixl_backend.contains(key, False) + assert not nixl_backend.exists_in_put_tasks(key) + + nixl_backend.batched_submit_put_task(keys, objs) + + for key in keys: + assert nixl_backend.contains(key, False) + + files_after_put = set(os.listdir(str(tmp_path))) + expected_files = {nixl_backend._format_object_key(k) for k in keys} + assert expected_files.issubset(files_after_put), ( + f"missing files in {tmp_path}: {expected_files - files_after_put}" + ) + + for key, obj in zip(keys, objs, strict=False): + returned = nixl_backend.get_blocking(key) + assert returned is not None + retained_objs.append(returned) + assert returned.get_size() == obj.get_size() + assert returned.get_shape() == obj.get_shape() + assert returned.get_dtype() == obj.get_dtype() + assert torch.equal(returned.tensor, obj.tensor) + + first_remove = nixl_backend.remove(keys[0]) + assert first_remove is True + assert not os.path.exists( + os.path.join(str(tmp_path), nixl_backend._format_object_key(keys[0])) + ) + + # Removing an already-gone file must return False + # instead of raising FileNotFoundError. + second_remove = nixl_backend.remove(keys[0]) + assert second_remove is False + finally: + _teardown_dynamic_file_backend(backends, thread_loop, thread, retained_objs) + + +@pytest.mark.no_shared_allocator +def test_nixl_posix_dynamic_file_backend(tmp_path): + BASE_DIR = Path(__file__).parent + config = LMCacheEngineConfig.from_file(BASE_DIR / "data/nixl.yaml") + + dtype = torch.bfloat16 + + config.nixl_buffer_device = "cpu" + config.extra_config["nixl_backend"] = "POSIX" + config.extra_config["nixl_pool_size"] = 0 # dynamic mode + config.extra_config["nixl_path"] = str(tmp_path) + config.extra_config["enable_cuda"] = False + + run_dynamic_file(config, dtype, tmp_path) + + +def _count_open_fds() -> int: + return len(os.listdir("/proc/self/fd")) + + +@pytest.mark.no_shared_allocator +@pytest.mark.skipif( + not sys.platform.startswith("linux"), + reason="Requires /proc/self/fd to count open FDs", +) +def test_nixl_dynamic_file_fd_leak_on_setup_failure(tmp_path, monkeypatch): + """ + If any operation between the per-key ``os.open`` loop and + ``release_storage_handler`` raises, the already-opened FDs must be + closed and the just-created files unlinked instead of leaked. + """ + BASE_DIR = Path(__file__).parent + config = LMCacheEngineConfig.from_file(BASE_DIR / "data/nixl.yaml") + + dtype = torch.bfloat16 + + config.nixl_buffer_device = "cpu" + config.extra_config["nixl_backend"] = "POSIX" + config.extra_config["nixl_pool_size"] = 0 + config.extra_config["nixl_path"] = str(tmp_path) + config.extra_config["nixl_async_put"] = False + config.extra_config["enable_cuda"] = False + + nixl_backend, backends, thread_loop, thread, keys, objs = ( + _build_dynamic_file_backend(config, dtype) + ) + + try: + baseline = _count_open_fds() + + def boom(*args, **kwargs): + raise RuntimeError("induced failure") + + monkeypatch.setattr(nixl_backend.agent, "create_batched_storage_handler", boom) + + # Sync mode: batched_submit_put_task calls future.result(), so the + # induced RuntimeError propagates here. + with pytest.raises(RuntimeError): + nixl_backend.batched_submit_put_task(keys, objs) + + assert _count_open_fds() == baseline, "FDs leaked on transfer-setup failure" + + # The put path opens the final key files with O_CREAT before + # registering the storage handler, so a failure here must clean + # up those just-created files. + for key in keys: + assert not os.path.exists( + os.path.join(str(tmp_path), nixl_backend._format_object_key(key)) + ), "final key file leaked on transfer-setup failure" + finally: + _teardown_dynamic_file_backend(backends, thread_loop, thread, objs) + + +@pytest.mark.no_shared_allocator +def test_nixl_dynamic_file_no_leak_on_transfer_failure(tmp_path, monkeypatch): + """ + When the NIXL transfer itself fails after the final key + files have been opened with ``O_CREAT``, the backend must remove + those empty / partially-written files. + """ + BASE_DIR = Path(__file__).parent + config = LMCacheEngineConfig.from_file(BASE_DIR / "data/nixl.yaml") + + dtype = torch.bfloat16 + + config.nixl_buffer_device = "cpu" + config.extra_config["nixl_backend"] = "POSIX" + config.extra_config["nixl_pool_size"] = 0 + config.extra_config["nixl_path"] = str(tmp_path) + config.extra_config["nixl_async_put"] = False + config.extra_config["enable_cuda"] = False + + nixl_backend, backends, thread_loop, thread, keys, objs = ( + _build_dynamic_file_backend(config, dtype) + ) + + try: + + def boom(*args, **kwargs): + raise RuntimeError("induced post_blocking failure") + + monkeypatch.setattr(nixl_backend.agent, "post_blocking", boom) + + with pytest.raises(RuntimeError): + nixl_backend.batched_submit_put_task(keys, objs) + + for key in keys: + final_path = os.path.join( + str(tmp_path), nixl_backend._format_object_key(key) + ) + assert not os.path.exists(final_path), ( + f"final key file leaked on transfer failure: {final_path}" + ) + assert not nixl_backend.contains(key, False), ( + "contains() reports key present after failed write" + ) + finally: + _teardown_dynamic_file_backend(backends, thread_loop, thread, objs) From ec40e74c62d2ff61d867271231e2a9e1103a31d4 Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Mon, 18 May 2026 13:22:58 -0700 Subject: [PATCH 27/69] [CI] DCO sign-off commits from sync_torch_version workflow (#3311) Signed-off-by: deng451e <838677410@qq.com> --- .github/workflows/sync_torch_version.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync_torch_version.yml b/.github/workflows/sync_torch_version.yml index 90dae917f85..4f481df5234 100644 --- a/.github/workflows/sync_torch_version.yml +++ b/.github/workflows/sync_torch_version.yml @@ -128,6 +128,7 @@ jobs: branch: automation/sync-torch-version base: dev delete-branch: true + signoff: true add-paths: | pyproject.toml commit-message: "[Build] sync torch version with vLLM (${{ From 2e36e1e42cf47c6fcb15875767a2a10585d8a3eb Mon Sep 17 00:00:00 2001 From: Yihua Cheng Date: Mon, 18 May 2026 13:25:18 -0700 Subject: [PATCH 28/69] =?UTF-8?q?docs:=20daily=20drift=20check=20=E2=80=94?= =?UTF-8?q?=20multi-process=20mode=20(2026-05-15)=20(#3296)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ApostaC --- docs/source/api_reference/configurations.rst | 2 +- docs/source/getting_started/installation.rst | 2 +- docs/source/production/docker_deployment.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/api_reference/configurations.rst b/docs/source/api_reference/configurations.rst index 57549a05f57..85b16f1bb9c 100644 --- a/docs/source/api_reference/configurations.rst +++ b/docs/source/api_reference/configurations.rst @@ -353,7 +353,7 @@ Settings for using Nixl as a storage backend instead of disaggregated prefill. T * - nixl_pool_size - Number of files or objects in the storage pool * - nixl_endpoint_list - - List of object-storage endpoint URLs for per-worker distribution. Overrides ``nixl_backend_params.endpoint_override`` when set. + - List of object-storage endpoint URLs for per-worker distribution. Each TP worker selects an entry round-robin by ``local_worker_id``, overriding ``nixl_backend_params.endpoint_override``. Only applied when ``nixl_backend`` is ``"OBJ"`` (silently ignored otherwise). Each entry must start with ``http://`` or ``https://``; an empty list raises ``ValueError`` at engine init. Additional Storage Configurations diff --git a/docs/source/getting_started/installation.rst b/docs/source/getting_started/installation.rst index 21e350fd8b4..53267eeee56 100644 --- a/docs/source/getting_started/installation.rst +++ b/docs/source/getting_started/installation.rst @@ -209,7 +209,7 @@ Install LMCache .. code-block:: bash - docker pull docker pull intel/vllm:0.17.0-xpu + docker pull intel/vllm:0.17.0-xpu See :ref:`docker_deployment` for running the container and ROCm images. diff --git a/docs/source/production/docker_deployment.rst b/docs/source/production/docker_deployment.rst index 2d97727ba0c..e5a50aeaae6 100644 --- a/docs/source/production/docker_deployment.rst +++ b/docs/source/production/docker_deployment.rst @@ -55,7 +55,7 @@ Validated environment: ``rocm/vllm-dev:nightly_0624_rc2_0624_rc2_20250620``, MI3 XPU (Intel) ----------- -The `Intel vLLM XPU hub`__ offers a prebuilt, optimized docker image designed for validating inference performance on the Intel GPU accelerators like ARC770, B60/B70, and future products. +The `Intel vLLM XPU hub `__ offers a prebuilt, optimized docker image designed for validating inference performance on the Intel GPU accelerators like ARC770, B60/B70, and future products. Validated environment: ``intel/vllm:0.17.0-xpu``, Intel B60 GPU, vLLM V1. From aa1abd4df154e6d8eb8c456aa17e689576705ac9 Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Mon, 18 May 2026 13:26:24 -0700 Subject: [PATCH 29/69] [CI] skip LMCache main release pipeline for operator-v* release tags (#3312) add operator release skip in workflow Signed-off-by: deng451e <838677410@qq.com> --- .github/workflows/publish.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b20e7ba088e..3b3e7c90764 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,7 +33,9 @@ jobs: permissions: pull-requests: read outputs: - lmcache: ${{ github.event_name == 'release' || startsWith(github.ref, 'refs/tags/') || steps.filter.outputs.lmcache == 'true' }} + # Skip the whole pipeline for operator releases (tags like `operator-v*`). + # operator_release.yml handles those; this workflow is lmcache-only. + lmcache: ${{ !startsWith(github.ref_name, 'operator-') && (github.event_name == 'release' || startsWith(github.ref, 'refs/tags/') || steps.filter.outputs.lmcache == 'true') }} steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 if: github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/') From a60ae02edd5dfa78863a252e885f2f330a799402 Mon Sep 17 00:00:00 2001 From: Jiayue Chen <96101996+glbyktjys@users.noreply.github.com> Date: Mon, 18 May 2026 15:55:28 -0700 Subject: [PATCH 30/69] [CI] add signoff for doc translation workflow (#3315) add signoff for doc translation workflow Signed-off-by: Jiayue Chen --- .github/workflows/translate_doc_zh.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/translate_doc_zh.yml b/.github/workflows/translate_doc_zh.yml index 9f9a62f2c3c..6da1c3f50fd 100644 --- a/.github/workflows/translate_doc_zh.yml +++ b/.github/workflows/translate_doc_zh.yml @@ -68,6 +68,7 @@ jobs: branch: automation/update-chinese-docs base: dev delete-branch: true + signoff: true add-paths: | docs/source/locale/zh_CN/LC_MESSAGES/**/*.po commit-message: "Update Chinese documentation translations" From 4d2d4f373c2487f89378c0ef1bf3d95a5c6ceea3 Mon Sep 17 00:00:00 2001 From: maobaolong Date: Tue, 19 May 2026 10:31:44 +0800 Subject: [PATCH 31/69] [MP] Add extra_config to construct lmcache mp adapter (#3026) * [MP] Add extra_config to construct lmcache mp adapter Signed-off-by: baoloongmao * fix Signed-off-by: baoloongmao --------- Signed-off-by: baoloongmao --- .../integration/vllm/lmcache_mp_connector.py | 26 +--- .../vllm/vllm_multi_process_adapter.py | 116 ++++++++++++++++-- tests/v1/test_vllm_mp_adapter.py | 2 +- 3 files changed, 111 insertions(+), 33 deletions(-) diff --git a/lmcache/integration/vllm/lmcache_mp_connector.py b/lmcache/integration/vllm/lmcache_mp_connector.py index b8433034412..25a510fab6e 100644 --- a/lmcache/integration/vllm/lmcache_mp_connector.py +++ b/lmcache/integration/vllm/lmcache_mp_connector.py @@ -122,8 +122,6 @@ def create_scheduler_adapter( server_url: str, zmq_context: zmq.Context, vllm_config: VllmConfig, - mq_timeout: float, - heartbeat_interval: float, ) -> LMCacheMPSchedulerAdapter: world_size, kv_rank = extract_world_size_and_kv_rank( vllm_config.parallel_config.world_size, @@ -140,14 +138,14 @@ def create_scheduler_adapter( vllm_config.parallel_config.pipeline_parallel_size, ) + extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config return LMCacheMPSchedulerAdapter( server_url=server_url, context=zmq_context, model_name=vllm_config.model_config.model, vllm_block_size=vllm_config.cache_config.block_size, parallel_strategy=parallel_strategy, - mq_timeout=mq_timeout, - heartbeat_interval=heartbeat_interval, + extra_config=extra_config, ) @@ -155,8 +153,6 @@ def create_worker_adapter( server_url: str, zmq_context: zmq.Context, vllm_config: VllmConfig, - mq_timeout: float, - heartbeat_interval: float, ) -> LMCacheMPWorkerAdapter: world_size, kv_rank = extract_world_size_and_kv_rank( vllm_config.parallel_config.world_size, @@ -173,14 +169,14 @@ def create_worker_adapter( vllm_config.parallel_config.pipeline_parallel_size, ) + extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config return LMCacheMPWorkerAdapter( server_url=server_url, context=zmq_context, model_name=vllm_config.model_config.model, vllm_block_size=vllm_config.cache_config.block_size, parallel_strategy=parallel_strategy, - mq_timeout=mq_timeout, - heartbeat_interval=heartbeat_interval, + extra_config=extra_config, ) @@ -499,16 +495,6 @@ def __init__( server_port = vllm_config.kv_transfer_config.get_from_extra_config( "lmcache.mp.port", 5555 ) - mq_timeout = float( - vllm_config.kv_transfer_config.get_from_extra_config( - "lmcache.mp.mq_timeout", 300.0 - ) - ) - heartbeat_interval = float( - vllm_config.kv_transfer_config.get_from_extra_config( - "lmcache.mp.heartbeat_interval", 10.0 - ) - ) server_url = f"{server_host}:{server_port}" zmq_context = zmq.Context.instance() @@ -517,8 +503,6 @@ def __init__( server_url, zmq_context, vllm_config, - mq_timeout, - heartbeat_interval, ) self.request_trackers: dict[str, LMCacheMPRequestTracker] = {} elif self.role == KVConnectorRole.WORKER: @@ -526,8 +510,6 @@ def __init__( server_url, zmq_context, vllm_config, - mq_timeout, - heartbeat_interval, ) else: raise ValueError(f"Unknown KVConnectorRole: {self.role}") diff --git a/lmcache/integration/vllm/vllm_multi_process_adapter.py b/lmcache/integration/vllm/vllm_multi_process_adapter.py index d57f7132300..b10ed5a4e56 100644 --- a/lmcache/integration/vllm/vllm_multi_process_adapter.py +++ b/lmcache/integration/vllm/vllm_multi_process_adapter.py @@ -3,6 +3,7 @@ # Standard from dataclasses import dataclass from typing import Any, Callable +import enum import os import threading @@ -25,11 +26,80 @@ logger = init_logger(__name__) -# Timeout (seconds) for blocking MQ requests: initial chunk-size query, -# KV cache registration/unregistration, and other synchronous operations. -DEFAULT_MQ_TIMEOUT: float = 300.0 -# Interval (seconds) between periodic heartbeat pings to the server. -DEFAULT_HEARTBEAT_INTERVAL: float = 10.0 + +class ExtraConfigDefault(enum.Enum): + """Centralized default values for extra_config keys. + + Each member's *name* is the key used in the extra_config dict, + and its *value* is the default. + """ + + # Timeout (seconds) for blocking MQ requests: initial + # chunk-size query, KV cache registration/unregistration, + # and other synchronous operations. + mq_timeout = 300.0 + # Interval (seconds) between periodic heartbeat pings + # to the server. + heartbeat_interval = 10.0 + + +# Backward-compatible aliases for the legacy `lmcache_mp_connector_0180` +# entry point, which still passes these as positional/keyword args. +DEFAULT_MQ_TIMEOUT: float = ExtraConfigDefault.mq_timeout.value +DEFAULT_HEARTBEAT_INTERVAL: float = ExtraConfigDefault.heartbeat_interval.value + +_EXTRA_CONFIG_KEY_PREFIX = "lmcache.mp." + + +def _resolve_extra_config( + extra_config: dict[str, Any] | None, +) -> dict[str, Any]: + """Resolve extra_config against :class:`ExtraConfigDefault`. + + Keys in *extra_config* are expected to carry the + ``lmcache.mp.`` prefix (e.g. ``lmcache.mp.mq_timeout``). + The prefix is stripped before matching against + :class:`ExtraConfigDefault` members. + + All ``lmcache.mp.*`` entries are logged. Entries whose + value differs from the default are additionally marked as + *overridden*. + + Args: + extra_config: User-supplied config dict (may be *None*). + + Returns: + A dict keyed by :pyattr:`ExtraConfigDefault` member names. + """ + stripped: dict[str, Any] = {} + if extra_config is not None: + for k, v in extra_config.items(): + if k.startswith(_EXTRA_CONFIG_KEY_PREFIX): + short = k[len(_EXTRA_CONFIG_KEY_PREFIX) :] + stripped[short] = v + + resolved: dict[str, Any] = {} + for item in ExtraConfigDefault: + default = item.value + raw = stripped.get(item.name) + value = type(default)(raw) if raw is not None else default + if value != default: + logger.info( + "%s%s = %s (overridden, default: %s)", + _EXTRA_CONFIG_KEY_PREFIX, + item.name, + value, + default, + ) + else: + logger.info( + "%s%s = %s", + _EXTRA_CONFIG_KEY_PREFIX, + item.name, + value, + ) + resolved[item.name] = value + return resolved def wrap_kv_caches(kv_caches: dict[str, torch.Tensor]) -> KVCache: @@ -77,18 +147,20 @@ def send_lmcache_request( def get_lmcache_chunk_size( mq_client: MessageQueueClient, + timeout: float = DEFAULT_MQ_TIMEOUT, ) -> int: """ Helper function to get the LMCache chunk size from the server Args: mq_client: The LMCache multiprocess mode message queue client + timeout: Timeout in seconds for the blocking request. Returns: An integer representing the LMCache chunk size """ future = send_lmcache_request(mq_client, RequestType.GET_CHUNK_SIZE, []) - chunk_size = future.result(timeout=DEFAULT_MQ_TIMEOUT) + chunk_size = future.result(timeout=timeout) return chunk_size @@ -322,6 +394,7 @@ def __init__( *, mq_timeout: float = DEFAULT_MQ_TIMEOUT, heartbeat_interval: float = DEFAULT_HEARTBEAT_INTERVAL, + extra_config: dict[str, Any] | None = None, ): """ Args: @@ -336,7 +409,12 @@ def __init__( legacy_block_size: The vLLM block size passed positionally by older vLLM connectors. mq_timeout: Timeout in seconds for message queue requests. + Ignored when ``extra_config`` is provided. heartbeat_interval: Interval in seconds between heartbeat pings. + Ignored when ``extra_config`` is provided. + extra_config: Optional dict with keys starting with + ``lmcache.mp.`` (e.g., ``lmcache.mp.mq_timeout``). When + provided, it overrides ``mq_timeout`` / ``heartbeat_interval``. """ vllm_block_size, parallel_strategy, mq_timeout = _normalize_adapter_init_args( vllm_block_size, @@ -344,6 +422,10 @@ def __init__( legacy_block_size, mq_timeout, ) + if extra_config is not None: + cfg = _resolve_extra_config(extra_config) + mq_timeout = cfg[ExtraConfigDefault.mq_timeout.name] + heartbeat_interval = cfg[ExtraConfigDefault.heartbeat_interval.name] self.mq_client = MessageQueueClient(server_url, context) self._mq_timeout = mq_timeout @@ -360,11 +442,13 @@ def __init__( # Read chunk size from lmcache try: - self.chunk_size = get_lmcache_chunk_size(self.mq_client) + self.chunk_size = get_lmcache_chunk_size( + self.mq_client, timeout=self._mq_timeout + ) except TimeoutError: self.mq_client.close() raise ConnectionError( - f"LMCache server did not respond within {mq_timeout}s. " + f"LMCache server did not respond within {self._mq_timeout}s. " "Is the server running?" ) from None assert self.chunk_size % vllm_block_size == 0, ( @@ -675,6 +759,7 @@ def __init__( *, mq_timeout: float = DEFAULT_MQ_TIMEOUT, heartbeat_interval: float = DEFAULT_HEARTBEAT_INTERVAL, + extra_config: dict[str, Any] | None = None, ): """Initialize the worker adapter for current or legacy vLLM callers. @@ -689,7 +774,12 @@ def __init__( legacy_block_size: The vLLM block size passed positionally by older vLLM connectors. mq_timeout: Timeout in seconds for message queue requests. + Ignored when ``extra_config`` is provided. heartbeat_interval: Interval in seconds between heartbeat pings. + Ignored when ``extra_config`` is provided. + extra_config: Optional dict with keys starting with + ``lmcache.mp.`` (e.g., ``lmcache.mp.mq_timeout``). When + provided, it overrides ``mq_timeout`` / ``heartbeat_interval``. Raises: TypeError: If the connector argument shape is unsupported. @@ -700,6 +790,10 @@ def __init__( legacy_block_size, mq_timeout, ) + if extra_config is not None: + cfg = _resolve_extra_config(extra_config) + mq_timeout = cfg[ExtraConfigDefault.mq_timeout.name] + heartbeat_interval = cfg[ExtraConfigDefault.heartbeat_interval.name] self.mq_client = MessageQueueClient(server_url, context) self._mq_timeout = mq_timeout @@ -733,11 +827,13 @@ def __init__( # Read chunk size from lmcache try: - chunk_size = get_lmcache_chunk_size(self.mq_client) + chunk_size = get_lmcache_chunk_size( + self.mq_client, timeout=self._mq_timeout + ) except TimeoutError: self.mq_client.close() raise ConnectionError( - f"LMCache server did not respond within {mq_timeout}s. " + f"LMCache server did not respond within {self._mq_timeout}s. " "Is the server running?" ) from None assert chunk_size % vllm_block_size == 0, ( diff --git a/tests/v1/test_vllm_mp_adapter.py b/tests/v1/test_vllm_mp_adapter.py index 7bf13f11a52..91d0b1d120a 100644 --- a/tests/v1/test_vllm_mp_adapter.py +++ b/tests/v1/test_vllm_mp_adapter.py @@ -36,7 +36,7 @@ def fake_adapter(monkeypatch): # send_lmcache_request call don't touch a real socket. fake_client = MagicMock(name="mq_client") monkeypatch.setattr(adapter_mod, "MessageQueueClient", lambda *a, **kw: fake_client) - monkeypatch.setattr(adapter_mod, "get_lmcache_chunk_size", lambda mq: 256) + monkeypatch.setattr(adapter_mod, "get_lmcache_chunk_size", lambda *a, **kw: 256) future = MagicMock(name="future") future.result.return_value = None From f9ce8cbe105bccfae81c82cdb5b32d57c7aba63a Mon Sep 17 00:00:00 2001 From: maobaolong Date: Tue, 19 May 2026 15:14:43 +0800 Subject: [PATCH 32/69] [MP] Lazy import cupy for gpu_cache_context (#3308) * Lazy import cupy for gpu_cache_context Signed-off-by: baoloongmao * Lazy import cupy for gpu_cache_context Signed-off-by: baoloongmao * Restore cupy type hints via TYPE_CHECKING Signed-off-by: baoloongmao --------- Signed-off-by: baoloongmao --- lmcache/v1/multiprocess/gpu_context.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/lmcache/v1/multiprocess/gpu_context.py b/lmcache/v1/multiprocess/gpu_context.py index 10c60d6ddd7..f53271bf134 100644 --- a/lmcache/v1/multiprocess/gpu_context.py +++ b/lmcache/v1/multiprocess/gpu_context.py @@ -8,13 +8,16 @@ """ # Standard -from typing import Any +from typing import TYPE_CHECKING, Any import array # Third Party -import cupy import torch +if TYPE_CHECKING: + # Third Party + import cupy + # First Party from lmcache import torch_dev from lmcache.logging import init_logger @@ -144,7 +147,10 @@ def __init__( # GPU streams self.cuda_stream_ = torch_dev.Stream(device=self.device_) - self.cupy_stream_ = cupy.cuda.ExternalStream( + # Third Party + import cupy + + self.cupy_stream_: "cupy.cuda.Stream" = cupy.cuda.ExternalStream( self.cuda_stream_.cuda_stream, self.device_.index ) @@ -184,7 +190,7 @@ def stream(self) -> Any: return self.cuda_stream_ @property - def cupy_stream(self) -> cupy.cuda.Stream: + def cupy_stream(self) -> "cupy.cuda.Stream": return self.cupy_stream_ @property @@ -192,7 +198,7 @@ def high_priority_stream(self) -> Any: return self.high_priority_cuda_stream_ @property - def high_priority_cupy_stream(self) -> cupy.cuda.Stream: + def high_priority_cupy_stream(self) -> "cupy.cuda.Stream": return self.high_priority_cupy_stream_ @property @@ -460,7 +466,10 @@ def __init__(self, kv_caches: KVCache, lmcache_chunk_size: int = 256): # GPU streams self._cuda_stream = torch_dev.Stream(device=self._device) - self._cupy_stream = cupy.cuda.ExternalStream( + # Third Party + import cupy + + self._cupy_stream: "cupy.cuda.Stream" = cupy.cuda.ExternalStream( self._cuda_stream.cuda_stream, self._device.index ) @@ -512,7 +521,7 @@ def stream(self) -> Any: return self._cuda_stream @property - def cupy_stream(self) -> cupy.cuda.Stream: + def cupy_stream(self) -> "cupy.cuda.Stream": return self._cupy_stream @property @@ -520,7 +529,7 @@ def high_priority_stream(self) -> Any: return self._high_priority_cuda_stream @property - def high_priority_cupy_stream(self) -> cupy.cuda.Stream: + def high_priority_cupy_stream(self) -> "cupy.cuda.Stream": return self._high_priority_cupy_stream @property From 9e38f1db9a6485b9b8974096859f9722309481de Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 19 May 2026 19:36:05 +0000 Subject: [PATCH 33/69] Update Chinese documentation translations (#3310) Co-authored-by: deng451e <57919305+deng451e@users.noreply.github.com> --- .../api_reference/configurations.po | 1527 +++++++++++++++++ .../api_reference/dynamic_connector.po | 94 + .../api_reference/multimodality.po | 171 ++ .../api_reference/storage_backends.po | 29 + .../locale/zh_CN/LC_MESSAGES/cli/bench.po | 882 ++++++++++ .../zh_CN/LC_MESSAGES/cli/bench_kvcache.po | 353 ++++ .../locale/zh_CN/LC_MESSAGES/cli/index.po | 103 ++ .../locale/zh_CN/LC_MESSAGES/cli/kvcache.po | 157 ++ .../zh_CN/LC_MESSAGES/community/blogs.po | 36 + .../zh_CN/LC_MESSAGES/community/meetings.po | 133 ++ .../LC_MESSAGES/controller/freeze_mode.po | 191 +++ .../zh_CN/LC_MESSAGES/controller/index.po | 118 ++ .../developer_guide/architecture.po | 260 +++ .../zh_CN/LC_MESSAGES/developer_guide/cli.po | 116 ++ .../developer_guide/contributing.po | 497 ++++++ .../developer_guide/docker_file.po | 58 + .../extending_lmcache/index.po | 267 +++ .../extending_lmcache/native_connectors.po | 729 ++++++++ .../remote_storage_plugins.po | 267 +++ .../extending_lmcache/runtime_plugins.po | 185 ++ .../extending_lmcache/storage_plugins.po | 379 ++++ .../developer_guide/integration.po | 114 ++ .../developer_guide/usage/basic_check.po | 209 +++ .../developer_guide/usage/index.po | 25 + .../usage/usage_stats_collection.po | 123 ++ .../disaggregated_prefill/nixl/1p1d.po | 346 ++++ .../disaggregated_prefill/nixl/index.po | 52 + .../disaggregated_prefill/nixl/xpyd.po | 403 +++++ .../disaggregated_prefill/shared_storage.po | 29 + .../getting_started/benchmarking.po | 262 +++ .../zh_CN/LC_MESSAGES/getting_started/cli.po | 478 ++++++ .../zh_CN/LC_MESSAGES/getting_started/faq.po | 142 ++ .../getting_started/installation.po | 238 +++ .../getting_started/kv_cache_calculator.po | 45 + .../LC_MESSAGES/getting_started/quickstart.po | 381 ++++ .../quickstart/disaggregated_prefill.po | 268 +++ .../getting_started/quickstart/index.po | 161 ++ .../quickstart/multimodality.po | 73 + .../quickstart/offload_kv_cache.po | 301 ++++ .../quickstart/share_kv_cache.po | 246 +++ .../quickstart/standalone_starter.po | 372 ++++ .../getting_started/troubleshoot.po | 29 + .../internal_api_server/common_apis.po | 403 +++++ .../internal_api_server/controller_apis.po | 309 ++++ .../dynamic_backend_management.po | 159 ++ .../internal_api_server.po | 117 ++ .../internal_api_server/vllm_apis.po | 1020 +++++++++++ .../LC_MESSAGES/kv_cache/async_loading.po | 214 +++ .../LC_MESSAGES/kv_cache/caching_policies.po | 43 + .../LC_MESSAGES/kv_cache/multiprocess_mode.po | 38 + .../zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po | 193 +++ .../kv_cache/storage_backends/3fs.po | 159 ++ .../kv_cache/storage_backends/cpu_ram.po | 235 +++ .../storage_backends/custom_backend.po | 39 + .../kv_cache/storage_backends/dax.po | 274 +++ .../kv_cache/storage_backends/eic.po | 72 + .../kv_cache/storage_backends/gds.po | 324 ++++ .../kv_cache/storage_backends/hfbucket.po | 168 ++ .../kv_cache/storage_backends/index.po | 35 + .../kv_cache/storage_backends/infinistore.po | 232 +++ .../storage_backends/local_storage.po | 363 ++++ .../kv_cache/storage_backends/maru.po | 203 +++ .../kv_cache/storage_backends/mock.po | 104 ++ .../kv_cache/storage_backends/mooncake.po | 525 ++++++ .../kv_cache/storage_backends/nixl.po | 203 +++ .../kv_cache/storage_backends/redis.po | 347 ++++ .../kv_cache/storage_backends/resp.po | 716 ++++++++ .../kv_cache/storage_backends/s3.po | 231 +++ .../storage_backends/sagemaker_hyperpod.po | 157 ++ .../kv_cache/storage_backends/valkey.po | 246 +++ .../kv_cache/storage_backends/weka.po | 132 ++ .../kv_cache_management/check_finish.po | 29 + .../LC_MESSAGES/kv_cache_management/clear.po | 79 + .../kv_cache_management/compress.po | 94 + .../LC_MESSAGES/kv_cache_management/health.po | 60 + .../LC_MESSAGES/kv_cache_management/index.po | 214 +++ .../LC_MESSAGES/kv_cache_management/lookup.po | 85 + .../LC_MESSAGES/kv_cache_management/move.po | 85 + .../LC_MESSAGES/kv_cache_management/pin.po | 76 + .../kv_cache_management/query_worker_info.po | 68 + .../kv_cache_optimizations/blending.po | 75 + .../compression/cachegen.po | 53 + .../compression/index.po | 39 + .../kv_cache_optimizations/layerwise.po | 299 ++++ .../zh_CN/LC_MESSAGES/mp/architecture.po | 792 +++++++++ .../zh_CN/LC_MESSAGES/mp/configuration.po | 788 +++++++++ .../locale/zh_CN/LC_MESSAGES/mp/deployment.po | 254 +++ .../locale/zh_CN/LC_MESSAGES/mp/http_api.po | 772 +++++++++ .../locale/zh_CN/LC_MESSAGES/mp/index.po | 140 ++ .../locale/zh_CN/LC_MESSAGES/mp/l2_storage.po | 1272 ++++++++++++++ .../zh_CN/LC_MESSAGES/mp/observability.po | 1458 ++++++++++++++++ .../locale/zh_CN/LC_MESSAGES/mp/operator.po | 978 +++++++++++ .../locale/zh_CN/LC_MESSAGES/mp/quickstart.po | 201 +++ .../locale/zh_CN/LC_MESSAGES/mp/serde.po | 159 ++ .../LC_MESSAGES/mp/tracing_and_debugging.po | 469 +++++ .../LC_MESSAGES/non_kv_cache/encoder_cache.po | 282 +++ .../production/docker_deployment.po | 79 + .../production/kubernetes_deployment.po | 58 + .../LC_MESSAGES/production/kv_cache_events.po | 186 ++ .../observability/chunk_statistics.po | 580 +++++++ .../production/observability/frontend.po | 222 +++ .../observability/health_monitor.po | 495 ++++++ .../production/observability/index.po | 25 + .../observability/internal_api_server.po | 138 ++ .../production/observability/metrics.po | 899 ++++++++++ .../observability/periodic_thread_api.po | 232 +++ .../production/observability/vllm_endpoint.po | 458 +++++ .../production/performance_tuning.po | 129 ++ .../zh_CN/LC_MESSAGES/recipes/devstral.po | 124 ++ .../zh_CN/LC_MESSAGES/recipes/gemma4.po | 122 ++ .../zh_CN/LC_MESSAGES/recipes/gpt_oss.po | 130 ++ .../locale/zh_CN/LC_MESSAGES/recipes/index.po | 211 +++ .../zh_CN/LC_MESSAGES/recipes/minimax_m2.po | 150 ++ .../locale/zh_CN/LC_MESSAGES/recipes/qwen3.po | 150 ++ 114 files changed, 31319 insertions(+) create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/api_reference/configurations.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/api_reference/dynamic_connector.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/api_reference/multimodality.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/api_reference/storage_backends.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/bench.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/bench_kvcache.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/kvcache.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/community/blogs.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/community/meetings.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/controller/freeze_mode.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/controller/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/architecture.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/cli.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/contributing.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/docker_file.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/native_connectors.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/remote_storage_plugins.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/runtime_plugins.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/storage_plugins.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/integration.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/basic_check.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/usage_stats_collection.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/1p1d.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/xpyd.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/shared_storage.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/benchmarking.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/cli.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/faq.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/installation.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/kv_cache_calculator.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/disaggregated_prefill.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/multimodality.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/offload_kv_cache.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/share_kv_cache.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/standalone_starter.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/troubleshoot.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/common_apis.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/controller_apis.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/dynamic_backend_management.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/internal_api_server.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/vllm_apis.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/async_loading.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/caching_policies.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/multiprocess_mode.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/3fs.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/cpu_ram.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/custom_backend.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/dax.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/eic.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/gds.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/hfbucket.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/infinistore.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/local_storage.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/maru.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mock.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mooncake.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/nixl.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/redis.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/resp.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/s3.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/sagemaker_hyperpod.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/valkey.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/weka.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/check_finish.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/clear.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/compress.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/health.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/lookup.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/move.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/pin.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/query_worker_info.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/blending.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/cachegen.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/layerwise.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/architecture.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/configuration.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/deployment.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/http_api.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/observability.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/operator.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/quickstart.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/serde.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/tracing_and_debugging.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/encoder_cache.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/docker_deployment.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/kubernetes_deployment.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/kv_cache_events.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/chunk_statistics.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/frontend.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/health_monitor.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/internal_api_server.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/metrics.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/periodic_thread_api.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/vllm_endpoint.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/performance_tuning.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/devstral.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/gemma4.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/gpt_oss.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/minimax_m2.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/qwen3.po diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/configurations.po b/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/configurations.po new file mode 100644 index 00000000000..88df708b347 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/configurations.po @@ -0,0 +1,1527 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/api_reference/configurations.rst:2 +msgid "Configuring LMCache" +msgstr "配置 LMCache" + +#: ../../source/api_reference/configurations.rst:4 +msgid "LMCache supports two types of configurations:" +msgstr "LMCache 支持两种类型的配置：" + +#: ../../source/api_reference/configurations.rst:6 +msgid "" +"**Configuration file**: a YAML (recommended) or JSON file that contains " +"the configuration items." +msgstr "**配置文件**：一个包含配置项的 YAML（推荐）或 JSON 文件。" + +#: ../../source/api_reference/configurations.rst:7 +msgid "" +"**Environment variables**: environment variables that start with " +"``LMCACHE_``." +msgstr "**环境变量**：以 ``LMCACHE_`` 开头的环境变量。" + +#: ../../source/api_reference/configurations.rst:9 +msgid "" +"To use a configuration file, you can set the ``LMCACHE_CONFIG_FILE`` " +"environment variable to the path of the configuration file." +msgstr "要使用配置文件，您可以将 ``LMCACHE_CONFIG_FILE`` 环境变量设置为配置文件的路径。" + +#: ../../source/api_reference/configurations.rst:13 +msgid "" +"The environment variable configurations will be ignored if the " +"configuration file is present." +msgstr "如果配置文件存在，则环境变量配置将被忽略。" + +#: ../../source/api_reference/configurations.rst:17 +msgid "General Configurations" +msgstr "通用配置" + +#: ../../source/api_reference/configurations.rst:19 +msgid "Basic cache settings that control the core functionality of LMCache." +msgstr "控制 LMCache 核心功能的基本缓存设置。" + +#: ../../source/api_reference/configurations.rst:25 +#: ../../source/api_reference/configurations.rst:114 +#: ../../source/api_reference/configurations.rst:150 +#: ../../source/api_reference/configurations.rst:175 +#: ../../source/api_reference/configurations.rst:203 +#: ../../source/api_reference/configurations.rst:236 +#: ../../source/api_reference/configurations.rst:368 +#: ../../source/api_reference/configurations.rst:472 +#: ../../source/api_reference/configurations.rst:500 +#: ../../source/api_reference/configurations.rst:516 +msgid "YAML Config Name" +msgstr "YAML 配置名称" + +#: ../../source/api_reference/configurations.rst:26 +#: ../../source/api_reference/configurations.rst:115 +#: ../../source/api_reference/configurations.rst:151 +#: ../../source/api_reference/configurations.rst:176 +#: ../../source/api_reference/configurations.rst:204 +#: ../../source/api_reference/configurations.rst:237 +#: ../../source/api_reference/configurations.rst:369 +#: ../../source/api_reference/configurations.rst:473 +#: ../../source/api_reference/configurations.rst:501 +#: ../../source/api_reference/configurations.rst:517 +msgid "Environment Variable" +msgstr "环境变量" + +#: ../../source/api_reference/configurations.rst:27 +#: ../../source/api_reference/configurations.rst:116 +#: ../../source/api_reference/configurations.rst:152 +#: ../../source/api_reference/configurations.rst:177 +#: ../../source/api_reference/configurations.rst:205 +#: ../../source/api_reference/configurations.rst:238 +#: ../../source/api_reference/configurations.rst:314 +#: ../../source/api_reference/configurations.rst:346 +#: ../../source/api_reference/configurations.rst:370 +#: ../../source/api_reference/configurations.rst:407 +#: ../../source/api_reference/configurations.rst:474 +#: ../../source/api_reference/configurations.rst:502 +#: ../../source/api_reference/configurations.rst:518 +msgid "Description" +msgstr "描述" + +#: ../../source/api_reference/configurations.rst:28 +msgid "chunk_size" +msgstr "chunk_size" + +#: ../../source/api_reference/configurations.rst:29 +msgid "LMCACHE_CHUNK_SIZE" +msgstr "LMCACHE_CHUNK_SIZE" + +#: ../../source/api_reference/configurations.rst:30 +msgid "Size of cache chunks. Default: 256" +msgstr "缓存块的大小。默认值：256" + +#: ../../source/api_reference/configurations.rst:31 +msgid "local_cpu" +msgstr "local_cpu" + +#: ../../source/api_reference/configurations.rst:32 +msgid "LMCACHE_LOCAL_CPU" +msgstr "LMCACHE_LOCAL_CPU" + +#: ../../source/api_reference/configurations.rst:33 +msgid "Whether to enable CPU caching. Values: true/false. Default: true" +msgstr "是否启用 CPU 缓存。值：true/false。默认：true" + +#: ../../source/api_reference/configurations.rst:34 +msgid "max_local_cpu_size" +msgstr "max_local_cpu_size" + +#: ../../source/api_reference/configurations.rst:35 +msgid "LMCACHE_MAX_LOCAL_CPU_SIZE" +msgstr "LMCACHE_MAX_LOCAL_CPU_SIZE" + +#: ../../source/api_reference/configurations.rst:36 +msgid "Maximum CPU cache size in GB. Default: 5.0" +msgstr "最大 CPU 缓存大小（以 GB 为单位）。默认值：5.0" + +#: ../../source/api_reference/configurations.rst:37 +msgid "local_disk" +msgstr "local_disk" + +#: ../../source/api_reference/configurations.rst:38 +msgid "LMCACHE_LOCAL_DISK" +msgstr "LMCACHE_LOCAL_DISK" + +#: ../../source/api_reference/configurations.rst:39 +msgid "" +"Path (or comma-separated paths) to local disk cache directories. Format: " +"``\"file:///path/to/cache\"`` or ``\"/path/a,/path/b\"`` for multi-device" +" I/O. See ``local_disk_path_sharding`` for how paths are assigned to " +"GPUs." +msgstr "本地磁盘缓存目录的路径（或以逗号分隔的路径）。格式：``\"file:///path/to/cache\"`` 或 ``\"/path/a,/path/b\"`` 用于多设备 I/O。有关路径如何分配给 GPU，请参见 ``local_disk_path_sharding``。" + +#: ../../source/api_reference/configurations.rst:40 +msgid "local_disk_path_sharding" +msgstr "local_disk_path_sharding" + +#: ../../source/api_reference/configurations.rst:41 +msgid "LMCACHE_LOCAL_DISK_PATH_SHARDING" +msgstr "LMCACHE_LOCAL_DISK_PATH_SHARDING" + +#: ../../source/api_reference/configurations.rst:42 +#: ../../source/api_reference/configurations.rst:376 +msgid "" +"Strategy for selecting a path when multiple paths are provided. Currently" +" only ``\"by_gpu\"`` is supported, which selects paths based on GPU " +"device ID (default: \"by_gpu\")." +msgstr "当提供多个路径时选择路径的策略。目前仅支持 ``\"by_gpu\"``，该策略根据 GPU 设备 ID 选择路径（默认值：\\\"by_gpu\\\"）。" + +#: ../../source/api_reference/configurations.rst:43 +msgid "max_local_disk_size" +msgstr "max_local_disk_size" + +#: ../../source/api_reference/configurations.rst:44 +msgid "LMCACHE_MAX_LOCAL_DISK_SIZE" +msgstr "LMCACHE_MAX_LOCAL_DISK_SIZE" + +#: ../../source/api_reference/configurations.rst:45 +msgid "Maximum disk cache size in GB. Default: 0.0" +msgstr "最大磁盘缓存大小（以 GB 为单位）。默认值：0.0" + +#: ../../source/api_reference/configurations.rst:46 +msgid "remote_url" +msgstr "remote_url" + +#: ../../source/api_reference/configurations.rst:47 +msgid "LMCACHE_REMOTE_URL" +msgstr "LMCACHE_REMOTE_URL" + +#: ../../source/api_reference/configurations.rst:48 +msgid "Remote storage URL. Format: \"protocol://host:port\"." +msgstr "远程存储 URL。格式：\\\"protocol://host:port\\\"。" + +#: ../../source/api_reference/configurations.rst:49 +msgid "remote_serde" +msgstr "remote_serde" + +#: ../../source/api_reference/configurations.rst:50 +msgid "LMCACHE_REMOTE_SERDE" +msgstr "LMCACHE_REMOTE_SERDE" + +#: ../../source/api_reference/configurations.rst:51 +msgid "" +"Serialization format. Values: \"naive\" or \"cachegen\". Default: " +"\"naive\"" +msgstr "序列化格式。值：\\\"naive\\\" 或 \\\"cachegen\\\"。默认值：\\\"naive\\\"" + +#: ../../source/api_reference/configurations.rst:52 +msgid "save_decode_cache" +msgstr "save_decode_cache" + +#: ../../source/api_reference/configurations.rst:53 +msgid "LMCACHE_SAVE_DECODE_CACHE" +msgstr "LMCACHE_SAVE_DECODE_CACHE" + +#: ../../source/api_reference/configurations.rst:54 +msgid "Whether to store decode KV cache. Values: true/false. Default: false" +msgstr "是否存储解码 KV Cache。值：true/false。默认：false" + +#: ../../source/api_reference/configurations.rst:55 +msgid "use_layerwise" +msgstr "use_layerwise" + +#: ../../source/api_reference/configurations.rst:56 +msgid "LMCACHE_USE_LAYERWISE" +msgstr "LMCACHE_USE_LAYERWISE" + +#: ../../source/api_reference/configurations.rst:57 +msgid "Whether to enable layerwise pipelining. Values: true/false. Default: false" +msgstr "是否启用逐层流水线。值：true/false。默认值：false" + +#: ../../source/api_reference/configurations.rst:58 +msgid "pre_caching_hash_algorithm" +msgstr "pre_caching_hash_algorithm" + +#: ../../source/api_reference/configurations.rst:59 +msgid "LMCACHE_PRE_CACHING_HASH_ALGORITHM" +msgstr "LMCACHE_PRE_CACHING_HASH_ALGORITHM" + +#: ../../source/api_reference/configurations.rst:60 +msgid "Hash algorithm for prefix-caching. Default: \"builtin\"" +msgstr "前缀缓存的哈希算法。默认值：“builtin”" + +#: ../../source/api_reference/configurations.rst:61 +msgid "save_unfull_chunk" +msgstr "save_unfull_chunk" + +#: ../../source/api_reference/configurations.rst:62 +msgid "LMCACHE_SAVE_UNFULL_CHUNK" +msgstr "LMCACHE_SAVE_UNFULL_CHUNK" + +#: ../../source/api_reference/configurations.rst:63 +msgid "Whether to save unfull chunks. Values: true/false. Default: false" +msgstr "是否保存未满块。值：true/false。默认值：false" + +#: ../../source/api_reference/configurations.rst:64 +msgid "blocking_timeout_secs" +msgstr "blocking_timeout_secs" + +#: ../../source/api_reference/configurations.rst:65 +msgid "LMCACHE_BLOCKING_TIMEOUT_SECS" +msgstr "LMCACHE_BLOCKING_TIMEOUT_SECS" + +#: ../../source/api_reference/configurations.rst:66 +msgid "Timeout for blocking operations in seconds. Default: 10" +msgstr "阻塞操作的超时时间（以秒为单位）。默认值：10" + +#: ../../source/api_reference/configurations.rst:67 +msgid "py_enable_gc" +msgstr "py_enable_gc" + +#: ../../source/api_reference/configurations.rst:68 +msgid "LMCACHE_PY_ENABLE_GC" +msgstr "LMCACHE_PY_ENABLE_GC" + +#: ../../source/api_reference/configurations.rst:69 +msgid "" +"Whether to enable Python garbage collection. Values: true/false. Default:" +" true" +msgstr "是否启用 Python 垃圾回收。值：true/false。默认：true" + +#: ../../source/api_reference/configurations.rst:70 +msgid "cache_policy" +msgstr "cache_policy" + +#: ../../source/api_reference/configurations.rst:71 +msgid "LMCACHE_CACHE_POLICY" +msgstr "LMCACHE_CACHE_POLICY" + +#: ../../source/api_reference/configurations.rst:72 +msgid "Cache eviction policy (e.g. \"LRU\", \"LFU\", \"FIFO\"). Default: \"LRU\"" +msgstr "缓存逐出策略（例如“LRU”、“LFU”、“FIFO”）。默认值：“LRU”" + +#: ../../source/api_reference/configurations.rst:73 +msgid "numa_mode" +msgstr "numa_mode" + +#: ../../source/api_reference/configurations.rst:74 +msgid "LMCACHE_NUMA_MODE" +msgstr "LMCACHE_NUMA_MODE" + +#: ../../source/api_reference/configurations.rst:75 +msgid "" +"NUMA-aware memory allocation mode. Values: \"auto\" (detect from system)," +" \"manual\" (use extra_config mapping), null (disabled). When enabled, " +"allocates pinned memory on specific NUMA nodes for better GPU-CPU memory " +"bandwidth. Default: null" +msgstr "NUMA 感知内存分配模式。值：\\\"auto\\\"（从系统检测）、\\\"manual\\\"（使用 extra_config 映射）、null（禁用）。启用时，在特定的 NUMA 节点上分配固定内存，以提高 GPU-CPU 内存带宽。默认值：null" + +#: ../../source/api_reference/configurations.rst:76 +msgid "external_lookup_client" +msgstr "external_lookup_client" + +#: ../../source/api_reference/configurations.rst:77 +msgid "LMCACHE_EXTERNAL_LOOKUP_CLIENT" +msgstr "LMCACHE_EXTERNAL_LOOKUP_CLIENT" + +#: ../../source/api_reference/configurations.rst:78 +msgid "" +"External KV lookup service URI (e.g., \"mooncakestore://address\"). If " +"null, defaults to LMCache's internal lookup client. Default: null" +msgstr "外部 KV 查找服务 URI（例如，\\\"mooncakestore://address\\\"）。如果为 null，则默认为 LMCache 的内部查找客户端。默认值：null" + +#: ../../source/api_reference/configurations.rst:79 +msgid "priority_limit" +msgstr "priority_limit" + +#: ../../source/api_reference/configurations.rst:80 +msgid "LMCACHE_PRIORITY_LIMIT" +msgstr "LMCACHE_PRIORITY_LIMIT" + +#: ../../source/api_reference/configurations.rst:81 +msgid "" +"Caches requests only if priority value ≤ limit. (**Not applicable for PD " +"Disaggregation**) Type: int. Default: None" +msgstr "仅在优先级值 ≤ 限制时缓存请求。（**不适用于 PD 分离**）类型：int。默认值：无" + +#: ../../source/api_reference/configurations.rst:82 +msgid "min_retrieve_tokens" +msgstr "min_retrieve_tokens" + +#: ../../source/api_reference/configurations.rst:83 +msgid "LMCACHE_MIN_RETRIEVE_TOKENS" +msgstr "LMCACHE_MIN_RETRIEVE_TOKENS" + +#: ../../source/api_reference/configurations.rst:84 +msgid "" +"Minimum number of hit tokens required to perform retrieve. If hit tokens " +"< this value, skip retrieve but still record the hits to avoid re-storing" +" existing chunks. See :ref:`performance_tuning` for a working example. " +"Default: 0 (disabled)" +msgstr "执行检索所需的最小命中令牌数。如果命中令牌少于此值，则跳过检索，但仍记录命中以避免重新存储现有块。有关工作示例，请参见 :ref:`performance_tuning`。默认值：0（禁用）" + +#: ../../source/api_reference/configurations.rst:85 +msgid "store_location" +msgstr "store_location" + +#: ../../source/api_reference/configurations.rst:86 +msgid "LMCACHE_STORE_LOCATION" +msgstr "LMCACHE_STORE_LOCATION" + +#: ../../source/api_reference/configurations.rst:87 +msgid "" +"A single storage backend name to store KV caches into. When specified, " +"only the matching backend receives store operations. Valid values are the" +" backend class names registered in the storage manager, including: " +"``\"LocalCPUBackend\"``, ``\"LocalDiskBackend\"``, ``\"RemoteBackend\"``," +" ``\"PDBackend\"``, ``\"P2PBackend\"``, ``\"GdsBackend\"``, etc, and any " +"storage plugin backends. Note: ``\"PDBackend\"`` cannot be used as a " +"store location for a decoder instance in a PD setup, since PDBackend is " +"one-way from prefiller to decoder only. Default: null (store to all " +"active backends)" +msgstr "一个存储后端名称，用于存储 KV 缓存。当指定时，只有匹配的后端接收存储操作。有效值是存储管理器中注册的后端类名称，包括：``\\\"LocalCPUBackend\\\"``、``\\\"LocalDiskBackend\\\"``、``\\\"RemoteBackend\\\"``、``\\\"PDBackend\\\"``、``\\\"P2PBackend\\\"``、``\\\"GdsBackend\\\"`` 等，以及任何存储插件后端。注意：``\\\"PDBackend\\\"`` 不能用作 PD 设置中解码器实例的存储位置，因为 PDBackend 仅从预填充器到解码器是单向的。默认值：null（存储到所有活动后端）" + +#: ../../source/api_reference/configurations.rst:88 +msgid "retrieve_locations" +msgstr "retrieve_locations" + +#: ../../source/api_reference/configurations.rst:89 +msgid "LMCACHE_RETRIEVE_LOCATIONS" +msgstr "LMCACHE_RETRIEVE_LOCATIONS" + +#: ../../source/api_reference/configurations.rst:90 +msgid "" +"List of storage backend names to search when retrieving or looking up KV " +"caches. When specified, only the listed backends are searched. Valid " +"values are the backend class names registered in the storage manager, " +"including: ``\"LocalCPUBackend\"``, ``\"LocalDiskBackend\"``, " +"``\"RemoteBackend\"``, ``\"PDBackend\"``, ``\"P2PBackend\"``, " +"``\"GdsBackend\"``, etc, and any storage plugin backends. Default: null " +"(search all active backends)" +msgstr "检索或查找 KV Cache 时要搜索的存储后端名称列表。当指定时，仅搜索列出的后端。有效值为在存储管理器中注册的后端类名称，包括：``\\\"LocalCPUBackend\\\"``、``\\\"LocalDiskBackend\\\"``、``\\\"RemoteBackend\\\"``、``\\\"PDBackend\\\"``、``\\\"P2PBackend\\\"``、``\\\"GdsBackend\\\"`` 等，以及任何存储插件后端。默认值：null（搜索所有活动后端）" + +#: ../../source/api_reference/configurations.rst:91 +msgid "extra_config" +msgstr "extra_config" + +#: ../../source/api_reference/configurations.rst:92 +#, python-brace-format +msgid "LMCACHE_EXTRA_CONFIG={\"key\": value, ...}" +msgstr "LMCACHE_EXTRA_CONFIG={\\\"key\\\": value, ...}" + +#: ../../source/api_reference/configurations.rst:93 +#, python-brace-format +msgid "" +"Additional configuration as JSON dict. For NUMA manual mode, include " +"\"gpu_to_numa_mapping\": {gpu_id: numa_node, ...}. Default: {}" +msgstr "额外的配置作为 JSON 字典。对于 NUMA 手动模式，包含 \\\"gpu_to_numa_mapping\\\": {gpu_id: numa_node, ...}。默认值：{}" + +#: ../../source/api_reference/configurations.rst:96 +msgid "Lazy Memory Allocator Configurations" +msgstr "惰性内存分配器配置" + +#: ../../source/api_reference/configurations.rst:98 +msgid "" +"Settings for the lazy memory allocator that enables gradual memory " +"allocation to reduce startup time and initial memory footprint." +msgstr "用于懒惰内存分配器的设置，允许逐步分配内存以减少启动时间和初始内存占用。" + +#: ../../source/api_reference/configurations.rst:102 +msgid "" +"The lazy memory allocator is designed for scenarios with large CPU memory" +" configurations. It starts with a small initial allocation and gradually " +"expands as needed, reducing startup wait time and avoiding unnecessary " +"memory consumption when the full capacity is not required." +msgstr "懒惰内存分配器旨在处理大 CPU 内存配置的场景。它从小的初始分配开始，并根据需要逐渐扩展，从而减少启动等待时间，并避免在不需要全部容量时的不必要内存消耗。" + +#: ../../source/api_reference/configurations.rst:104 +msgid "**Key characteristics:**" +msgstr "**关键特性：**" + +#: ../../source/api_reference/configurations.rst:106 +msgid "" +"**One-time expansion**: Memory expands until target size is reached, then" +" stops" +msgstr "**一次性扩展**：内存扩展直到达到目标大小，然后停止" + +#: ../../source/api_reference/configurations.rst:107 +msgid "" +"**No shrinking**: Once allocated, memory is never released back to the " +"system" +msgstr "**不收缩**：一旦分配，内存将永远不会释放回系统" + +#: ../../source/api_reference/configurations.rst:108 +msgid "" +"**Automatic activation**: Only activates when ``max_local_cpu_size`` " +"exceeds ``lazy_memory_safe_size``" +msgstr "**自动激活**：仅在 ``max_local_cpu_size`` 超过 ``lazy_memory_safe_size`` 时激活" + +#: ../../source/api_reference/configurations.rst:117 +msgid "enable_lazy_memory_allocator" +msgstr "enable_lazy_memory_allocator" + +#: ../../source/api_reference/configurations.rst:118 +msgid "LMCACHE_ENABLE_LAZY_MEMORY_ALLOCATOR" +msgstr "LMCACHE_ENABLE_LAZY_MEMORY_ALLOCATOR" + +#: ../../source/api_reference/configurations.rst:119 +msgid "" +"Whether to enable lazy memory allocator. Values: true/false. Default: " +"false" +msgstr "是否启用延迟内存分配器。值：true/false。默认值：false" + +#: ../../source/api_reference/configurations.rst:120 +msgid "lazy_memory_initial_ratio" +msgstr "lazy_memory_initial_ratio" + +#: ../../source/api_reference/configurations.rst:121 +msgid "LMCACHE_LAZY_MEMORY_INITIAL_RATIO" +msgstr "LMCACHE_LAZY_MEMORY_INITIAL_RATIO" + +#: ../../source/api_reference/configurations.rst:122 +msgid "" +"Initial memory allocation ratio (0.0-1.0). Determines the fraction of " +"max_local_cpu_size to allocate at startup. Default: 0.2 (20%)" +msgstr "初始内存分配比例 (0.0-1.0)。决定在启动时分配的 max_local_cpu_size 的比例。默认值：0.2 (20%)" + +#: ../../source/api_reference/configurations.rst:123 +msgid "lazy_memory_expand_trigger_ratio" +msgstr "lazy_memory_expand_trigger_ratio" + +#: ../../source/api_reference/configurations.rst:124 +msgid "LMCACHE_LAZY_MEMORY_EXPAND_TRIGGER_RATIO" +msgstr "LMCACHE_LAZY_MEMORY_EXPAND_TRIGGER_RATIO" + +#: ../../source/api_reference/configurations.rst:125 +msgid "" +"Memory usage ratio (0.0-1.0) that triggers expansion. When used memory " +"exceeds this ratio of current capacity, expansion begins. Default: 0.5 " +"(50%)" +msgstr "触发扩展的内存使用比例（0.0-1.0）。当使用的内存超过当前容量的此比例时，扩展开始。默认值：0.5（50%）" + +#: ../../source/api_reference/configurations.rst:126 +msgid "lazy_memory_step_ratio" +msgstr "lazy_memory_step_ratio" + +#: ../../source/api_reference/configurations.rst:127 +msgid "LMCACHE_LAZY_MEMORY_STEP_RATIO" +msgstr "LMCACHE_LAZY_MEMORY_STEP_RATIO" + +#: ../../source/api_reference/configurations.rst:128 +msgid "" +"Memory expansion step ratio (0.0-1.0). Each expansion adds this fraction " +"of max_local_cpu_size. Default: 0.1 (10%)" +msgstr "内存扩展步骤比例（0.0-1.0）。每次扩展增加此比例的 max_local_cpu_size。默认值：0.1（10%）" + +#: ../../source/api_reference/configurations.rst:129 +msgid "lazy_memory_safe_size" +msgstr "lazy_memory_safe_size" + +#: ../../source/api_reference/configurations.rst:130 +msgid "LMCACHE_LAZY_MEMORY_SAFE_SIZE" +msgstr "LMCACHE_LAZY_MEMORY_SAFE_SIZE" + +#: ../../source/api_reference/configurations.rst:131 +msgid "" +"Threshold in GB above which lazy allocator activates. If " +"max_local_cpu_size ≤ this value, lazy allocator is disabled regardless of" +" enable_lazy_memory_allocator setting. Default: 0.0" +msgstr "在 GB 中的阈值，超过该值后懒惰分配器激活。如果 max_local_cpu_size ≤ 此值，则无论 enable_lazy_memory_allocator 设置如何，懒惰分配器都将被禁用。默认值：0.0" + +#: ../../source/api_reference/configurations.rst:132 +msgid "reserve_local_cpu_size" +msgstr "reserve_local_cpu_size" + +#: ../../source/api_reference/configurations.rst:133 +msgid "LMCACHE_RESERVE_LOCAL_CPU_SIZE" +msgstr "LMCACHE_RESERVE_LOCAL_CPU_SIZE" + +#: ../../source/api_reference/configurations.rst:134 +msgid "" +"Reserved system memory in GB that should not be allocated by LMCache. " +"Used to prevent out-of-memory conditions. Default: 0.0" +msgstr "保留的系统内存（以 GB 为单位），不应由 LMCache 分配。用于防止内存不足的情况。默认值：0.0" + +#: ../../source/api_reference/configurations.rst:137 +msgid "Cache Blending Configurations" +msgstr "缓存混合配置" + +#: ../../source/api_reference/configurations.rst:139 +msgid "Settings related to cache blending functionality." +msgstr "与缓存混合功能相关的设置。" + +#: ../../source/api_reference/configurations.rst:143 +msgid "" +"We have an end-to-end `example " +"`_. We " +"also have more :doc:`detailed documentation " +"<../kv_cache_optimizations/blending>`." +msgstr "我们有一个端到端的 `示例 `_。我们还有更多的 :doc:`详细文档 <../kv_cache_optimizations/blending>`。" + +#: ../../source/api_reference/configurations.rst:153 +msgid "enable_blending" +msgstr "enable_blending" + +#: ../../source/api_reference/configurations.rst:154 +msgid "LMCACHE_ENABLE_BLENDING" +msgstr "LMCACHE_ENABLE_BLENDING" + +#: ../../source/api_reference/configurations.rst:155 +msgid "Whether to enable blending. Values: true/false. Default: false" +msgstr "是否启用混合。值：true/false。默认值：false" + +#: ../../source/api_reference/configurations.rst:156 +msgid "blend_recompute_ratios" +msgstr "blend_recompute_ratios" + +#: ../../source/api_reference/configurations.rst:157 +msgid "LMCACHE_BLEND_RECOMPUTE_RATIOS" +msgstr "LMCACHE_BLEND_RECOMPUTE_RATIOS" + +#: ../../source/api_reference/configurations.rst:158 +msgid "Ratio of blending recompute. Default: 0.15" +msgstr "混合重计算的比例。默认值：0.15" + +#: ../../source/api_reference/configurations.rst:159 +msgid "blend_check_layers" +msgstr "blend_check_layers" + +#: ../../source/api_reference/configurations.rst:160 +msgid "LMCACHE_BLEND_CHECK_LAYERS" +msgstr "LMCACHE_BLEND_CHECK_LAYERS" + +#: ../../source/api_reference/configurations.rst:161 +msgid "Layers to determine the recomputed tokens. Default: 1" +msgstr "确定重计算令牌的层数。默认值：1" + +#: ../../source/api_reference/configurations.rst:162 +msgid "blend_special_str" +msgstr "blend_special_str" + +#: ../../source/api_reference/configurations.rst:163 +msgid "LMCACHE_BLEND_SPECIAL_STR" +msgstr "LMCACHE_BLEND_SPECIAL_STR" + +#: ../../source/api_reference/configurations.rst:164 +msgid "Separator string for blending. Default: \" # # \"" +msgstr "用于混合的分隔符字符串。默认值：\\\" # # \\\"" + +#: ../../source/api_reference/configurations.rst:167 +msgid "Peer-to-Peer Sharing Configurations" +msgstr "点对点共享配置" + +#: ../../source/api_reference/configurations.rst:169 +msgid "" +"Settings for enabling and configuring peer-to-peer CPU KV cache sharing " +"and global KV cache lookup." +msgstr "用于启用和配置点对点 CPU KV Cache 共享及全局 KV Cache 查找的设置。" + +#: ../../source/api_reference/configurations.rst:178 +msgid "enable_p2p" +msgstr "enable_p2p" + +#: ../../source/api_reference/configurations.rst:179 +msgid "LMCACHE_ENABLE_P2P" +msgstr "LMCACHE_ENABLE_P2P" + +#: ../../source/api_reference/configurations.rst:180 +msgid "Whether to enable peer-to-peer sharing. Values: true/false. Default: false" +msgstr "是否启用点对点共享。值：true/false。默认：false" + +#: ../../source/api_reference/configurations.rst:181 +msgid "p2p_host" +msgstr "p2p_host" + +#: ../../source/api_reference/configurations.rst:182 +msgid "LMCACHE_P2P_HOST" +msgstr "LMCACHE_P2P_HOST" + +#: ../../source/api_reference/configurations.rst:183 +msgid "Ip address. Required if enable_p2p is true" +msgstr "IP 地址。如果 enable_p2p 为 true，则必需。" + +#: ../../source/api_reference/configurations.rst:184 +msgid "peer_init_ports" +msgstr "peer_init_ports" + +#: ../../source/api_reference/configurations.rst:185 +msgid "LMCACHE_PEER_INIT_PORTS" +msgstr "LMCACHE_PEER_INIT_PORTS" + +#: ../../source/api_reference/configurations.rst:186 +msgid "Ports for p2p peer init. Required if enable_p2p is true" +msgstr "p2p 对等初始化的端口。如果 enable_p2p 为 true，则需要此项。" + +#: ../../source/api_reference/configurations.rst:187 +msgid "peer_lookup_ports" +msgstr "peer_lookup_ports" + +#: ../../source/api_reference/configurations.rst:188 +msgid "LMCACHE_PEER_lookup_PORTS" +msgstr "LMCACHE_PEER_lookup_PORTS" + +#: ../../source/api_reference/configurations.rst:189 +msgid "Ports for p2p peer lookup. Required if enable_p2p is true" +msgstr "用于点对点（p2p）对等查找的端口。如果 enable_p2p 为 true，则必需。" + +#: ../../source/api_reference/configurations.rst:190 +#: ../../source/api_reference/configurations.rst:242 +msgid "transfer_channel" +msgstr "transfer_channel" + +#: ../../source/api_reference/configurations.rst:191 +#: ../../source/api_reference/configurations.rst:243 +msgid "LMCACHE_TRANSFER_CHANNEL" +msgstr "LMCACHE_TRANSFER_CHANNEL" + +#: ../../source/api_reference/configurations.rst:192 +msgid "Such as `nixl`. Required if enable_p2p is true" +msgstr "例如 `nixl`。如果 enable_p2p 为 true，则必需。" + +#: ../../source/api_reference/configurations.rst:195 +msgid "Controller Configurations" +msgstr "控制器配置" + +#: ../../source/api_reference/configurations.rst:197 +msgid "Settings for the KV cache controller functionality." +msgstr "KV 缓存控制器功能的设置。" + +#: ../../source/api_reference/configurations.rst:206 +msgid "enable_controller" +msgstr "enable_controller" + +#: ../../source/api_reference/configurations.rst:207 +msgid "LMCACHE_ENABLE_CONTROLLER" +msgstr "LMCACHE_ENABLE_CONTROLLER" + +#: ../../source/api_reference/configurations.rst:208 +msgid "Whether to enable controller. Values: true/false. Default: false" +msgstr "是否启用控制器。值：true/false。默认：false" + +#: ../../source/api_reference/configurations.rst:209 +msgid "lmcache_instance_id" +msgstr "lmcache_instance_id" + +#: ../../source/api_reference/configurations.rst:210 +msgid "LMCACHE_LMCACHE_INSTANCE_ID" +msgstr "LMCACHE_LMCACHE_INSTANCE_ID" + +#: ../../source/api_reference/configurations.rst:211 +msgid "ID of the LMCache instance. Default: \"lmcache_default_instance\"" +msgstr "LMCache 实例的 ID。默认值：\\\"lmcache_default_instance\\\"" + +#: ../../source/api_reference/configurations.rst:212 +msgid "controller_url" +msgstr "controller_url" + +#: ../../source/api_reference/configurations.rst:213 +msgid "LMCACHE_CONTROLLER_URL" +msgstr "LMCACHE_CONTROLLER_URL" + +#: ../../source/api_reference/configurations.rst:214 +msgid "URL of the controller server" +msgstr "控制器服务器的 URL" + +#: ../../source/api_reference/configurations.rst:215 +msgid "lmcache_worker_port" +msgstr "lmcache_worker_port" + +#: ../../source/api_reference/configurations.rst:216 +msgid "LMCACHE_LMCACHE_WORKER_PORT" +msgstr "LMCACHE_LMCACHE_WORKER_PORT" + +#: ../../source/api_reference/configurations.rst:217 +msgid "Port number for LMCache worker" +msgstr "LMCache 工作进程的端口号" + +#: ../../source/api_reference/configurations.rst:220 +msgid "Disaggregated Prefill Configurations" +msgstr "分离式 Prefill 配置" + +#: ../../source/api_reference/configurations.rst:222 +msgid "" +"Settings for disaggregated prefill functionality. The latest/default PD " +"is implemented inside of `lmcache/v1/storage_backend/pd_backend.py`." +msgstr "分离式 Prefill 功能的设置。最新/默认的 PD 实现在 `lmcache/v1/storage_backend/pd_backend.py` 中。" + +#: ../../source/api_reference/configurations.rst:226 +msgid "" +"When PD is enabled, the following restrictions apply (welcome " +"contributions to remove these restrictions):" +msgstr "启用 PD 时，以下限制适用（欢迎贡献以消除这些限制）：" + +#: ../../source/api_reference/configurations.rst:228 +msgid "remote_url must be null" +msgstr "remote_url 必须为 null" + +#: ../../source/api_reference/configurations.rst:229 +msgid "save_decode_cache must be false" +msgstr "save_decode_cache 必须为 false" + +#: ../../source/api_reference/configurations.rst:230 +msgid "enable_p2p must be false" +msgstr "enable_p2p 必须为 false" + +#: ../../source/api_reference/configurations.rst:239 +msgid "enable_pd" +msgstr "enable_pd" + +#: ../../source/api_reference/configurations.rst:240 +msgid "LMCACHE_ENABLE_PD" +msgstr "LMCACHE_ENABLE_PD" + +#: ../../source/api_reference/configurations.rst:241 +msgid "Whether to enable PD. Values: true/false. Default: false" +msgstr "是否启用 PD。值：true/false。默认值：false" + +#: ../../source/api_reference/configurations.rst:244 +msgid "Transfer channel used for PD. Values: \"nixl\". Default: none" +msgstr "用于 PD 的传输通道。值：“nixl”。默认：无" + +#: ../../source/api_reference/configurations.rst:245 +msgid "pd_role" +msgstr "pd_role" + +#: ../../source/api_reference/configurations.rst:246 +msgid "LMCACHE_PD_ROLE" +msgstr "LMCACHE_PD_ROLE" + +#: ../../source/api_reference/configurations.rst:247 +msgid "PD role. Values: \"sender\" (prefiller) or \"receiver\" (decoder)." +msgstr "PD 角色。值：\\\"sender\\\"（预填充器）或 \\\"receiver\\\"（解码器）。" + +#: ../../source/api_reference/configurations.rst:248 +msgid "pd_buffer_size" +msgstr "pd_buffer_size" + +#: ../../source/api_reference/configurations.rst:249 +msgid "LMCACHE_PD_BUFFER_SIZE" +msgstr "LMCACHE_PD_BUFFER_SIZE" + +#: ../../source/api_reference/configurations.rst:250 +msgid "" +"Upper bound of PD transport buffer size (in bytes), aligned to chunk " +"size. Required for both senders and receivers when enable_pd=true" +msgstr "PD 传输缓冲区大小的上限（以字节为单位），与块大小对齐。当 enable_pd=true 时，发送方和接收方均需要此设置。" + +#: ../../source/api_reference/configurations.rst:251 +msgid "pd_buffer_device" +msgstr "pd_buffer_device" + +#: ../../source/api_reference/configurations.rst:252 +msgid "LMCACHE_PD_BUFFER_DEVICE" +msgstr "LMCACHE_PD_BUFFER_DEVICE" + +#: ../../source/api_reference/configurations.rst:253 +msgid "" +"Device for PD buffer. Values: \"cpu\", \"cuda\". Required for both " +"senders and receivers when enable_pd=true" +msgstr "PD 缓冲区的设备。值：\\\"cpu\\\"，\\\"cuda\\\"。当 enable_pd=true 时，发送者和接收者都需要此项。" + +#: ../../source/api_reference/configurations.rst:254 +msgid "nixl_backends" +msgstr "nixl_backends" + +#: ../../source/api_reference/configurations.rst:255 +msgid "LMCACHE_NIXL_BACKENDS" +msgstr "LMCACHE_NIXL_BACKENDS" + +#: ../../source/api_reference/configurations.rst:256 +msgid "" +"List of Nixl transport backends. Useful for non-disaggregated use case " +"(see below). UCX default is sufficient for disagg use case. Default: " +"[\"UCX\"]" +msgstr "Nixl 传输后端列表。适用于非分离式用例（见下文）。UCX 默认设置足以满足分离式用例。默认值：[\\\"UCX\\\"]" + +#: ../../source/api_reference/configurations.rst:257 +msgid "pd_peer_host" +msgstr "pd_peer_host" + +#: ../../source/api_reference/configurations.rst:258 +msgid "LMCACHE_PD_PEER_HOST" +msgstr "LMCACHE_PD_PEER_HOST" + +#: ../../source/api_reference/configurations.rst:259 +msgid "Host for peer connections. Required for receivers to bind to" +msgstr "对等连接的主机。接收方绑定所需。" + +#: ../../source/api_reference/configurations.rst:260 +msgid "pd_peer_init_port" +msgstr "pd_peer_init_port" + +#: ../../source/api_reference/configurations.rst:261 +msgid "LMCACHE_PD_PEER_INIT_PORT" +msgstr "LMCACHE_PD_PEER_INIT_PORT" + +#: ../../source/api_reference/configurations.rst:262 +msgid "" +"Initialization port for peer connections. Required for receivers to bind " +"to" +msgstr "用于对等连接的初始化端口。接收方需要绑定到此端口。" + +#: ../../source/api_reference/configurations.rst:263 +msgid "pd_peer_alloc_port" +msgstr "pd_peer_alloc_port" + +#: ../../source/api_reference/configurations.rst:264 +msgid "LMCACHE_PD_PEER_ALLOC_PORT" +msgstr "LMCACHE_PD_PEER_ALLOC_PORT" + +#: ../../source/api_reference/configurations.rst:265 +msgid "Allocation port for peer connections. Required for receivers to bind to" +msgstr "用于对等连接的分配端口。接收方需要绑定到此端口。" + +#: ../../source/api_reference/configurations.rst:266 +msgid "pd_proxy_host" +msgstr "pd_proxy_host" + +#: ../../source/api_reference/configurations.rst:267 +msgid "LMCACHE_PD_PROXY_HOST" +msgstr "LMCACHE_PD_PROXY_HOST" + +#: ../../source/api_reference/configurations.rst:268 +msgid "" +"Host for proxy server. Required for senders to connect to inform the " +"proxy when transfer to decoder has been completed" +msgstr "代理服务器的主机。发送方连接以通知代理解码器传输完成时必需。" + +#: ../../source/api_reference/configurations.rst:269 +msgid "pd_proxy_port" +msgstr "pd_proxy_port" + +#: ../../source/api_reference/configurations.rst:270 +msgid "LMCACHE_PD_PROXY_PORT" +msgstr "LMCACHE_PD_PROXY_PORT" + +#: ../../source/api_reference/configurations.rst:271 +msgid "" +"Port for proxy server. Required for senders to connect to inform the " +"proxy when transfer to decoder has been completed" +msgstr "代理服务器的端口。发送方需要连接以通知代理传输到解码器已完成。" + +#: ../../source/api_reference/configurations.rst:272 +msgid "pd_allocation_timeout_sec" +msgstr "pd_allocation_timeout_sec" + +#: ../../source/api_reference/configurations.rst:273 +msgid "LMCACHE_PD_ALLOCATION_TIMEOUT_SEC" +msgstr "LMCACHE_PD_ALLOCATION_TIMEOUT_SEC" + +#: ../../source/api_reference/configurations.rst:274 +msgid "Maximum seconds to retry memory allocation before giving up. Default: 5.0" +msgstr "在放弃之前重试内存分配的最大秒数。默认值：5.0" + +#: ../../source/api_reference/configurations.rst:275 +msgid "pd_shutdown_timeout_sec" +msgstr "pd_shutdown_timeout_sec" + +#: ../../source/api_reference/configurations.rst:276 +msgid "LMCACHE_PD_SHUTDOWN_TIMEOUT_SEC" +msgstr "LMCACHE_PD_SHUTDOWN_TIMEOUT_SEC" + +#: ../../source/api_reference/configurations.rst:277 +msgid "" +"Maximum seconds to wait for event loop shutdown and thread join. Default:" +" 5.0" +msgstr "事件循环关闭和线程加入的最大等待秒数。默认值：5.0" + +#: ../../source/api_reference/configurations.rst:278 +msgid "pd_condition_poll_interval_sec" +msgstr "pd_condition_poll_interval_sec" + +#: ../../source/api_reference/configurations.rst:279 +msgid "LMCACHE_PD_CONDITION_POLL_INTERVAL_SEC" +msgstr "LMCACHE_PD_CONDITION_POLL_INTERVAL_SEC" + +#: ../../source/api_reference/configurations.rst:280 +msgid "" +"Polling interval in seconds when waiting on a threading/asyncio " +"Condition. Small enough to be responsive, large enough not to spin-waste " +"CPU. Default: 0.05" +msgstr "在等待线程/asyncio Condition 时的轮询间隔（以秒为单位）。足够小以保持响应，足够大以避免浪费 CPU。默认值：0.05" + +#: ../../source/api_reference/configurations.rst:281 +msgid "pd_max_prefill_len" +msgstr "pd_max_prefill_len" + +#: ../../source/api_reference/configurations.rst:282 +msgid "LMCACHE_PD_MAX_PREFILL_LEN" +msgstr "LMCACHE_PD_MAX_PREFILL_LEN" + +#: ../../source/api_reference/configurations.rst:283 +msgid "" +"Maximum prefill token length that the PD buffer must be able to hold. If " +"> 0, initialization raises ValueError when the buffer capacity (in " +"tokens) is smaller than this value. Set to 0 (default) to skip the check." +msgstr "PD 缓冲区必须能够容纳的最大 Prefill 令牌长度。如果大于 0，当缓冲区容量（以令牌为单位）小于此值时，初始化将引发 ValueError。设置为 0（默认值）以跳过检查。" + +#: ../../source/api_reference/configurations.rst:284 +msgid "pd_backend_mode" +msgstr "pd_backend_mode" + +#: ../../source/api_reference/configurations.rst:285 +msgid "LMCACHE_PD_BACKEND_MODE" +msgstr "LMCACHE_PD_BACKEND_MODE" + +#: ../../source/api_reference/configurations.rst:286 +msgid "" +"Select the PD backend implementation: 'async' (default) uses the asyncio-" +"based implementation; 'sync' uses the original thread-based synchronous " +"implementation. Default: \"async\"" +msgstr "选择 PD 后端实现：'async'（默认）使用基于 asyncio 的实现；'sync' 使用原始的基于线程的同步实现。默认值：\\\"async\\\"" + +#: ../../source/api_reference/configurations.rst:287 +msgid "pd_skip_proxy_notification" +msgstr "pd_skip_proxy_notification" + +#: ../../source/api_reference/configurations.rst:288 +msgid "LMCACHE_PD_SKIP_PROXY_NOTIFICATION" +msgstr "LMCACHE_PD_SKIP_PROXY_NOTIFICATION" + +#: ../../source/api_reference/configurations.rst:289 +msgid "" +"When true, the sender skips ZMQ proxy notification after KV transfer and " +"does not require pd_proxy_host/pd_proxy_port. This option is intended for" +" external orchestrators only (e.g., vLLM Production Stack router) that " +"manage the prefill-decode request flow via HTTP and do not rely on ZMQ " +"notifications. It must not be used with LMCache's built-in disaggregation" +" proxy (``disagg_proxy_server.py``), which depends on ZMQ notifications " +"to know when KV transfer is complete before forwarding the decode " +"request. Values: true/false. Default: false" +msgstr "当为真时，发送方在 KV 传输后跳过 ZMQ 代理通知，并且不需要 pd_proxy_host/pd_proxy_port。此选项仅适用于管理通过 HTTP 的分离式解码请求流的外部调度器（例如，vLLM 生产堆栈路由器），并且不依赖于 ZMQ 通知。它不得与 LMCache 的内置分离代理（``disagg_proxy_server.py``）一起使用，因为该代理依赖于 ZMQ 通知来知道 KV 传输何时完成，然后再转发解码请求。值：true/false。默认值：false" + +#: ../../source/api_reference/configurations.rst:290 +msgid "pd_bidirectional" +msgstr "pd_bidirectional" + +#: ../../source/api_reference/configurations.rst:291 +msgid "LMCACHE_PD_BIDIRECTIONAL" +msgstr "LMCACHE_PD_BIDIRECTIONAL" + +#: ../../source/api_reference/configurations.rst:292 +msgid "" +"When true, enables bidirectional NIXL cache probe. The prefiller queries " +"the decoder for cached KV blocks before transfer, and reads cached blocks" +" via NIXL RDMA instead of recomputing. Values: true/false. Default: false" +msgstr "当为真时，启用双向 NIXL 缓存探测。Prefiller 在传输之前查询解码器以获取缓存的 KV 块，并通过 NIXL RDMA 读取缓存的块，而不是重计算。值：true/false。默认值：false" + +#: ../../source/api_reference/configurations.rst:293 +msgid "pd_peer_query_port" +msgstr "pd_peer_query_port" + +#: ../../source/api_reference/configurations.rst:294 +msgid "LMCACHE_PD_PEER_QUERY_PORT" +msgstr "LMCACHE_PD_PEER_QUERY_PORT" + +#: ../../source/api_reference/configurations.rst:295 +msgid "" +"ZMQ ports for the bidirectional cache query channel (one per TP rank). " +"Required on both prefiller and decoder when pd_bidirectional=true. " +"Example: [7500, 7501, 7502, 7503]" +msgstr "用于双向缓存查询通道的 ZMQ 端口（每个 TP 排名一个）。当 pd_bidirectional=true 时，预填充器和解码器都需要此配置。示例：[7500, 7501, 7502, 7503]" + +#: ../../source/api_reference/configurations.rst:298 +msgid "P2P Backend Configurations" +msgstr "P2P 后端配置" + +#: ../../source/api_reference/configurations.rst:300 +msgid "" +"Settings for P2P (peer-to-peer) backend timeout behavior. These " +"configurations are specified through ``extra_config``." +msgstr "P2P（点对点）后端超时行为的设置。这些配置通过 ``extra_config`` 指定。" + +#: ../../source/api_reference/configurations.rst:312 +#: ../../source/api_reference/configurations.rst:345 +msgid "Configuration Key" +msgstr "配置键" + +#: ../../source/api_reference/configurations.rst:313 +msgid "Default" +msgstr "默认" + +#: ../../source/api_reference/configurations.rst:315 +msgid "p2p_socket_recv_timeout_ms" +msgstr "p2p_socket_recv_timeout_ms" + +#: ../../source/api_reference/configurations.rst:316 +msgid "30000" +msgstr "30000" + +#: ../../source/api_reference/configurations.rst:317 +msgid "Timeout in milliseconds for socket receive operations" +msgstr "套接字接收操作的超时时间（以毫秒为单位）" + +#: ../../source/api_reference/configurations.rst:318 +msgid "p2p_socket_send_timeout_ms" +msgstr "p2p_socket_send_timeout_ms" + +#: ../../source/api_reference/configurations.rst:319 +msgid "10000" +msgstr "10000" + +#: ../../source/api_reference/configurations.rst:320 +msgid "Timeout in milliseconds for socket send operations" +msgstr "套接字发送操作的超时时间（以毫秒为单位）" + +#: ../../source/api_reference/configurations.rst:323 +msgid "Nixl (as a storage backend) Configurations" +msgstr "Nixl（作为存储后端）配置" + +#: ../../source/api_reference/configurations.rst:325 +msgid "" +"Settings for using Nixl as a storage backend instead of disaggregated " +"prefill. This mode requires additional configurations in " +"``extra_config``." +msgstr "将 Nixl 用作存储后端而不是分离式 Prefill 的设置。此模式需要在 ``extra_config`` 中进行额外配置。" + +#: ../../source/api_reference/configurations.rst:329 +msgid "" +"This is a different mode from disaggregated prefill. When using Nixl as a" +" storage backend, you need to configure it through ``extra_config``." +msgstr "这是一种不同于分离式 Prefill 的模式。当使用 Nixl 作为存储后端时，您需要通过 ``extra_config`` 进行配置。" + +#: ../../source/api_reference/configurations.rst:347 +msgid "enable_nixl_storage" +msgstr "enable_nixl_storage" + +#: ../../source/api_reference/configurations.rst:348 +msgid "Whether to enable Nixl storage backend. Values: true/false" +msgstr "是否启用 Nixl 存储后端。值：true/false" + +#: ../../source/api_reference/configurations.rst:349 +msgid "nixl_backend" +msgstr "nixl_backend" + +#: ../../source/api_reference/configurations.rst:350 +msgid "" +"Storage backend type. Options: \"GDS\", \"GDS_MT\", \"POSIX\", \"HF3FS\"," +" \"OBJ\"" +msgstr "存储后端类型。选项：“GDS”、“GDS_MT”、“POSIX”、“HF3FS”、“OBJ”" + +#: ../../source/api_reference/configurations.rst:351 +msgid "nixl_path" +msgstr "nixl_path" + +#: ../../source/api_reference/configurations.rst:352 +msgid "File system path for Nixl storage" +msgstr "Nixl 存储的文件系统路径" + +#: ../../source/api_reference/configurations.rst:353 +msgid "nixl_pool_size" +msgstr "nixl_pool_size" + +#: ../../source/api_reference/configurations.rst:354 +msgid "Number of files or objects in the storage pool" +msgstr "存储池中的文件或对象数量" + +#: ../../source/api_reference/configurations.rst:355 +msgid "nixl_endpoint_list" +msgstr "nixl_endpoint_list" + +#: ../../source/api_reference/configurations.rst:356 +msgid "" +"List of object-storage endpoint URLs for per-worker distribution. " +"Overrides ``nixl_backend_params.endpoint_override`` when set." +msgstr "每个工作节点分发的对象存储端点 URL 列表。当设置时，会覆盖 ``nixl_backend_params.endpoint_override``。" + +#: ../../source/api_reference/configurations.rst:360 +msgid "Additional Storage Configurations" +msgstr "附加存储配置" + +#: ../../source/api_reference/configurations.rst:362 +msgid "Settings for different storage backends and paths." +msgstr "不同存储后端和路径的设置。" + +#: ../../source/api_reference/configurations.rst:371 +msgid "gds_path" +msgstr "gds_path" + +#: ../../source/api_reference/configurations.rst:372 +msgid "LMCACHE_GDS_PATH" +msgstr "LMCACHE_GDS_PATH" + +#: ../../source/api_reference/configurations.rst:373 +msgid "" +"Path for GDS backend. Supports comma-separated paths for multi-device I/O" +" (e.g. ``/mnt/nvme0/cache,/mnt/nvme1/cache``). See ``gds_path_sharding`` " +"for how paths are assigned to GPUs." +msgstr "GDS 后端的路径。支持以逗号分隔的多设备 I/O 路径（例如 ``/mnt/nvme0/cache,/mnt/nvme1/cache``）。有关路径如何分配给 GPU，请参见 ``gds_path_sharding``。" + +#: ../../source/api_reference/configurations.rst:374 +msgid "gds_path_sharding" +msgstr "gds_path_sharding" + +#: ../../source/api_reference/configurations.rst:375 +msgid "LMCACHE_GDS_PATH_SHARDING" +msgstr "LMCACHE_GDS_PATH_SHARDING" + +#: ../../source/api_reference/configurations.rst:377 +msgid "gds_buffer_size" +msgstr "gds_buffer_size" + +#: ../../source/api_reference/configurations.rst:378 +msgid "LMCACHE_GDS_BUFFER_SIZE" +msgstr "LMCACHE_GDS_BUFFER_SIZE" + +#: ../../source/api_reference/configurations.rst:379 +msgid "Buffer size for GDS operations" +msgstr "GDS 操作的缓冲区大小" + +#: ../../source/api_reference/configurations.rst:380 +msgid "use_gds" +msgstr "use_gds" + +#: ../../source/api_reference/configurations.rst:381 +msgid "LMCACHE_USE_GDS" +msgstr "LMCACHE_USE_GDS" + +#: ../../source/api_reference/configurations.rst:382 +msgid "Enable or disable GPU Direct Storage API usage (default: true)" +msgstr "启用或禁用 GPU 直接存储 API 的使用（默认：true）" + +#: ../../source/api_reference/configurations.rst:383 +msgid "gds_backend" +msgstr "gds_backend" + +#: ../../source/api_reference/configurations.rst:384 +msgid "LMCACHE_GDS_BACKEND" +msgstr "LMCACHE_GDS_BACKEND" + +#: ../../source/api_reference/configurations.rst:385 +msgid "GDS library backend to use (default: \"cufile\")" +msgstr "使用的 GDS 库后端（默认：\\\"cufile\\\"）" + +#: ../../source/api_reference/configurations.rst:388 +msgid "Custom Prometheus Histogram Buckets" +msgstr "自定义 Prometheus 直方图桶" + +#: ../../source/api_reference/configurations.rst:390 +msgid "" +"You can override the default bucket boundaries for any Prometheus " +"histogram metric by adding a key ``histogram_bucket_`` to " +"``extra_config``, where ```` is the metric name **without** the " +"``lmcache:`` prefix." +msgstr "您可以通过向 ``extra_config`` 添加一个键 ``histogram_bucket_`` 来覆盖任何 Prometheus 直方图度量的默认桶边界，其中 ```` 是度量名称 **不带** ``lmcache:`` 前缀。" + +#: ../../source/api_reference/configurations.rst:394 +msgid "The value must be a list of numeric boundaries (floats or ints)." +msgstr "值必须是一个数字边界的列表（浮点数或整数）。" + +#: ../../source/api_reference/configurations.rst:406 +msgid "Available Histogram Names" +msgstr "可用直方图名称" + +#: ../../source/api_reference/configurations.rst:408 +msgid "``time_to_retrieve``" +msgstr "``time_to_retrieve``" + +#: ../../source/api_reference/configurations.rst:409 +msgid "Time to retrieve from cache (seconds)" +msgstr "从缓存中检索所需时间（秒）" + +#: ../../source/api_reference/configurations.rst:410 +msgid "``time_to_store``" +msgstr "``time_to_store``" + +#: ../../source/api_reference/configurations.rst:411 +msgid "Time to store to cache (seconds)" +msgstr "缓存存储时间（秒）" + +#: ../../source/api_reference/configurations.rst:412 +msgid "``time_to_lookup``" +msgstr "``time_to_lookup``" + +#: ../../source/api_reference/configurations.rst:413 +msgid "Time to lookup in cache (seconds)" +msgstr "缓存查找时间（秒）" + +#: ../../source/api_reference/configurations.rst:414 +msgid "``retrieve_process_tokens_time``" +msgstr "``retrieve_process_tokens_time``" + +#: ../../source/api_reference/configurations.rst:415 +msgid "Time to process tokens in retrieve (seconds)" +msgstr "在检索中处理令牌的时间（秒）" + +#: ../../source/api_reference/configurations.rst:416 +msgid "``retrieve_broadcast_time``" +msgstr "``retrieve_broadcast_time``" + +#: ../../source/api_reference/configurations.rst:417 +msgid "Time to broadcast memory objects in retrieve (seconds)" +msgstr "在检索中广播内存对象的时间（秒）" + +#: ../../source/api_reference/configurations.rst:418 +msgid "``retrieve_to_gpu_time``" +msgstr "``retrieve_to_gpu_time``" + +#: ../../source/api_reference/configurations.rst:419 +msgid "Time to move data to GPU in retrieve (seconds)" +msgstr "在检索中将数据移动到 GPU 的时间（秒）" + +#: ../../source/api_reference/configurations.rst:420 +msgid "``remote_backend_batched_get_blocking_time``" +msgstr "``remote_backend_batched_get_blocking_time``" + +#: ../../source/api_reference/configurations.rst:421 +msgid "Time to get data from remote backend (seconds)" +msgstr "从远程后端获取数据的时间（秒）" + +#: ../../source/api_reference/configurations.rst:422 +msgid "``instrumented_connector_batched_get_time``" +msgstr "``instrumented_connector_batched_get_time``" + +#: ../../source/api_reference/configurations.rst:423 +msgid "Time used by the connector (seconds)" +msgstr "连接器使用的时间（秒）" + +#: ../../source/api_reference/configurations.rst:424 +msgid "``store_process_tokens_time``" +msgstr "``store_process_tokens_time``" + +#: ../../source/api_reference/configurations.rst:425 +msgid "Time to process tokens in store (seconds)" +msgstr "存储中处理令牌的时间（秒）" + +#: ../../source/api_reference/configurations.rst:426 +msgid "``store_from_gpu_time``" +msgstr "``store_from_gpu_time``" + +#: ../../source/api_reference/configurations.rst:427 +msgid "Time to move data from GPU in store (seconds)" +msgstr "从 GPU 移动数据到存储的时间（秒）" + +#: ../../source/api_reference/configurations.rst:428 +msgid "``store_put_time``" +msgstr "``store_put_time``" + +#: ../../source/api_reference/configurations.rst:429 +msgid "Time to put data to storage in store (seconds)" +msgstr "将数据存储到存储中的时间（秒）" + +#: ../../source/api_reference/configurations.rst:430 +msgid "``retrieve_speed``" +msgstr "``retrieve_speed``" + +#: ../../source/api_reference/configurations.rst:431 +msgid "Retrieve speed (tokens per second)" +msgstr "检索速度（每秒令牌数）" + +#: ../../source/api_reference/configurations.rst:432 +msgid "``store_speed``" +msgstr "``store_speed``" + +#: ../../source/api_reference/configurations.rst:433 +msgid "Store speed (tokens per second)" +msgstr "存储速度（每秒令牌数）" + +#: ../../source/api_reference/configurations.rst:434 +msgid "``p2p_time_to_transfer``" +msgstr "``p2p_time_to_transfer``" + +#: ../../source/api_reference/configurations.rst:435 +msgid "Time to transfer via P2P (seconds)" +msgstr "通过 P2P 传输的时间（秒）" + +#: ../../source/api_reference/configurations.rst:436 +msgid "``p2p_transfer_speed``" +msgstr "``p2p_transfer_speed``" + +#: ../../source/api_reference/configurations.rst:437 +msgid "P2P transfer speed (tokens per second)" +msgstr "P2P 传输速度（每秒令牌数）" + +#: ../../source/api_reference/configurations.rst:438 +msgid "``remote_time_to_get``" +msgstr "``remote_time_to_get``" + +#: ../../source/api_reference/configurations.rst:439 +msgid "Time to get from remote backends (ms)" +msgstr "从远程后端获取的时间（毫秒）" + +#: ../../source/api_reference/configurations.rst:440 +msgid "``remote_time_to_put``" +msgstr "``remote_time_to_put``" + +#: ../../source/api_reference/configurations.rst:441 +msgid "Time to put to remote backends (ms)" +msgstr "远程后端的放置时间（毫秒）" + +#: ../../source/api_reference/configurations.rst:442 +msgid "``remote_time_to_get_sync``" +msgstr "``remote_time_to_get_sync``" + +#: ../../source/api_reference/configurations.rst:443 +msgid "Time to get from remote backends synchronously (ms)" +msgstr "从远程后端同步获取的时间（毫秒）" + +#: ../../source/api_reference/configurations.rst:444 +msgid "``request_cache_hit_rate``" +msgstr "``request_cache_hit_rate``" + +#: ../../source/api_reference/configurations.rst:445 +msgid "Request cache hit rate (0.0 to 1.0)" +msgstr "请求缓存命中率 (0.0 到 1.0)" + +#: ../../source/api_reference/configurations.rst:446 +msgid "``request_cache_lifespan``" +msgstr "``request_cache_lifespan``" + +#: ../../source/api_reference/configurations.rst:447 +msgid "Request cache lifespan (minutes)" +msgstr "请求缓存生命周期（分钟）" + +#: ../../source/api_reference/configurations.rst:450 +msgid "Internal API Server Configurations" +msgstr "内部 API 服务器配置" + +#: ../../source/api_reference/configurations.rst:452 +msgid "" +"Settings for the internal API server that provides management and " +"debugging APIs for LMCache engines. The API server runs on each worker " +"and scheduler, allowing you to inspect and control LMCache behavior at " +"runtime." +msgstr "用于内部 API 服务器的设置，该服务器为 LMCache 引擎提供管理和调试 API。API 服务器在每个工作节点和调度器上运行，允许您在运行时检查和控制 LMCache 的行为。" + +#: ../../source/api_reference/configurations.rst:456 +msgid "The internal API server provides endpoints for:" +msgstr "内部 API 服务器提供以下端点：" + +#: ../../source/api_reference/configurations.rst:458 +msgid "**Metrics**: Performance and cache statistics" +msgstr "**指标**：性能和缓存统计信息" + +#: ../../source/api_reference/configurations.rst:459 +msgid "**Configuration**: Runtime configuration inspection" +msgstr "**配置**: 运行时配置检查" + +#: ../../source/api_reference/configurations.rst:460 +msgid "**Metadata**: Engine and model metadata" +msgstr "**元数据**：引擎和模型元数据" + +#: ../../source/api_reference/configurations.rst:461 +msgid "**Threads**: Thread debugging information" +msgstr "**线程**：线程调试信息" + +#: ../../source/api_reference/configurations.rst:462 +msgid "**Log Level**: Dynamic log level adjustment" +msgstr "**日志级别**：动态日志级别调整" + +#: ../../source/api_reference/configurations.rst:463 +msgid "" +"**Script Execution**: Run custom Python scripts with access to the " +"LMCache engine" +msgstr "**脚本执行**：运行自定义 Python 脚本，访问 LMCache 引擎" + +#: ../../source/api_reference/configurations.rst:466 +msgid "Configuration Options" +msgstr "配置选项" + +#: ../../source/api_reference/configurations.rst:475 +msgid "internal_api_server_enabled" +msgstr "internal_api_server_enabled" + +#: ../../source/api_reference/configurations.rst:476 +msgid "LMCACHE_INTERNAL_API_SERVER_ENABLED" +msgstr "LMCACHE_INTERNAL_API_SERVER_ENABLED" + +#: ../../source/api_reference/configurations.rst:477 +msgid "Whether to enable internal API server. Default: false" +msgstr "是否启用内部 API 服务器。默认值：false" + +#: ../../source/api_reference/configurations.rst:478 +msgid "internal_api_server_host" +msgstr "internal_api_server_host" + +#: ../../source/api_reference/configurations.rst:479 +msgid "LMCACHE_INTERNAL_API_SERVER_HOST" +msgstr "LMCACHE_INTERNAL_API_SERVER_HOST" + +#: ../../source/api_reference/configurations.rst:480 +msgid "Host for internal API server to bind to. Default: \"0.0.0.0\"" +msgstr "用于绑定内部 API 服务器的主机。默认值：“0.0.0.0”" + +#: ../../source/api_reference/configurations.rst:481 +msgid "internal_api_server_port_start" +msgstr "internal_api_server_port_start" + +#: ../../source/api_reference/configurations.rst:482 +msgid "LMCACHE_INTERNAL_API_SERVER_PORT_START" +msgstr "LMCACHE_INTERNAL_API_SERVER_PORT_START" + +#: ../../source/api_reference/configurations.rst:483 +msgid "" +"Starting port for internal API server. Port assignment: Scheduler = " +"port_start + 0, Worker i = port_start + i + 1. Example: If " +"port_start=6999, then Scheduler=6999, Worker 0=7000, Worker 1=7001, etc. " +"Default: 6999" +msgstr "内部 API 服务器的起始端口。端口分配：调度器 = port_start + 0，工作者 i = port_start + i + 1。示例：如果 port_start=6999，则调度器=6999，工作者 0=7000，工作者 1=7001，等等。默认值：6999" + +#: ../../source/api_reference/configurations.rst:484 +msgid "internal_api_server_include_index_list" +msgstr "internal_api_server_include_index_list" + +#: ../../source/api_reference/configurations.rst:485 +msgid "LMCACHE_INTERNAL_API_SERVER_INCLUDE_INDEX_LIST" +msgstr "LMCACHE_INTERNAL_API_SERVER_INCLUDE_INDEX_LIST" + +#: ../../source/api_reference/configurations.rst:486 +msgid "" +"List of worker/scheduler indices to enable API server on. Use 0 for " +"scheduler, 1 for worker 0, 2 for worker 1, etc. If null, enables on all " +"workers/scheduler. Example: [0, 1] enables only on scheduler and worker " +"0. Default: null" +msgstr "启用 API 服务器的工作者/调度器索引列表。使用 0 表示调度器，1 表示工作者 0，2 表示工作者 1，依此类推。如果为 null，则在所有工作者/调度器上启用。示例：[0, 1] 仅在调度器和工作者 0 上启用。默认值：null" + +#: ../../source/api_reference/configurations.rst:487 +msgid "internal_api_server_socket_path_prefix" +msgstr "internal_api_server_socket_path_prefix" + +#: ../../source/api_reference/configurations.rst:488 +msgid "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX" +msgstr "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX" + +#: ../../source/api_reference/configurations.rst:489 +#, python-brace-format +msgid "" +"If specified, use Unix domain sockets instead of TCP ports. Socket paths " +"will be \"{prefix}_{port}\". Example: \"/tmp/lmcache_api_socket\" creates" +" \"/tmp/lmcache_api_socket_6999\", \"/tmp/lmcache_api_socket_7000\", etc." +" Default: null" +msgstr "如果指定，则使用 Unix 域套接字而不是 TCP 端口。套接字路径将为 \\\"{prefix}_{port}\\\"。示例：\\\"/tmp/lmcache_api_socket\\\" 创建 \\\"/tmp/lmcache_api_socket_6999\\\"、\\\"/tmp/lmcache_api_socket_7000\\\" 等。默认值：null" + +#: ../../source/api_reference/configurations.rst:492 +msgid "Plugin Configurations" +msgstr "插件配置" + +#: ../../source/api_reference/configurations.rst:494 +msgid "Settings for plugin system." +msgstr "插件系统的设置。" + +#: ../../source/api_reference/configurations.rst:503 +msgid "plugin_locations" +msgstr "plugin_locations" + +#: ../../source/api_reference/configurations.rst:504 +msgid "LMCACHE_PLUGIN_LOCATIONS" +msgstr "LMCACHE_PLUGIN_LOCATIONS" + +#: ../../source/api_reference/configurations.rst:505 +msgid "List of plugin locations. Default: []" +msgstr "插件位置列表。默认值：[]" + +#: ../../source/api_reference/configurations.rst:508 +msgid "Deprecated Configurations" +msgstr "已弃用的配置" + +#: ../../source/api_reference/configurations.rst:510 +msgid "These configurations are deprecated and may be removed in future versions." +msgstr "这些配置已被弃用，可能会在未来的版本中被移除。" + +#: ../../source/api_reference/configurations.rst:519 +msgid "audit_actual_remote_url" +msgstr "audit_actual_remote_url" + +#: ../../source/api_reference/configurations.rst:520 +msgid "LMCACHE_AUDIT_ACTUAL_REMOTE_URL" +msgstr "LMCACHE_AUDIT_ACTUAL_REMOTE_URL" + +#: ../../source/api_reference/configurations.rst:521 +msgid "" +"(Deprecated) URL of actual remote LMCache instance for auditing. Use " +"extra_config['audit_actual_remote_url'] instead" +msgstr "(已弃用) 用于审计的实际远程 LMCache 实例的 URL。请改用 extra_config['audit_actual_remote_url']。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/dynamic_connector.po b/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/dynamic_connector.po new file mode 100644 index 00000000000..a6239386678 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/dynamic_connector.po @@ -0,0 +1,94 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/api_reference/dynamic_connector.rst:2 +msgid "vLLM Dynamic Connector" +msgstr "vLLM 动态连接器" + +#: ../../source/api_reference/dynamic_connector.rst:5 +msgid "Upstream Integration:" +msgstr "上游集成：" + +#: ../../source/api_reference/dynamic_connector.rst:7 +msgid "" +"LMCache integration with official upstream vLLM was introduced in `early " +"February 2025 `_." +msgstr "LMCache 与官方上游 vLLM 的集成是在 `2025 年 2 月初 `_ 中引入的。" + +#: ../../source/api_reference/dynamic_connector.rst:9 +msgid "" +"vLLM imports the connector from the lmcache package and wraps it in " +"`vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py " +"`_:" +msgstr "vLLM 从 lmcache 包导入连接器，并将其封装在 `vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py `_ 中：" + +#: ../../source/api_reference/dynamic_connector.rst:15 +msgid "" +"This means that any updates to LMCache connector need to be " +"synced/updated in the upstream vLLM." +msgstr "这意味着对 LMCache 连接器的任何更新都需要在上游 vLLM 中同步/更新。" + +#: ../../source/api_reference/dynamic_connector.rst:17 +msgid "Example usage of vLLM upstream connector:" +msgstr "vLLM 上游连接器的示例用法：" + +#: ../../source/api_reference/dynamic_connector.rst:19 +#: ../../source/api_reference/dynamic_connector.rst:44 +msgid "**Pythonic Transfer Config:**" +msgstr "**Pythonic Transfer Config:**" + +#: ../../source/api_reference/dynamic_connector.rst:29 +msgid "**Command Line Transfer Configs:**" +msgstr "**命令行传输配置:**" + +#: ../../source/api_reference/dynamic_connector.rst:38 +msgid "Dynamic Connector:" +msgstr "动态连接器：" + +#: ../../source/api_reference/dynamic_connector.rst:40 +msgid "" +"`In June 2025 `_, vLLM " +"supports dynamic loading of KV connector implementations so we can " +"directly reference connectors from the LMCache package without having to " +"update vLLM." +msgstr "在 2025 年 6 月，vLLM 支持动态加载 KV 连接器实现，因此我们可以直接引用 LMCache 包中的连接器，而无需更新 vLLM。" + +#: ../../source/api_reference/dynamic_connector.rst:42 +msgid "Example usage of dynamic connector from LMCache:" +msgstr "来自 LMCache 的动态连接器示例用法：" + +#: ../../source/api_reference/dynamic_connector.rst:55 +msgid "**Command Line Transfer Config:**" +msgstr "**命令行传输配置：**" + +#: ../../source/api_reference/dynamic_connector.rst:63 +msgid "" +"This allows LMCache to modify/develop connectors and quickly plug-and-" +"play." +msgstr "这允许 LMCache 修改/开发连接器并快速插拔。" + +#: ../../source/api_reference/dynamic_connector.rst:65 +msgid "" +"Any custom adapters will be documented here in the future as well as " +"possible deprecations to the upstream connector." +msgstr "未来任何自定义适配器将会在这里记录，以及可能对上游连接器的弃用。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/multimodality.po b/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/multimodality.po new file mode 100644 index 00000000000..b96448d61a4 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/multimodality.po @@ -0,0 +1,171 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/api_reference/multimodality.rst:2 +msgid "KV Caching for Multimodal Models with vLLM" +msgstr "多模态模型的 KV Cache 与 vLLM" + +#: ../../source/api_reference/multimodality.rst:5 +msgid "Overview" +msgstr "概述" + +#: ../../source/api_reference/multimodality.rst:7 +msgid "" +"vLLM is building on its multimodal capability and currently supports the " +"following `List of Multimodal Language Models " +"`_." +msgstr "vLLM 正在构建其多模态能力，目前支持以下 `多模态语言模型列表 `_。" + +#: ../../source/api_reference/multimodality.rst:9 +msgid "" +"LMCache can therefore be used to speed up inference time for all " +"multimodal models supported by vLLM. This document shows the speedup " +"improvements using LMCache for KV caching in vLLM for multimodal models." +msgstr "因此，LMCache 可用于加速 vLLM 支持的所有多模态模型的推理时间。本文档展示了在 vLLM 中使用 LMCache 进行 KV 缓存的加速改进。" + +#: ../../source/api_reference/multimodality.rst:12 +msgid "Examples of TTFT speed up for different multimodal types" +msgstr "不同多模态类型的 TTFT 加速示例" + +#: ../../source/api_reference/multimodality.rst:15 +msgid "Prerequisites" +msgstr "前提条件" + +#: ../../source/api_reference/multimodality.rst:17 +msgid "" +"A Machine with at least one GPU. You can adjust the max model length of " +"your vLLM instance depending on your GPU memory" +msgstr "一台至少配备一块 GPU 的机器。您可以根据您的显存调整 vLLM 实例的最大模型长度。" + +#: ../../source/api_reference/multimodality.rst:18 +msgid "" +"vLLM and LMCache installed (:doc:`Installation Guide " +"<../getting_started/installation>`)" +msgstr "vLLM 和 LMCache 已安装 (:doc:`安装指南 <../getting_started/installation>`)" + +#: ../../source/api_reference/multimodality.rst:19 +msgid "vLLM audio dependencies installed: ``pip install vllm[audio]``" +msgstr "vLLM 音频依赖项已安装：``pip install vllm[audio]``" + +#: ../../source/api_reference/multimodality.rst:22 +msgid "Examples" +msgstr "示例" + +#: ../../source/api_reference/multimodality.rst:26 +msgid "" +"The examples below use a python script for inferencing multimodal models " +"hosted by vLLM. The script is the " +"`openai_chat_completion_client_for_multimodal python script in vLLM " +"`_." +" You will need to download it locally for running the examples below. The" +" script is printed in the `reference section <#reference-inferencing-" +"multimodal-models-in-vllm-example-python-script>`_ that follows for you " +"perusal. Go to the `Example output <#example-output>`_ section to see the" +" output in the vLLM logs that demonstrate the speedup improvements." +msgstr "下面的示例使用 Python 脚本对托管在 vLLM 上的多模态模型进行推理。该脚本是 `vLLM 中的 openai_chat_completion_client_for_multimodal python 脚本 `_。您需要将其下载到本地以运行下面的示例。该脚本在后面的 `参考部分 <#reference-inferencing-multimodal-models-in-vllm-example-python-script>`_ 中打印供您查阅。请转到 `示例输出 <#example-output>`_ 部分查看 vLLM 日志中的输出，以展示加速改进。" + +#: ../../source/api_reference/multimodality.rst:34 +msgid "Audio Inference with Ultravox:" +msgstr "使用 Ultravox 进行音频推理：" + +#: ../../source/api_reference/multimodality.rst:36 +msgid "" +"Start vLLM server with ``fixie-ai/ultravox-v0_5-llama-3_2-1b`` model and " +"LMCache KV caching:" +msgstr "使用 ``fixie-ai/ultravox-v0_5-llama-3_2-1b`` 模型和 LMCache KV 缓存启动 vLLM 服务器：" + +#: ../../source/api_reference/multimodality.rst:44 +#: ../../source/api_reference/multimodality.rst:61 +#: ../../source/api_reference/multimodality.rst:79 +#: ../../source/api_reference/multimodality.rst:96 +msgid "" +"Run the python script twice to demonstrate TTFT speedup on the second " +"turn because of the caching:" +msgstr "运行 Python 脚本两次，以演示由于缓存导致的第二次推理的 TTFT 加速：" + +#: ../../source/api_reference/multimodality.rst:52 +msgid "Single Image Inference with Llava:" +msgstr "使用 Llava 进行单图像推理：" + +#: ../../source/api_reference/multimodality.rst:54 +msgid "" +"Start vLLM server with ``llava-hf/llava-1.5-7b-hf`` model and LMCache KV " +"caching:" +msgstr "使用 ``llava-hf/llava-1.5-7b-hf`` 模型和 LMCache KV 缓存启动 vLLM 服务器：" + +#: ../../source/api_reference/multimodality.rst:69 +msgid "Multi-image Inference with Phi-3.5-vision-instruct:" +msgstr "使用 Phi-3.5-vision-instruct 进行多图像推理：" + +#: ../../source/api_reference/multimodality.rst:71 +msgid "" +"Start vLLM server with ``microsoft/Phi-3.5-vision-instruct`` model and " +"LMCache KV caching:" +msgstr "使用 ``microsoft/Phi-3.5-vision-instruct`` 模型和 LMCache KV 缓存启动 vLLM 服务器：" + +#: ../../source/api_reference/multimodality.rst:87 +msgid "Video Inference with Llava-OneVision:" +msgstr "使用 Llava-OneVision 进行视频推理：" + +#: ../../source/api_reference/multimodality.rst:89 +msgid "" +"Start vLLM server with ``llava-hf/llava-onevision-qwen2-7b-ov-hf`` model " +"and LMCache KV caching:" +msgstr "使用 ``llava-hf/llava-onevision-qwen2-7b-ov-hf`` 模型和 LMCache KV 缓存启动 vLLM 服务器：" + +#: ../../source/api_reference/multimodality.rst:105 +msgid "Example output" +msgstr "示例输出" + +#: ../../source/api_reference/multimodality.rst:107 +msgid "" +"When running the examples above you will notice output in the vLLM logs " +"similar to below." +msgstr "在运行上述示例时，您会注意到 vLLM 日志中类似于以下的输出。" + +#: ../../source/api_reference/multimodality.rst:109 +msgid "This first output demonstrates the tokens being cached and loaded." +msgstr "该第一个输出演示了令牌被缓存和加载的过程。" + +#: ../../source/api_reference/multimodality.rst:115 +msgid "This then shows the speedup between the first and second runs." +msgstr "这显示了第一次和第二次运行之间的加速。" + +#: ../../source/api_reference/multimodality.rst:117 +msgid "First request:" +msgstr "第一次请求：" + +#: ../../source/api_reference/multimodality.rst:128 +msgid "Second request:" +msgstr "第二个请求：" + +#: ../../source/api_reference/multimodality.rst:139 +msgid "Reference: Inferencing multimodal models in vLLM example Python script" +msgstr "参考：在 vLLM 示例 Python 脚本中推理多模态模型" + +#: ../../source/api_reference/multimodality.rst:141 +msgid "" +"Source: https://github.com/vllm-" +"project/vllm/blob/main/examples/online_serving/openai_chat_completion_client_for_multimodal.py" +msgstr "源: https://github.com/vllm-project/vllm/blob/main/examples/online_serving/openai_chat_completion_client_for_multimodal.py" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/storage_backends.po b/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/storage_backends.po new file mode 100644 index 00000000000..5c780e170f5 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/storage_backends.po @@ -0,0 +1,29 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/api_reference/storage_backends.rst:2 +msgid "Adding new storage backends" +msgstr "添加新的存储后端" + +#: ../../source/api_reference/storage_backends.rst:4 +msgid "Coming soon..." +msgstr "敬请期待..." + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/cli/bench.po b/docs/source/locale/zh_CN/LC_MESSAGES/cli/bench.po new file mode 100644 index 00000000000..00dece9cc56 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/cli/bench.po @@ -0,0 +1,882 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/cli/bench.rst:4 +msgid "lmcache bench engine" +msgstr "lmcache 基准引擎" + +#: ../../source/cli/bench.rst:6 +msgid "" +"The ``lmcache bench engine`` command runs sustained performance " +"benchmarks against an inference engine (e.g., vLLM). It supports multiple" +" workload types that exercise different caching patterns and reports " +"TTFT, decoding speed, and throughput metrics." +msgstr "``lmcache bench engine`` 命令对推理引擎（例如 vLLM）进行持续性能基准测试。它支持多种工作负载类型，以测试不同的缓存模式，并报告 TTFT、解码速度和吞吐量指标。" + +#: ../../source/cli/bench.rst:15 +msgid "There are three ways to configure the benchmark:" +msgstr "配置基准测试有三种方法：" + +#: ../../source/cli/bench.rst:17 +msgid "**CLI arguments** -- pass all options on the command line." +msgstr "**CLI 参数** -- 在命令行中传递所有选项。" + +#: ../../source/cli/bench.rst:18 +msgid "" +"**Interactive mode** -- run ``lmcache bench engine`` without required " +"args and follow the step-by-step prompts." +msgstr "**交互模式** -- 运行 ``lmcache bench engine`` 而不需要参数，并按照逐步提示进行操作。" + +#: ../../source/cli/bench.rst:20 +msgid "" +"**Config file** -- save a configuration to JSON and replay it with " +"``--config``." +msgstr "**配置文件** -- 将配置保存为 JSON 并使用 ``--config`` 进行重放。" + +#: ../../source/cli/bench.rst:25 +msgid "Quick Start" +msgstr "快速开始" + +#: ../../source/cli/bench.rst:27 +msgid "**Minimal (with all required arguments):**" +msgstr "**最小化（包含所有必需参数）：**" + +#: ../../source/cli/bench.rst:36 +msgid "**Interactive mode (guided setup):**" +msgstr "**交互模式（引导设置）：**" + +#: ../../source/cli/bench.rst:42 +msgid "" +"The interactive mode walks you through each required setting, then asks " +"whether you want to configure general and workload-specific options or " +"use defaults." +msgstr "交互模式会引导您完成每个必需的设置，然后询问您是否希望配置通用选项和特定于工作负载的选项，还是使用默认值。" + +#: ../../source/cli/bench.rst:46 +msgid "**From a saved config file:**" +msgstr "**从保存的配置文件中：**" + +#: ../../source/cli/bench.rst:53 +msgid "" +"Config files contain benchmark parameters (workload, KV cache settings, " +"etc.) but not the engine URL, so you can reuse the same config against " +"different engines." +msgstr "配置文件包含基准测试参数（工作负载、KV Cache 设置等），但不包括引擎 URL，因此您可以在不同的引擎上重用相同的配置。" + +#: ../../source/cli/bench.rst:57 +msgid "**Export a config without running the benchmark:**" +msgstr "**导出配置而不运行基准测试：**" + +#: ../../source/cli/bench.rst:67 +msgid "" +"This resolves all auto-detected values (model name, tokens per GB) and " +"saves them to a portable JSON file that works without an LMCache server." +msgstr "这将解析所有自动检测到的值（模型名称、每 GB 的令牌数），并将它们保存到一个便携式 JSON 文件中，该文件可以在没有 LMCache 服务器的情况下使用。" + +#: ../../source/cli/bench.rst:70 +msgid "**Non-interactive mode (for scripts and CI):**" +msgstr "**非交互模式（用于脚本和 CI）：**" + +#: ../../source/cli/bench.rst:80 +msgid "" +"Errors immediately if any required argument is missing, instead of " +"entering interactive mode. Useful in automated pipelines." +msgstr "如果缺少任何必需的参数，则立即报错，而不是进入交互模式。这在自动化管道中非常有用。" + +#: ../../source/cli/bench.rst:83 +msgid "" +"If you don't have an LMCache server, you can pass ``--tokens-per-gb-" +"kvcache`` directly instead of ``--lmcache-url`` (see :ref:`bench-tokens-" +"per-gb` for how to find this value)." +msgstr "如果您没有 LMCache 服务器，可以直接传递 ``--tokens-per-gb-kvcache``，而不是 ``--lmcache-url``（有关如何找到此值，请参见 :ref:`bench-tokens-per-gb`）。" + +#: ../../source/cli/bench.rst:89 +msgid "General Options" +msgstr "常规选项" + +#: ../../source/cli/bench.rst:95 ../../source/cli/bench.rst:188 +#: ../../source/cli/bench.rst:229 ../../source/cli/bench.rst:280 +#: ../../source/cli/bench.rst:367 ../../source/cli/bench.rst:431 +msgid "Flag" +msgstr "标志" + +#: ../../source/cli/bench.rst:96 +msgid "Required" +msgstr "必需" + +#: ../../source/cli/bench.rst:97 ../../source/cli/bench.rst:190 +#: ../../source/cli/bench.rst:231 ../../source/cli/bench.rst:282 +#: ../../source/cli/bench.rst:369 ../../source/cli/bench.rst:433 +msgid "Description" +msgstr "描述" + +#: ../../source/cli/bench.rst:98 +msgid "``--config FILE``" +msgstr "``--config FILE``" + +#: ../../source/cli/bench.rst:99 ../../source/cli/bench.rst:104 +#: ../../source/cli/bench.rst:110 ../../source/cli/bench.rst:127 +#: ../../source/cli/bench.rst:131 ../../source/cli/bench.rst:134 +#: ../../source/cli/bench.rst:137 ../../source/cli/bench.rst:140 +#: ../../source/cli/bench.rst:143 ../../source/cli/bench.rst:146 +#: ../../source/cli/bench.rst:149 +msgid "No" +msgstr "不" + +#: ../../source/cli/bench.rst:100 +msgid "" +"Load configuration from a JSON file. Skips interactive mode. CLI flags " +"override values in the file. The engine URL is not stored in config files" +" and must be provided separately." +msgstr "从 JSON 文件加载配置。跳过交互模式。CLI 标志会覆盖文件中的值。引擎 URL 不会存储在配置文件中，必须单独提供。" + +#: ../../source/cli/bench.rst:103 +msgid "``--export-config FILE``" +msgstr "``--export-config FILE``" + +#: ../../source/cli/bench.rst:105 +msgid "" +"Export resolved configuration to a JSON file and exit. Does not run the " +"benchmark. Auto-detected values (model, tokens per GB) are resolved and " +"saved so the config is portable. Environment- specific values (engine " +"URL, LMCache URL) are excluded." +msgstr "将解析后的配置导出到 JSON 文件并退出。不会运行基准测试。自动检测的值（模型，每 GB 的令牌数）被解析并保存，以便配置可移植。特定于环境的值（引擎 URL，LMCache URL）被排除在外。" + +#: ../../source/cli/bench.rst:109 +msgid "``--no-interactive``" +msgstr "``--no-interactive``" + +#: ../../source/cli/bench.rst:111 +msgid "" +"Disable interactive mode. Errors if required arguments are missing " +"instead of prompting. Useful for scripts and CI." +msgstr "禁用交互模式。如果缺少必需的参数则报错，而不是提示。对脚本和 CI 很有用。" + +#: ../../source/cli/bench.rst:113 +msgid "``--engine-url URL``" +msgstr "``--engine-url URL``" + +#: ../../source/cli/bench.rst:114 ../../source/cli/bench.rst:118 +msgid "Yes" +msgstr "是" + +#: ../../source/cli/bench.rst:115 +msgid "" +"Inference engine URL (e.g., ``http://localhost:8000``). Set " +"``OPENAI_API_KEY`` env var if authentication is needed." +msgstr "推理引擎 URL（例如，``http://localhost:8000``）。如果需要身份验证，请设置 ``OPENAI_API_KEY`` 环境变量。" + +#: ../../source/cli/bench.rst:117 +msgid "``--workload TYPE``" +msgstr "``--workload TYPE``" + +#: ../../source/cli/bench.rst:119 +msgid "" +"Workload type: ``long-doc-qa``, ``multi-round-chat``, ``long-doc-" +"permutator``, ``prefix-suffix-tuner``, or ``random-prefill``." +msgstr "工作负载类型：``long-doc-qa``、``multi-round-chat``、``long-doc-permutator``、``prefix-suffix-tuner`` 或 ``random-prefill``。" + +#: ../../source/cli/bench.rst:122 +msgid "``--tokens-per-gb-kvcache N``" +msgstr "``--tokens-per-gb-kvcache N``" + +#: ../../source/cli/bench.rst:123 +msgid "\\*" +msgstr "\\*" + +#: ../../source/cli/bench.rst:124 +msgid "" +"Tokens per GB of KV cache. Required unless ``--lmcache-url`` is set. See " +":ref:`bench-tokens-per-gb` for how to find this value." +msgstr "每 GB KV Cache 的令牌数。除非设置了 ``--lmcache-url``，否则这是必需的。有关如何找到此值的信息，请参见 :ref:`bench-tokens-per-gb`。" + +#: ../../source/cli/bench.rst:126 +msgid "``--lmcache-url URL``" +msgstr "``--lmcache-url URL``" + +#: ../../source/cli/bench.rst:128 +msgid "" +"LMCache HTTP server URL. When provided, ``--tokens-per-gb-kvcache`` is " +"auto-detected from the server." +msgstr "LMCache HTTP 服务器 URL。当提供时，``--tokens-per-gb-kvcache`` 会从服务器自动检测。" + +#: ../../source/cli/bench.rst:130 +msgid "``--model NAME``" +msgstr "``--model NAME``" + +#: ../../source/cli/bench.rst:132 +msgid "Model name. Auto-detected from the engine if omitted." +msgstr "模型名称。如果省略，将从引擎自动检测。" + +#: ../../source/cli/bench.rst:133 +msgid "``--kv-cache-volume GB``" +msgstr "``--kv-cache-volume GB``" + +#: ../../source/cli/bench.rst:135 +msgid "Target active KV cache volume in GB (default: 100)." +msgstr "目标活动 KV Cache 容量（单位：GB，默认值：100）。" + +#: ../../source/cli/bench.rst:136 +msgid "``--seed N``" +msgstr "``--seed N``" + +#: ../../source/cli/bench.rst:138 +msgid "Random seed (default: 42)." +msgstr "随机种子（默认值：42）。" + +#: ../../source/cli/bench.rst:139 +msgid "``--output-dir DIR``" +msgstr "``--output-dir DIR``" + +#: ../../source/cli/bench.rst:141 +msgid "Directory for CSV and JSON output files (default: current directory)." +msgstr "CSV 和 JSON 输出文件的目录（默认：当前目录）。" + +#: ../../source/cli/bench.rst:142 +msgid "``--no-csv``" +msgstr "``--no-csv``" + +#: ../../source/cli/bench.rst:144 +msgid "Skip CSV export." +msgstr "跳过 CSV 导出。" + +#: ../../source/cli/bench.rst:145 +msgid "``--json``" +msgstr "``--json``" + +#: ../../source/cli/bench.rst:147 +msgid "Export a JSON summary file." +msgstr "导出 JSON 摘要文件。" + +#: ../../source/cli/bench.rst:148 +msgid "``-q`` / ``--quiet``" +msgstr "``-q`` / ``--quiet``" + +#: ../../source/cli/bench.rst:150 +msgid "Suppress the real-time progress display." +msgstr "抑制实时进度显示。" + +#: ../../source/cli/bench.rst:156 +msgid "Finding ``--tokens-per-gb-kvcache``" +msgstr "查找 ``--tokens-per-gb-kvcache``" + +#: ../../source/cli/bench.rst:158 +msgid "" +"If you have an LMCache server running, the easiest approach is to pass " +"``--lmcache-url`` and let the tool auto-detect the value." +msgstr "如果您有一个正在运行的 LMCache 服务器，最简单的方法是传递 ``--lmcache-url`` 并让工具自动检测该值。" + +#: ../../source/cli/bench.rst:161 +msgid "" +"If you are using **vLLM without LMCache**, look for these lines in vLLM's" +" startup log:" +msgstr "如果您正在使用 **vLLM 而没有 LMCache**，请在 vLLM 的启动日志中查找这些行：" + +#: ../../source/cli/bench.rst:169 +msgid "Then compute::" +msgstr "然后计算::" + +#: ../../source/cli/bench.rst:175 +msgid "Workloads" +msgstr "工作负载" + +#: ../../source/cli/bench.rst:178 +msgid "long-doc-qa" +msgstr "长文档问答" + +#: ../../source/cli/bench.rst:180 +msgid "" +"Simulates repeated Q&A over long documents. Warmup sends each document " +"once to populate the KV cache, then benchmark queries are dispatched with" +" semaphore-controlled concurrency." +msgstr "模拟对长文档的重复问答。热身阶段将每个文档发送一次以填充 KV Cache，然后以信号量控制的并发方式发送基准查询。" + +#: ../../source/cli/bench.rst:189 ../../source/cli/bench.rst:230 +#: ../../source/cli/bench.rst:281 ../../source/cli/bench.rst:368 +#: ../../source/cli/bench.rst:432 +msgid "Default" +msgstr "默认" + +#: ../../source/cli/bench.rst:191 +msgid "``--ldqa-document-length``" +msgstr "``--ldqa-document-length``" + +#: ../../source/cli/bench.rst:192 ../../source/cli/bench.rst:236 +#: ../../source/cli/bench.rst:435 +msgid "10000" +msgstr "10000" + +#: ../../source/cli/bench.rst:193 +msgid "Token length of each synthetic document." +msgstr "每个合成文档的令牌长度。" + +#: ../../source/cli/bench.rst:194 +msgid "``--ldqa-query-per-document``" +msgstr "``--ldqa-query-per-document``" + +#: ../../source/cli/bench.rst:195 ../../source/cli/bench.rst:333 +msgid "2" +msgstr "2" + +#: ../../source/cli/bench.rst:196 +msgid "Number of questions asked per document." +msgstr "每个文档提问的数量。" + +#: ../../source/cli/bench.rst:197 +msgid "``--ldqa-shuffle-policy``" +msgstr "``--ldqa-shuffle-policy``" + +#: ../../source/cli/bench.rst:198 +msgid "random" +msgstr "随机" + +#: ../../source/cli/bench.rst:199 +msgid "Request ordering: ``random`` (shuffled) or ``tile`` (round-by-round)." +msgstr "请求排序：``random``（随机）或``tile``（逐轮）。" + +#: ../../source/cli/bench.rst:200 +msgid "``--ldqa-num-inflight-requests``" +msgstr "``--ldqa-num-inflight-requests``" + +#: ../../source/cli/bench.rst:201 ../../source/cli/bench.rst:337 +msgid "3" +msgstr "3" + +#: ../../source/cli/bench.rst:202 ../../source/cli/bench.rst:298 +msgid "Maximum concurrent in-flight requests." +msgstr "最大并发进行中的请求数量。" + +#: ../../source/cli/bench.rst:204 ../../source/cli/bench.rst:251 +#: ../../source/cli/bench.rst:300 ../../source/cli/bench.rst:391 +#: ../../source/cli/bench.rst:441 +msgid "**Example:**" +msgstr "**示例：**" + +#: ../../source/cli/bench.rst:219 +msgid "multi-round-chat" +msgstr "多轮聊天" + +#: ../../source/cli/bench.rst:221 +msgid "" +"Simulates multi-round chat with stateful sessions. Creates concurrent " +"user sessions, dispatches requests at a fixed QPS rate, and records " +"responses in session history so each subsequent query includes prior " +"context." +msgstr "模拟具有状态的多轮聊天。创建并发用户会话，以固定的 QPS 速率调度请求，并在会话历史中记录响应，以便每个后续查询都包含先前的上下文。" + +#: ../../source/cli/bench.rst:232 +msgid "``--mrc-shared-prompt-length``" +msgstr "``--mrc-shared-prompt-length``" + +#: ../../source/cli/bench.rst:233 +msgid "2000" +msgstr "2000" + +#: ../../source/cli/bench.rst:234 +msgid "System prompt token length per session." +msgstr "每个会话的系统提示令牌长度。" + +#: ../../source/cli/bench.rst:235 +msgid "``--mrc-chat-history-length``" +msgstr "``--mrc-chat-history-length``" + +#: ../../source/cli/bench.rst:237 +msgid "Pre-filled chat history token length." +msgstr "预填充聊天历史令牌长度。" + +#: ../../source/cli/bench.rst:238 +msgid "``--mrc-user-input-length``" +msgstr "``--mrc-user-input-length``" + +#: ../../source/cli/bench.rst:239 ../../source/cli/bench.rst:438 +msgid "50" +msgstr "50" + +#: ../../source/cli/bench.rst:240 +msgid "Tokens per user query." +msgstr "每个用户查询的令牌数。" + +#: ../../source/cli/bench.rst:241 +msgid "``--mrc-output-length``" +msgstr "``--mrc-output-length``" + +#: ../../source/cli/bench.rst:242 +msgid "200" +msgstr "200" + +#: ../../source/cli/bench.rst:243 +msgid "Max tokens to generate per response." +msgstr "每个响应生成的最大令牌数。" + +#: ../../source/cli/bench.rst:244 +msgid "``--mrc-qps``" +msgstr "``--mrc-qps``" + +#: ../../source/cli/bench.rst:245 +msgid "1.0" +msgstr "1.0" + +#: ../../source/cli/bench.rst:246 +msgid "Target queries per second." +msgstr "每秒目标查询数。" + +#: ../../source/cli/bench.rst:247 +msgid "``--mrc-duration``" +msgstr "``--mrc-duration``" + +#: ../../source/cli/bench.rst:248 +msgid "60.0" +msgstr "60.0" + +#: ../../source/cli/bench.rst:249 +msgid "Benchmark duration in seconds." +msgstr "基准测试持续时间（以秒为单位）。" + +#: ../../source/cli/bench.rst:264 +msgid "long-doc-permutator" +msgstr "long-doc-permutator" + +#: ../../source/cli/bench.rst:266 +msgid "" +"Stress-tests blended KV cache reuse by sending permutations of a set of " +"context documents. Each request concatenates all context documents in a " +"different order:" +msgstr "通过发送一组上下文文档的排列组合，压力测试混合 KV Cache 的重用。每个请求以不同的顺序连接所有上下文文档：" + +#: ../../source/cli/bench.rst:273 +msgid "" +"A single dummy warmup request is sent before the benchmark phase. " +"Requests are dispatched with semaphore-controlled concurrency." +msgstr "在基准测试阶段之前，会发送一个单一的虚拟预热请求。请求以信号量控制的并发方式分发。" + +#: ../../source/cli/bench.rst:283 +msgid "``--ldp-num-contexts``" +msgstr "``--ldp-num-contexts``" + +#: ../../source/cli/bench.rst:284 +msgid "5" +msgstr "5" + +#: ../../source/cli/bench.rst:285 +msgid "Number of unique context documents." +msgstr "唯一上下文文档的数量。" + +#: ../../source/cli/bench.rst:286 +msgid "``--ldp-context-length``" +msgstr "``--ldp-context-length``" + +#: ../../source/cli/bench.rst:287 +msgid "5000" +msgstr "5000" + +#: ../../source/cli/bench.rst:288 +msgid "Token length of each context document." +msgstr "每个上下文文档的令牌长度。" + +#: ../../source/cli/bench.rst:289 +msgid "``--ldp-system-prompt-length``" +msgstr "``--ldp-system-prompt-length``" + +#: ../../source/cli/bench.rst:290 +msgid "1000" +msgstr "1000" + +#: ../../source/cli/bench.rst:291 +msgid "Token length of the shared system prompt. Use ``0`` for no system prompt." +msgstr "共享系统提示的令牌长度。使用 ``0`` 表示没有系统提示。" + +#: ../../source/cli/bench.rst:292 +msgid "``--ldp-num-permutations``" +msgstr "``--ldp-num-permutations``" + +#: ../../source/cli/bench.rst:293 +msgid "10" +msgstr "10" + +#: ../../source/cli/bench.rst:294 +msgid "" +"Number of distinct permutations to send. Capped at N! where N = ``--ldp-" +"num-contexts``." +msgstr "发送的不同排列组合的数量。上限为 N!，其中 N = ``--ldp-num-contexts``。" + +#: ../../source/cli/bench.rst:296 +msgid "``--ldp-num-inflight-requests``" +msgstr "``--ldp-num-inflight-requests``" + +#: ../../source/cli/bench.rst:297 ../../source/cli/bench.rst:329 +msgid "1" +msgstr "1" + +#: ../../source/cli/bench.rst:315 +msgid "prefix-suffix-tuner" +msgstr "prefix-suffix-tuner" + +#: ../../source/cli/bench.rst:317 +msgid "" +"A two-pass sequential workload designed to be run **unchanged** across " +"three LMCache configurations to demonstrate the value of each cache tier " +"(L0 HBM, L1 DRAM, L2 disk):" +msgstr "一个两遍顺序工作负载，旨在 **不变** 地在三个 LMCache 配置中运行，以展示每个缓存层级（L0 HBM、L1 DRAM、L2 磁盘）的价值：" + +#: ../../source/cli/bench.rst:325 +msgid "Baseline" +msgstr "基线" + +#: ../../source/cli/bench.rst:326 +msgid "LMCache config" +msgstr "LMCache 配置" + +#: ../../source/cli/bench.rst:327 +msgid "Targeted overflow" +msgstr "目标溢出" + +#: ../../source/cli/bench.rst:328 +msgid "Expected pass-2 hits" +msgstr "预期的 pass-2 命中" + +#: ../../source/cli/bench.rst:330 +msgid "vanilla vLLM (L0 only)" +msgstr "原生 vLLM (仅 L0)" + +#: ../../source/cli/bench.rst:331 +msgid "L0 (HBM)" +msgstr "L0 (HBM)" + +#: ../../source/cli/bench.rst:332 +msgid "none -- every request a cold prefill" +msgstr "无 -- 每个请求都是冷预填充" + +#: ../../source/cli/bench.rst:334 +msgid "vLLM + LMCache L1 + L2" +msgstr "vLLM + LMCache L1 + L2" + +#: ../../source/cli/bench.rst:335 ../../source/cli/bench.rst:339 +msgid "L1 (DRAM)" +msgstr "L1 (DRAM)" + +#: ../../source/cli/bench.rst:336 +msgid "L2 prefix hits (suffix recomputed)" +msgstr "L2 前缀命中（后缀重计算）" + +#: ../../source/cli/bench.rst:338 +msgid "vLLM + LMCache L1 + L2 + CacheBlend" +msgstr "vLLM + LMCache L1 + L2 + CacheBlend" + +#: ../../source/cli/bench.rst:340 +msgid "L2 prefix hits + CacheBlend suffix hits" +msgstr "L2 前缀命中 + CacheBlend 后缀命中" + +#: ../../source/cli/bench.rst:342 +msgid "" +"Set ``--kv-cache-volume`` to the size in GB of the tier you want to " +"overflow (L0 size for Baseline 1, L1 size for Baselines 2 and 3). The " +"workload itself is identical across baselines." +msgstr "将 ``--kv-cache-volume`` 设置为您希望溢出的层的大小（基线 1 的 L0 大小，基线 2 和 3 的 L1 大小）。工作负载在各个基线之间是相同的。" + +#: ../../source/cli/bench.rst:346 +msgid "Each request has the layout::" +msgstr "每个请求的布局::" + +#: ../../source/cli/bench.rst:350 +msgid "" +"``num_prefixes`` distinct prefixes, each starting with ``PREFIX_<8-hex>``" +" so the prefix's tokenized hash differs across the pool." +msgstr "``num_prefixes`` 个不同的前缀，每个前缀以 ``PREFIX_<8-hex>`` 开头，因此前缀的标记化哈希在池中是不同的。" + +#: ../../source/cli/bench.rst:352 +msgid "" +"A fresh random 32-token breaker per request, defeating ordinary prefix " +"caching past the prefix boundary." +msgstr "每个请求一个新的随机 32-token 断路器，打破普通前缀缓存的前缀边界。" + +#: ../../source/cli/bench.rst:354 +msgid "" +"A single shared suffix used by every request -- the only entry CacheBlend" +" can reuse." +msgstr "每个请求使用的单个共享后缀——这是 CacheBlend 唯一可以重用的条目。" + +#: ../../source/cli/bench.rst:357 +msgid "" +"Pass 1 (warmup) sends each prefix once to populate the cache; its stats " +"are discarded. Pass 2 sends them again in identical order. Because LRU " +"evicts the next-needed prefix on each pass-2 access, even a 1.05x " +"overflow of the targeted tier is enough to make every pass-2 request miss" +" that tier and fall through to the next one." +msgstr "第 1 次传递（预热）将每个前缀发送一次以填充缓存；其统计数据会被丢弃。第 2 次传递以相同的顺序再次发送它们。由于 LRU 在每次第 2 次访问时逐出下一个所需的前缀，即使目标层溢出 1.05 倍也足以使每个第 2 次请求未命中该层并降级到下一层。" + +#: ../../source/cli/bench.rst:370 +msgid "``--psf-context-length``" +msgstr "``--psf-context-length``" + +#: ../../source/cli/bench.rst:371 +msgid "8000" +msgstr "8000" + +#: ../../source/cli/bench.rst:372 +msgid "Total tokens per request (prefix + breaker + suffix)." +msgstr "每个请求的总令牌数（前缀 + 分隔符 + 后缀）。" + +#: ../../source/cli/bench.rst:373 +msgid "``--psf-prefix-ratio``" +msgstr "``--psf-prefix-ratio``" + +#: ../../source/cli/bench.rst:374 +msgid "0.8" +msgstr "0.8" + +#: ../../source/cli/bench.rst:375 +msgid "" +"Fraction of context-length used by the prefix. Must be in (0.0, 1.0). The" +" remainder (minus a 32-token breaker) is the shared suffix." +msgstr "前缀使用的上下文长度的比例。必须在 (0.0, 1.0) 之间。其余部分（减去 32 个标记的分隔符）是共享后缀。" + +#: ../../source/cli/bench.rst:377 +msgid "``--psf-thrash``" +msgstr "``--psf-thrash``" + +#: ../../source/cli/bench.rst:378 +msgid "20.0" +msgstr "20.0" + +#: ../../source/cli/bench.rst:379 +#, python-format +msgid "" +"**Size in GB of the KV-cache tier to overflow.** Use the L0 (HBM) size " +"for vanilla vLLM, or the L1 (LMCache DRAM) size for tiered baselines. The" +" workload sizes its prefix pool to slightly more than this (5% overflow " +"internally), enough to drive every pass-2 request to a miss of that tier " +"under sequential dispatch + LRU." +msgstr "**KV-cache 层的大小（以 GB 为单位），用于溢出。** 对于普通的 vLLM，请使用 L0（HBM）大小，或者对于分层基线，请使用 L1（LMCache DRAM）大小。工作负载将其前缀池的大小设置为略大于此值（内部溢出 5%），足以使每个顺序调度 + LRU 的 pass-2 请求在该层中未命中。" + +#: ../../source/cli/bench.rst:385 +msgid "" +"The number of pass-2 (measured) requests equals the prefix pool size, " +"computed as ``floor(psf_thrash * 1.05 * tokens_per_gb / prefix_tokens)``." +" ``--kv-cache-volume`` is unused by this workload — sizing is driven " +"solely by ``--psf-thrash``." +msgstr "通过 ``floor(psf_thrash * 1.05 * tokens_per_gb / prefix_tokens)`` 计算的 pass-2（测量）请求数量等于前缀池大小。此工作负载未使用 ``--kv-cache-volume`` — 尺寸完全由 ``--psf-thrash`` 驱动。" + +#: ../../source/cli/bench.rst:404 +#, python-format +msgid "" +"For the analytical-model claim \"thrash ≈ L1 size → ~0% LMCache hit " +"rate\" to hold empirically, the LMCache server must be started with " +"``--eviction-ratio 0.99`` (default ``0.20`` only clears 20% per cycle, " +"leaving ~60% of pass-1 content in cache through pass 2):" +msgstr "为了使分析模型声明“thrash ≈ L1 大小 → ~0% LMCache 命中率”在经验上成立，LMCache 服务器必须以 ``--eviction-ratio 0.99`` 启动（默认的 ``0.20`` 仅在每个周期清除 20%，在第 2 次传递中大约保留 60% 的第 1 次传递内容）：" + +#: ../../source/cli/bench.rst:415 +msgid "" +"The workload itself sleeps 5 seconds between pass 1 (warmup) and pass 2 " +"(measured), so LMCache's 1Hz batched-eviction polling thread has time to " +"actually run. Without that sleep, fast benchmarks complete before any " +"eviction fires." +msgstr "工作负载在第 1 次（热身）和第 2 次（测量）之间会休眠 5 秒，因此 LMCache 的 1Hz 批量逐出轮询线程有时间实际运行。没有这个休眠，快速基准测试在任何逐出触发之前就完成了。" + +#: ../../source/cli/bench.rst:422 +msgid "random-prefill" +msgstr "随机预填充" + +#: ../../source/cli/bench.rst:424 +msgid "" +"Fires all requests simultaneously with ``max_tokens=1`` to measure pure " +"prefill performance. No warmup phase." +msgstr "同时发送所有请求，``max_tokens=1``，以测量纯粹的 Prefill 性能。没有预热阶段。" + +#: ../../source/cli/bench.rst:434 +msgid "``--rp-request-length``" +msgstr "``--rp-request-length``" + +#: ../../source/cli/bench.rst:436 +msgid "Token length per prefill request." +msgstr "每个 Prefill 请求的令牌长度。" + +#: ../../source/cli/bench.rst:437 +msgid "``--rp-num-requests``" +msgstr "``--rp-num-requests``" + +#: ../../source/cli/bench.rst:439 +msgid "Number of requests to fire." +msgstr "要发送的请求数量。" + +#: ../../source/cli/bench.rst:454 +msgid "Interactive Mode" +msgstr "交互模式" + +#: ../../source/cli/bench.rst:456 +msgid "Interactive mode demo" +msgstr "交互模式演示" + +#: ../../source/cli/bench.rst:460 +msgid "" +"When ``--engine-url`` or ``--workload`` is not provided (and ``--no-" +"interactive`` is not set), the tool enters interactive mode. It guides " +"you through four phases:" +msgstr "当未提供 ``--engine-url`` 或 ``--workload``（并且未设置 ``--no-interactive``）时，工具将进入交互模式。它将引导您完成四个阶段：" + +#: ../../source/cli/bench.rst:464 +msgid "" +"**Required settings** -- engine URL, workload type, LMCache server (or " +"tokens per GB)." +msgstr "**必需设置** -- 引擎 URL、工作负载类型、LMCache 服务器（或每 GB 的令牌）。" + +#: ../../source/cli/bench.rst:466 +msgid "**General settings** (optional gate) -- model name, KV cache volume." +msgstr "**通用设置**（可选门）-- 模型名称，KV Cache 容量。" + +#: ../../source/cli/bench.rst:467 +msgid "**Workload settings** (optional gate) -- workload-specific parameters." +msgstr "**工作负载设置**（可选网关）-- 工作负载特定参数。" + +#: ../../source/cli/bench.rst:468 +msgid "" +"**Summary and action** -- review configuration, then start the benchmark " +"or export to a JSON file." +msgstr "**摘要和操作** -- 审查配置，然后启动基准测试或导出到 JSON 文件。" + +#: ../../source/cli/bench.rst:471 +msgid "" +"Each prompt focuses on a single setting. Selection prompts use arrow " +"keys; text and number prompts accept typed input with defaults shown in " +"brackets." +msgstr "每个提示专注于单个设置。选择提示使用箭头键；文本和数字提示接受键入的输入，默认值以括号显示。" + +#: ../../source/cli/bench.rst:514 +msgid "" +"When you choose \"Export configuration\", all auto-detected values (model" +" name, tokens per GB) are resolved and saved to a portable JSON file." +msgstr "当您选择“导出配置”时，所有自动检测到的值（模型名称、每 GB 的令牌数）将被解析并保存到一个可移植的 JSON 文件中。" + +#: ../../source/cli/bench.rst:519 +msgid "Config File" +msgstr "配置文件" + +#: ../../source/cli/bench.rst:521 +msgid "" +"Config files store benchmark parameters but **not** environment-specific " +"values like engine URL or LMCache URL. This lets you reuse the same " +"config across different environments." +msgstr "配置文件存储基准测试参数，但**不**存储特定于环境的值，例如引擎 URL 或 LMCache URL。这使您可以在不同环境中重用相同的配置。" + +#: ../../source/cli/bench.rst:525 +msgid "You can create a config file in three ways:" +msgstr "您可以通过三种方式创建配置文件：" + +#: ../../source/cli/bench.rst:527 +msgid "" +"**Interactive mode** -- choose \"Export configuration\" at the summary " +"step." +msgstr "**交互模式** -- 在摘要步骤选择“导出配置”。" + +#: ../../source/cli/bench.rst:528 +msgid "**``--export-config``** -- resolve and export from CLI without running." +msgstr "**``--export-config``** -- 从 CLI 解析并导出，而不运行。" + +#: ../../source/cli/bench.rst:529 +msgid "" +"**Manually** -- write JSON with keys matching CLI arg names (dashes " +"replaced by underscores)." +msgstr "**手动** -- 编写 JSON，键名与 CLI 参数名称匹配（用下划线替换破折号）。" + +#: ../../source/cli/bench.rst:532 +msgid "Example config file:" +msgstr "示例配置文件：" + +#: ../../source/cli/bench.rst:547 +msgid "Load it with ``--config`` (engine URL must be provided separately):" +msgstr "使用 ``--config`` 加载它（引擎 URL 必须单独提供）：" + +#: ../../source/cli/bench.rst:554 +msgid "" +"CLI arguments override config file values, so you can use a base config " +"and tweak individual settings:" +msgstr "CLI 参数会覆盖配置文件中的值，因此您可以使用基本配置并调整单个设置：" + +#: ../../source/cli/bench.rst:565 +msgid "Output" +msgstr "输出" + +#: ../../source/cli/bench.rst:568 +msgid "Terminal (real-time progress)" +msgstr "终端（实时进度）" + +#: ../../source/cli/bench.rst:570 +msgid "" +"During the benchmark, a live progress display shows in-flight requests, " +"average TTFT, decode speed, and throughput. Suppress it with ``-q``." +msgstr "在基准测试期间，实时进度显示会显示正在进行的请求、平均 TTFT、解码速度和吞吐量。使用 ``-q`` 可以抑制它。" + +#: ../../source/cli/bench.rst:574 +msgid "Terminal (final summary)" +msgstr "终端（最终摘要）" + +#: ../../source/cli/bench.rst:576 +msgid "After completion, a summary table is printed:" +msgstr "完成后，将打印一个摘要表：" + +#: ../../source/cli/bench.rst:606 +msgid "CSV and JSON" +msgstr "CSV 和 JSON" + +#: ../../source/cli/bench.rst:608 +msgid "" +"``bench_results.csv`` -- per-request metrics (TTFT, latency, decode " +"speed, token counts). Written by default; skip with ``--no-csv``." +msgstr "``bench_results.csv`` -- 每个请求的指标（TTFT、延迟、解码速度、令牌计数）。默认情况下写入；使用 ``--no-csv`` 跳过。" + +#: ../../source/cli/bench.rst:610 +msgid "" +"``bench_summary.json`` -- aggregate statistics with percentiles and " +"config metadata. Opt-in with ``--json``." +msgstr "``bench_summary.json`` -- 聚合统计数据，包括百分位数和配置元数据。通过 ``--json`` 选项启用。" + +#: ../../source/cli/bench.rst:613 +msgid "Both files are written to ``--output-dir`` (default: current directory)." +msgstr "两个文件都写入到 ``--output-dir``（默认：当前目录）。" + +#: ../../source/cli/bench.rst:617 +msgid "Exit Codes" +msgstr "退出代码" + +#: ../../source/cli/bench.rst:623 +msgid "Code" +msgstr "代码" + +#: ../../source/cli/bench.rst:624 +msgid "Meaning" +msgstr "含义" + +#: ../../source/cli/bench.rst:625 +msgid "``0``" +msgstr "``0``" + +#: ../../source/cli/bench.rst:626 +msgid "All requests succeeded." +msgstr "所有请求均成功。" + +#: ../../source/cli/bench.rst:627 +msgid "``1``" +msgstr "``1``" + +#: ../../source/cli/bench.rst:628 +msgid "One or more requests failed." +msgstr "一个或多个请求失败。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/cli/bench_kvcache.po b/docs/source/locale/zh_CN/LC_MESSAGES/cli/bench_kvcache.po new file mode 100644 index 00000000000..c2a0543ee80 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/cli/bench_kvcache.po @@ -0,0 +1,353 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/cli/bench_kvcache.rst:2 +msgid "lmcache bench kvcache" +msgstr "lmcache bench kvcache" + +#: ../../source/cli/bench_kvcache.rst:4 +msgid "" +"The ``lmcache bench kvcache`` command is an end-to-end sanity test for " +"the LMCache Multi-Process (MP) cache server. It connects to a running " +"server over ZMQ and exercises the full KV-cache data path for a sequence " +"of synthetic requests, then optionally verifies per-chunk checksums " +"through the HTTP API." +msgstr "``lmcache bench kvcache`` 命令是对 LMCache 多进程 (MP) 缓存服务器的端到端完整性测试。它通过 ZMQ 连接到正在运行的服务器，并对一系列合成请求执行完整的 KV 缓存数据路径，然后可选地通过 HTTP API 验证每个块的校验和。" + +#: ../../source/cli/bench_kvcache.rst:14 +msgid "" +"Unlike :ref:`lmcache bench engine `, this command " +"does **not** require an inference engine. It only needs a running LMCache" +" MP server (ZMQ + HTTP) and a GPU." +msgstr "与 :ref:`lmcache bench engine ` 不同，此命令**不**需要推理引擎。它只需要一个正在运行的 LMCache MP 服务器（ZMQ + HTTP）和一个 GPU。" + +#: ../../source/cli/bench_kvcache.rst:20 +msgid "What it does" +msgstr "它的作用" + +#: ../../source/cli/bench_kvcache.rst:22 +msgid "For each sequence in ``[--start, --end)``, the tool runs two passes:" +msgstr "对于 ``[--start, --end)`` 中的每个序列，该工具运行两个阶段：" + +#: ../../source/cli/bench_kvcache.rst:24 +msgid "" +"**Cold pass** -- ``LOOKUP`` is expected to miss, so the generated KV " +"tensors are ``STORE``\\ d on the server." +msgstr "**冷启动** -- ``LOOKUP`` 预计会未命中，因此生成的 KV 张量会被 ``STORE``\\\\ 存储在服务器上。" + +#: ../../source/cli/bench_kvcache.rst:26 +msgid "" +"**Warm pass** -- ``LOOKUP`` is expected to hit; the tool issues " +"``RETRIEVE`` and compares the retrieved KV chunks' checksums to the " +"originals." +msgstr "**热通道** -- ``LOOKUP`` 预计会命中；工具发出 ``RETRIEVE`` 并将检索到的 KV 块的校验和与原始值进行比较。" + +#: ../../source/cli/bench_kvcache.rst:30 +msgid "The full RPC path exercised is::" +msgstr "完整的 RPC 路径是::" + +#: ../../source/cli/bench_kvcache.rst:36 +msgid "" +"When ``--url`` points to the server's HTTP endpoint, per-chunk checksums " +"are additionally cross-checked against the server-side computation, so a " +"mismatch between producer and consumer surfaces as a loud ``CHECKSUM " +"MISMATCH`` log line." +msgstr "当 ``--url`` 指向服务器的 HTTP 端点时，逐块的校验和会额外与服务器端的计算进行交叉检查，因此生产者和消费者之间的不匹配会以显眼的 ``CHECKSUM MISMATCH`` 日志行的形式显现出来。" + +#: ../../source/cli/bench_kvcache.rst:43 +msgid "Quick start" +msgstr "快速开始" + +#: ../../source/cli/bench_kvcache.rst:45 +msgid "Start the MP server in one terminal:" +msgstr "在一个终端中启动 MP 服务器：" + +#: ../../source/cli/bench_kvcache.rst:54 +msgid "Then in another terminal:" +msgstr "然后在另一个终端中：" + +#: ../../source/cli/bench_kvcache.rst:62 +msgid "" +"By default the tool runs forever (``--end`` unset); stop it with " +"``Ctrl-C`` at any time. Pass ``--end N`` for a bounded run." +msgstr "默认情况下，该工具会一直运行（``--end`` 未设置）；您可以随时使用 ``Ctrl-C`` 停止它。传递 ``--end N`` 以进行有限次运行。" + +#: ../../source/cli/bench_kvcache.rst:67 +msgid "Options" +msgstr "选项" + +#: ../../source/cli/bench_kvcache.rst:73 +msgid "Flag" +msgstr "标志" + +#: ../../source/cli/bench_kvcache.rst:74 +msgid "Default" +msgstr "默认" + +#: ../../source/cli/bench_kvcache.rst:75 +msgid "Description" +msgstr "描述" + +#: ../../source/cli/bench_kvcache.rst:76 +msgid "``--rpc-url URL``" +msgstr "``--rpc-url URL``" + +#: ../../source/cli/bench_kvcache.rst:77 +msgid "``tcp://localhost:5555``" +msgstr "``tcp://localhost:5555``" + +#: ../../source/cli/bench_kvcache.rst:78 +msgid "ZMQ endpoint of the MP cache server." +msgstr "MP 缓存服务器的 ZMQ 端点。" + +#: ../../source/cli/bench_kvcache.rst:79 +msgid "``--url URL``" +msgstr "``--url URL``" + +#: ../../source/cli/bench_kvcache.rst:80 +msgid "``http://localhost:8080``" +msgstr "``http://localhost:8080``" + +#: ../../source/cli/bench_kvcache.rst:81 +msgid "" +"HTTP base URL of the server's checksum API. Used to verify per-chunk " +"checksums end-to-end." +msgstr "服务器的校验和 API 的 HTTP 基础 URL。用于端到端验证每个块的校验和。" + +#: ../../source/cli/bench_kvcache.rst:83 +#, python-brace-format +msgid "``--mode {gpu}``" +msgstr "``--mode {gpu}``" + +#: ../../source/cli/bench_kvcache.rst:84 +msgid "``gpu``" +msgstr "``gpu``" + +#: ../../source/cli/bench_kvcache.rst:85 +msgid "" +"Run mode. Only ``gpu`` is supported today; CPU mode is a planned follow-" +"up." +msgstr "运行模式。目前仅支持 ``gpu``；CPU 模式是计划中的后续功能。" + +#: ../../source/cli/bench_kvcache.rst:87 +msgid "``--num-tokens N``" +msgstr "``--num-tokens N``" + +#: ../../source/cli/bench_kvcache.rst:88 +msgid "``512``" +msgstr "``512``" + +#: ../../source/cli/bench_kvcache.rst:89 +msgid "Tokens per synthetic request." +msgstr "每个合成请求的令牌数。" + +#: ../../source/cli/bench_kvcache.rst:90 +msgid "``--num-blocks N``" +msgstr "``--num-blocks N``" + +#: ../../source/cli/bench_kvcache.rst:91 +msgid "``1024``" +msgstr "``1024``" + +#: ../../source/cli/bench_kvcache.rst:92 +msgid "Number of paged blocks allocated on the GPU." +msgstr "在 GPU 上分配的分页块数量。" + +#: ../../source/cli/bench_kvcache.rst:93 +msgid "``--block-size N``" +msgstr "``--block-size N``" + +#: ../../source/cli/bench_kvcache.rst:94 +msgid "``16``" +msgstr "``16``" + +#: ../../source/cli/bench_kvcache.rst:95 +msgid "Tokens per paged block." +msgstr "每个分页块的令牌数。" + +#: ../../source/cli/bench_kvcache.rst:96 +msgid "``--start N``" +msgstr "``--start N``" + +#: ../../source/cli/bench_kvcache.rst:97 ../../source/cli/bench_kvcache.rst:178 +msgid "``0``" +msgstr "``0``" + +#: ../../source/cli/bench_kvcache.rst:98 +msgid "First sequence number to run." +msgstr "要运行的第一个序列号。" + +#: ../../source/cli/bench_kvcache.rst:99 +msgid "``--end N``" +msgstr "``--end N``" + +#: ../../source/cli/bench_kvcache.rst:100 +msgid "*(unset)*" +msgstr "*(未设置)*" + +#: ../../source/cli/bench_kvcache.rst:101 +msgid "" +"Exclusive upper bound on sequence numbers. When omitted the loop runs " +"forever." +msgstr "序列号的独占上限。当省略时，循环将永远运行。" + +#: ../../source/cli/bench_kvcache.rst:103 +msgid "``--interval SECS``" +msgstr "``--interval SECS``" + +#: ../../source/cli/bench_kvcache.rst:104 +msgid "``0.5``" +msgstr "``0.5``" + +#: ../../source/cli/bench_kvcache.rst:105 +msgid "Delay between successive sub-passes." +msgstr "连续子通道之间的延迟。" + +#: ../../source/cli/bench_kvcache.rst:106 +msgid "``--kvcache-shape-spec SPEC``" +msgstr "``--kvcache-shape-spec SPEC``" + +#: ../../source/cli/bench_kvcache.rst:107 +msgid "``(2,1024,16,8,128):float16:32``" +msgstr "``(2,1024,16,8,128):float16:32``" + +#: ../../source/cli/bench_kvcache.rst:108 +msgid "KV cache shape spec (see below)." +msgstr "KV Cache 形状规格（见下文）。" + +#: ../../source/cli/bench_kvcache.rst:112 +msgid "KV cache shape spec" +msgstr "KV 缓存形状规范" + +#: ../../source/cli/bench_kvcache.rst:114 +msgid "" +"The ``--kvcache-shape-spec`` flag describes how KV tensors are laid out " +"on the GPU. A spec is one or more groups separated by ``;``:" +msgstr "``--kvcache-shape-spec`` 标志描述了 KV 张量在 GPU 上的布局。规格是一个或多个用 ``;`` 分隔的组：" + +#: ../../source/cli/bench_kvcache.rst:121 +msgid "Fields:" +msgstr "字段：" + +#: ../../source/cli/bench_kvcache.rst:123 +msgid "``kv_size`` -- 2 for classical attention (separate K/V), 1 for MLA." +msgstr "``kv_size`` -- 经典注意力（分离 K/V）为 2，MLA 为 1。" + +#: ../../source/cli/bench_kvcache.rst:124 +msgid "``NB`` -- number of paged blocks." +msgstr "``NB`` -- 分页块的数量。" + +#: ../../source/cli/bench_kvcache.rst:125 +msgid "``BS`` -- block size (tokens per block)." +msgstr "``BS`` -- 块大小（每块的令牌数）。" + +#: ../../source/cli/bench_kvcache.rst:126 +msgid "``NH`` -- number of attention heads per layer." +msgstr "``NH`` -- 每层的注意力头数量。" + +#: ../../source/cli/bench_kvcache.rst:127 +msgid "``HS`` -- head size (in elements)." +msgstr "``HS`` -- 头部大小（以元素为单位）。" + +#: ../../source/cli/bench_kvcache.rst:128 +msgid "" +"``dtype`` -- element dtype (e.g. ``float16``, ``bfloat16``, ``float32``, " +"``uint8``). The full set matches the keys of ``DTYPE_MAP`` in " +"``lmcache/v1/kv_layer_groups.py``." +msgstr "``dtype`` -- 元素数据类型（例如 ``float16``、``bfloat16``、``float32``、``uint8``）。完整的集合与 ``lmcache/v1/kv_layer_groups.py`` 中的 ``DTYPE_MAP`` 的键匹配。" + +#: ../../source/cli/bench_kvcache.rst:131 +msgid "``layers`` -- number of layers in this group." +msgstr "``layers`` -- 该组中的层数。" + +#: ../../source/cli/bench_kvcache.rst:133 +msgid "" +"Multi-group specs let you model heterogeneous layers (for example, MLA " +"layers + classical attention layers in the same model):" +msgstr "多组规格允许您建模异构层（例如，在同一模型中结合 MLA 层和经典注意力层）：" + +#: ../../source/cli/bench_kvcache.rst:142 +msgid "" +"All groups must share the same ``NB`` and ``BS`` (this is a physical " +"constraint of paged KV). Layer counts across groups sum to the total " +"layer count registered with the server." +msgstr "所有组必须共享相同的 ``NB`` 和 ``BS``（这是分页 KV 的物理限制）。各组的层数总和等于注册到服务器的总层数。" + +#: ../../source/cli/bench_kvcache.rst:146 +msgid "" +"See ``parse_kvcache_shape_spec`` in ``lmcache/v1/kv_layer_groups.py`` for" +" the authoritative parsing rules and validation errors." +msgstr "请参阅 ``parse_kvcache_shape_spec`` 在 ``lmcache/v1/kv_layer_groups.py`` 中的权威解析规则和验证错误。" + +#: ../../source/cli/bench_kvcache.rst:151 +msgid "Example output" +msgstr "示例输出" + +#: ../../source/cli/bench_kvcache.rst:165 +msgid "" +"Any ``CHECKSUM MISMATCH``, ``ERROR``, or Python traceback in the log " +"indicates a real problem worth investigating." +msgstr "日志中任何 ``CHECKSUM MISMATCH``、``ERROR`` 或 Python 回溯都表示存在值得调查的实际问题。" + +#: ../../source/cli/bench_kvcache.rst:170 +msgid "Exit codes" +msgstr "退出代码" + +#: ../../source/cli/bench_kvcache.rst:176 +msgid "Code" +msgstr "代码" + +#: ../../source/cli/bench_kvcache.rst:177 +msgid "Meaning" +msgstr "含义" + +#: ../../source/cli/bench_kvcache.rst:179 +msgid "" +"Test loop completed (or was interrupted cleanly with Ctrl-C) with no " +"checksum mismatches." +msgstr "测试循环完成（或通过 Ctrl-C 干净地中断），没有校验和不匹配。" + +#: ../../source/cli/bench_kvcache.rst:181 +msgid "``1``" +msgstr "``1``" + +#: ../../source/cli/bench_kvcache.rst:182 +msgid "" +"Fatal error (for example, CUDA unavailable in ``--mode gpu``, server " +"unreachable, or a checksum mismatch)." +msgstr "致命错误（例如，在 ``--mode gpu`` 中 CUDA 不可用、服务器无法访问或校验和不匹配）。" + +#: ../../source/cli/bench_kvcache.rst:187 +msgid "See also" +msgstr "另请参阅" + +#: ../../source/cli/bench_kvcache.rst:189 +msgid "" +":doc:`bench` -- ``lmcache bench engine`` for engine-side workload " +"benchmarks." +msgstr "：doc:`bench` -- ``lmcache bench engine`` 用于引擎端工作负载基准测试。" + +#: ../../source/cli/bench_kvcache.rst:191 +msgid "" +":doc:`kvcache` -- ``lmcache kvcache`` for managing KV cache state on a " +"running server (clear, etc.)." +msgstr ":doc:`kvcache` -- ``lmcache kvcache`` 用于管理运行服务器上的 KV Cache 状态（清除等）。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/cli/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/cli/index.po new file mode 100644 index 00000000000..5554f060708 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/cli/index.po @@ -0,0 +1,103 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/cli/index.rst:2 +msgid "CLI Reference" +msgstr "CLI 参考" + +#: ../../source/cli/index.rst:4 +msgid "" +"The ``lmcache`` command-line interface provides tools for managing and " +"inspecting LMCache servers." +msgstr "``lmcache`` 命令行接口提供了管理和检查 LMCache 服务器的工具。" + +#: ../../source/cli/index.rst:11 +msgid "" +"After installing LMCache, the ``lmcache`` command is available globally. " +"Run ``lmcache -h`` to see all commands." +msgstr "安装 LMCache 后，``lmcache`` 命令可以全局使用。运行 ``lmcache -h`` 查看所有命令。" + +#: ../../source/cli/index.rst:15 +msgid "Available Commands" +msgstr "可用命令" + +#: ../../source/cli/index.rst:21 +msgid "Command" +msgstr "命令" + +#: ../../source/cli/index.rst:22 +msgid "Description" +msgstr "描述" + +#: ../../source/cli/index.rst:23 +msgid "``describe``" +msgstr "``describe``" + +#: ../../source/cli/index.rst:24 +msgid "Show detailed status of a running LMCache service." +msgstr "显示正在运行的 LMCache 服务的详细状态。" + +#: ../../source/cli/index.rst:25 +msgid "``query``" +msgstr "``query``" + +#: ../../source/cli/index.rst:26 +msgid "Single-shot query interface for the serving engine." +msgstr "服务引擎的单次查询接口。" + +#: ../../source/cli/index.rst:27 +msgid "``ping``" +msgstr "``ping``" + +#: ../../source/cli/index.rst:28 +msgid "Liveness check for LMCache or vLLM servers." +msgstr "对 LMCache 或 vLLM 服务器的存活检查。" + +#: ../../source/cli/index.rst:29 +msgid "``bench``" +msgstr "``bench``" + +#: ../../source/cli/index.rst:30 +msgid "" +"Run sustained performance benchmarks against an inference engine, or an " +"end-to-end sanity test against an LMCache MP server." +msgstr "对推理引擎运行持续性能基准测试，或对 LMCache MP 服务器进行端到端的完整性测试。" + +#: ../../source/cli/index.rst:32 +msgid "``kvcache``" +msgstr "``kvcache``" + +#: ../../source/cli/index.rst:33 +msgid "Manage KV cache state (e.g. clear L1 cache)." +msgstr "管理 KV Cache 状态（例如，清除 L1 缓存）。" + +#: ../../source/cli/index.rst:34 +msgid "``server``" +msgstr "``server``" + +#: ../../source/cli/index.rst:35 +msgid "Launch the LMCache server (ZMQ + HTTP)." +msgstr "启动 LMCache 服务器（ZMQ + HTTP）。" + +#: ../../source/cli/index.rst:37 +msgid "For a comprehensive guide with examples, see :doc:`/getting_started/cli`." +msgstr "有关带有示例的全面指南，请参见 :doc:`/getting_started/cli`。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/cli/kvcache.po b/docs/source/locale/zh_CN/LC_MESSAGES/cli/kvcache.po new file mode 100644 index 00000000000..10203f02020 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/cli/kvcache.po @@ -0,0 +1,157 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/cli/kvcache.rst:2 +msgid "lmcache kvcache" +msgstr "lmcache kvcache" + +#: ../../source/cli/kvcache.rst:4 +msgid "" +"The ``lmcache kvcache`` command manages KV cache state on a running " +"LMCache server." +msgstr "``lmcache kvcache`` 命令管理正在运行的 LMCache 服务器上的 KV Cache 状态。" + +#: ../../source/cli/kvcache.rst:28 +msgid "clear" +msgstr "清除" + +#: ../../source/cli/kvcache.rst:30 +msgid "" +"Clear all cached KV data in **L1 (CPU memory)** on the target LMCache " +"server." +msgstr "清除目标 LMCache 服务器上 **L1 (CPU 内存)** 中所有缓存的 KV 数据。" + +#: ../../source/cli/kvcache.rst:36 +msgid "**Example:**" +msgstr "**示例:**" + +#: ../../source/cli/kvcache.rst:46 +msgid "**JSON output** (for scripting with ``jq``):" +msgstr "**JSON 输出** (用于与 ``jq`` 脚本交互):" + +#: ../../source/cli/kvcache.rst:58 +msgid "**Quiet mode** (exit code only, no output):" +msgstr "**静默模式**（仅退出代码，无输出）：" + +#: ../../source/cli/kvcache.rst:67 +msgid "Options" +msgstr "选项" + +#: ../../source/cli/kvcache.rst:73 +msgid "Flag" +msgstr "标志" + +#: ../../source/cli/kvcache.rst:74 +msgid "Required" +msgstr "必需的" + +#: ../../source/cli/kvcache.rst:75 +msgid "Description" +msgstr "描述" + +#: ../../source/cli/kvcache.rst:76 +msgid "``--url``" +msgstr "``--url``" + +#: ../../source/cli/kvcache.rst:77 +msgid "Yes" +msgstr "是的" + +#: ../../source/cli/kvcache.rst:78 +msgid "URL of the LMCache MP HTTP server (e.g. ``http://localhost:8000``)." +msgstr "LMCache MP HTTP 服务器的 URL（例如 ``http://localhost:8000``）。" + +#: ../../source/cli/kvcache.rst:79 +msgid "``--format``" +msgstr "``--format``" + +#: ../../source/cli/kvcache.rst:80 ../../source/cli/kvcache.rst:83 +#: ../../source/cli/kvcache.rst:86 +msgid "No" +msgstr "否" + +#: ../../source/cli/kvcache.rst:81 +msgid "Output format: ``terminal`` (default) or ``json``." +msgstr "输出格式：``terminal``（默认）或``json``。" + +#: ../../source/cli/kvcache.rst:82 +msgid "``--output``" +msgstr "``--output``" + +#: ../../source/cli/kvcache.rst:84 +msgid "Save output to a file (uses the format chosen by ``--format``)." +msgstr "将输出保存到文件中（使用 ``--format`` 选择的格式）。" + +#: ../../source/cli/kvcache.rst:85 +msgid "``-q`` / ``--quiet``" +msgstr "``-q`` / ``--quiet``" + +#: ../../source/cli/kvcache.rst:87 +msgid "Suppress stdout. Useful in scripts where you only need the exit code." +msgstr "抑制标准输出。在只需要退出代码的脚本中很有用。" + +#: ../../source/cli/kvcache.rst:90 +msgid "Exit Codes" +msgstr "退出代码" + +#: ../../source/cli/kvcache.rst:96 +msgid "Code" +msgstr "代码" + +#: ../../source/cli/kvcache.rst:97 +msgid "Meaning" +msgstr "含义" + +#: ../../source/cli/kvcache.rst:98 +msgid "``0``" +msgstr "``0``" + +#: ../../source/cli/kvcache.rst:99 +msgid "Success." +msgstr "成功。" + +#: ../../source/cli/kvcache.rst:100 +msgid "``1``" +msgstr "``1``" + +#: ../../source/cli/kvcache.rst:101 +msgid "Error (connection failure, server error, bad arguments)." +msgstr "错误（连接失败、服务器错误、参数错误）。" + +#: ../../source/cli/kvcache.rst:104 +msgid "Common Patterns" +msgstr "常见模式" + +#: ../../source/cli/kvcache.rst:106 +msgid "**Handle temporary server unavailability:**" +msgstr "**处理临时服务器不可用:**" + +#: ../../source/cli/kvcache.rst:108 +msgid "" +"If the server is temporarily unreachable (e.g. due to network issue), the" +" command fails with exit code 1. For persistent connectivity issues, use " +"``lmcache ping`` to diagnose." +msgstr "如果服务器暂时无法访问（例如，由于网络问题），命令将以退出代码 1 失败。对于持续的连接问题，请使用 ``lmcache ping`` 进行诊断。" + +#: ../../source/cli/kvcache.rst:120 +msgid "**Clear cache and capture JSON result:**" +msgstr "**清除缓存并捕获 JSON 结果：**" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/community/blogs.po b/docs/source/locale/zh_CN/LC_MESSAGES/community/blogs.po new file mode 100644 index 00000000000..3ac742ac3b5 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/community/blogs.po @@ -0,0 +1,36 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/community/blogs.rst:2 +msgid "Blogs" +msgstr "博客" + +#: ../../source/community/blogs.rst:4 +msgid "" +"LMCache is a community-driven open-source project and we publish regular " +"blog posts to share updates, performance comparison and new features. You" +" can find the latest blog posts below:" +msgstr "LMCache 是一个由社区驱动的开源项目，我们定期发布博客文章以分享更新、性能比较和新功能。您可以在下面找到最新的博客文章：" + +#: ../../source/community/blogs.rst:7 +msgid "`LMCache blogs `_" +msgstr "`LMCache blogs `_" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/community/meetings.po b/docs/source/locale/zh_CN/LC_MESSAGES/community/meetings.po new file mode 100644 index 00000000000..fc4f545b6b0 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/community/meetings.po @@ -0,0 +1,133 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/community/meetings.rst:2 +msgid "Community meetings" +msgstr "社区会议" + +#: ../../source/community/meetings.rst:4 +msgid "" +"LMCache hosts regular community meetings to discuss updates, address new " +"feature requests, and feedback from the community. If you are interested " +"in contributing to the LMCache projects (core LMCache or Production " +"Stack), we encourage you to join the meetings. We also host a monthly " +"\"Office Hour\" on select topics." +msgstr "LMCache 定期举办社区会议，以讨论更新、处理新功能请求以及收集社区反馈。如果您有兴趣为 LMCache 项目（核心 LMCache 或生产堆栈）做出贡献，我们鼓励您参加会议。我们还会就特定主题举办每月的“办公时间”。" + +#: ../../source/community/meetings.rst:10 +msgid "Meeting schedule" +msgstr "会议日程" + +#: ../../source/community/meetings.rst:11 +msgid "The LMCache community meetings held separately for each project:" +msgstr "LMCache 社区会议为每个项目单独举行：" + +#: ../../source/community/meetings.rst:13 +msgid "LMCache - `Github `__" +msgstr "LMCache - `Github `__" + +#: ../../source/community/meetings.rst:14 +msgid "" +"Production Stack - `Github `__" +msgstr "生产栈 - `Github `__" + +#: ../../source/community/meetings.rst:17 +msgid "LMCache Project" +msgstr "LMCache 项目" + +#: ../../source/community/meetings.rst:19 +msgid "" +"The LMCache community meeting is held bi-weekly on **Tuesdays** at " +"**9:00-9:30 AM PT**." +msgstr "LMCache 社区会议每两周在 **星期二** 的 **上午 9:00-9:30（太平洋时间）** 举行。" + +#: ../../source/community/meetings.rst:21 +msgid "Please find the meeting invite link below:" +msgstr "请在下面找到会议邀请链接：" + +#: ../../source/community/meetings.rst:23 +#: ../../source/community/meetings.rst:33 +msgid "" +"**Meeting link**: `Zoom link " +"`_" +msgstr "**会议链接**: `Zoom 链接 `_" + +#: ../../source/community/meetings.rst:24 +msgid "" +"**Calendar Invite**: `Google Calendar " +"`__" +msgstr "**日历邀请**: `Google 日历 `__" + +#: ../../source/community/meetings.rst:25 +msgid "" +"**Slack Channel**: `#lmcache " +"`_" +msgstr "**Slack 频道**: `#lmcache `_" + +#: ../../source/community/meetings.rst:28 +msgid "vLLM Production Stack Project" +msgstr "vLLM 生产栈项目" + +#: ../../source/community/meetings.rst:30 +msgid "" +"The Production Stack community meeting is held bi-weekly on **Tuesdays** " +"at **5:30-6:00 PM PT**. Please find the meeting invite link below:" +msgstr "生产堆栈社区会议每两周在**星期二**的**下午5:30-6:00（太平洋时间）**举行。请在下面找到会议邀请链接：" + +#: ../../source/community/meetings.rst:34 +msgid "" +"**Calendar Invite**: `Google Calendar " +"`__" +msgstr "**日历邀请**： `Google 日历 `__" + +#: ../../source/community/meetings.rst:35 +msgid "" +"**Slack Channel**: `#production-stack `_" +msgstr "**Slack 频道**: `#production-stack `_" + +#: ../../source/community/meetings.rst:39 +msgid "" +"The Zoom meeting link is the same for both LMCache and Production Stack " +"community meetings. Meeting notes are available here: `Meeting notes " +"`_." +msgstr "Zoom 会议链接对于 LMCache 和 Production Stack 社区会议是相同的。会议记录可以在这里查看：`会议记录 `_。" + +#: ../../source/community/meetings.rst:43 +msgid "LMCache Office Hours" +msgstr "LMCache 办公时间" + +#: ../../source/community/meetings.rst:45 +msgid "Held monthly on the second Wednesday of the month at 2PM ET, 11AM PT." +msgstr "每月在东部时间下午2点，太平洋时间上午11点的第二个星期三举行。" + +#: ../../source/community/meetings.rst:47 +msgid "Topic announced on the LMCache #office-hour Slack channel" +msgstr "在 LMCache #office-hour Slack 频道上宣布主题" + +#: ../../source/community/meetings.rst:48 +msgid "" +"Request a calendar invite via `this form `_" +msgstr "通过 `此表单 `_ 请求日历邀请" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/controller/freeze_mode.po b/docs/source/locale/zh_CN/LC_MESSAGES/controller/freeze_mode.po new file mode 100644 index 00000000000..a1f1babb820 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/controller/freeze_mode.po @@ -0,0 +1,191 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/controller/freeze_mode.rst:2 +msgid "Freeze Mode" +msgstr "冻结模式" + +#: ../../source/controller/freeze_mode.rst:5 +msgid "Overview" +msgstr "概述" + +#: ../../source/controller/freeze_mode.rst:7 +msgid "" +"Freeze Mode is a safety mechanism in LMCache Controller designed to " +"prevent data inconsistency and potential data loss during critical system" +" events. When the controller detects severe state inconsistencies or is " +"undergoing restart/recovery, it can activate Freeze Mode to temporarily " +"restrict certain operations." +msgstr "冻结模式是 LMCache 控制器中的一种安全机制，旨在防止在关键系统事件期间出现数据不一致和潜在的数据丢失。当控制器检测到严重的状态不一致或正在进行重启/恢复时，它可以激活冻结模式，以暂时限制某些操作。" + +#: ../../source/controller/freeze_mode.rst:10 +msgid "Motivation" +msgstr "动机" + +#: ../../source/controller/freeze_mode.rst:12 +msgid "The primary motivations for Freeze Mode are:" +msgstr "冻结模式的主要动机包括：" + +#: ../../source/controller/freeze_mode.rst:14 +msgid "" +"**Controller Failure Recovery** - When the controller crashes and " +"restarts, it may need to perform a full synchronization with workers - " +"During this sync period, the system state may be incomplete or " +"inconsistent" +msgstr "**控制器故障恢复** - 当控制器崩溃并重启时，它可能需要与工作节点进行完全同步 - 在此同步期间，系统状态可能不完整或不一致" + +#: ../../source/controller/freeze_mode.rst:18 +msgid "" +"**State Inconsistency Detection** - When severe state mismatches are " +"detected between the controller and workers - This could happen due to " +"network issues, worker failures, or unexpected system behavior" +msgstr "**状态不一致检测** - 当检测到控制器与工作节点之间存在严重的状态不匹配时 - 这可能是由于网络问题、工作节点故障或意外的系统行为导致的" + +#: ../../source/controller/freeze_mode.rst:22 +msgid "" +"**Full Synchronization Safety** - During full sync operations triggered " +"by the above events - To prevent operations that could cause eviction or " +"admission events while state is being rebuilt" +msgstr "**完全同步安全** - 在上述事件触发的完全同步操作期间 - 为了防止在状态重建时可能导致逐出或接纳事件的操作" + +#: ../../source/controller/freeze_mode.rst:27 +msgid "Purpose" +msgstr "目的" + +#: ../../source/controller/freeze_mode.rst:29 +msgid "When Freeze Mode is activated:" +msgstr "当冻结模式被激活时：" + +#: ../../source/controller/freeze_mode.rst:31 +msgid "**All operations that could generate evict/admit events are disabled**" +msgstr "**所有可能生成逐出/接纳事件的操作均被禁用**" + +#: ../../source/controller/freeze_mode.rst:32 +msgid "Only read operations and basic queries are allowed" +msgstr "仅允许读取操作和基本查询" + +#: ../../source/controller/freeze_mode.rst:33 +msgid "" +"The system essentially enters a \"read-only\" state for cache management " +"operations" +msgstr "系统本质上进入了一个“只读”状态，以进行缓存管理操作。" + +#: ../../source/controller/freeze_mode.rst:34 +msgid "This prevents data corruption or loss during the recovery/sync process" +msgstr "这可以防止在恢复/同步过程中数据损坏或丢失。" + +#: ../../source/controller/freeze_mode.rst:37 +msgid "Current Implementation Status" +msgstr "当前实现状态" + +#: ../../source/controller/freeze_mode.rst:39 +msgid "**Currently Implemented:**" +msgstr "**当前实现：**" + +#: ../../source/controller/freeze_mode.rst:41 +msgid "" +"**Freeze Mode Mechanism** - StorageManager supports freeze mode - When " +"frozen, only LocalCPUBackend is used for retrieval operations - Other " +"storage backends are temporarily excluded from active use" +msgstr "**冻结模式机制** - StorageManager 支持冻结模式 - 当处于冻结状态时，仅使用 LocalCPUBackend 进行检索操作 - 其他存储后端暂时不参与活动使用" + +#: ../../source/controller/freeze_mode.rst:46 +msgid "" +"**API Endpoints** - Freeze mode can be toggled via controller API - " +"Status can be queried to check if system is in freeze mode" +msgstr "**API 端点** - 冻结模式可以通过控制器 API 切换 - 可以查询状态以检查系统是否处于冻结模式" + +#: ../../source/controller/freeze_mode.rst:50 +msgid "**Not Yet Implemented:**" +msgstr "**尚未实现：**" + +#: ../../source/controller/freeze_mode.rst:52 +msgid "" +"**Automatic Triggering** - Controller does not yet automatically trigger " +"freeze mode - Manual activation via API is currently required" +msgstr "**自动触发** - 控制器尚未自动触发冻结模式 - 目前需要通过 API 手动激活" + +#: ../../source/controller/freeze_mode.rst:56 +msgid "" +"**Full Sync Integration** - Full synchronization process that would " +"trigger freeze mode is not implemented - Controller restart detection and" +" automatic freeze mode activation not yet developed" +msgstr "**完全同步集成** - 触发冻结模式的完整同步过程尚未实现 - 控制器重启检测和自动冻结模式激活尚未开发" + +#: ../../source/controller/freeze_mode.rst:60 +msgid "" +"**Recovery Completion** - Automatic unfreeze after sync completion not " +"yet implemented - Manual intervention currently required to exit freeze " +"mode" +msgstr "**恢复完成** - 同步完成后自动解冻尚未实现 - 目前需要手动干预以退出冻结模式" + +#: ../../source/controller/freeze_mode.rst:65 +msgid "Usage" +msgstr "使用方法" + +#: ../../source/controller/freeze_mode.rst:67 +msgid "**Manual Activation/Deactivation:**" +msgstr "**手动激活/停用：**" + +#: ../../source/controller/freeze_mode.rst:69 +msgid "Freeze mode can be manually controlled through the Controller API:" +msgstr "冻结模式可以通过控制器 API 手动控制：" + +#: ../../source/controller/freeze_mode.rst:82 +msgid "**System Behavior in Freeze Mode:**" +msgstr "**冻结模式下的系统行为：**" + +#: ../../source/controller/freeze_mode.rst:84 +msgid "Cache retrieval operations continue to work" +msgstr "缓存检索操作继续正常工作" + +#: ../../source/controller/freeze_mode.rst:85 +msgid "Only LocalCPUBackend is used for all retrievals" +msgstr "仅使用 LocalCPUBackend 进行所有检索。" + +#: ../../source/controller/freeze_mode.rst:86 +msgid "Cache admission/eviction operations are blocked" +msgstr "缓存接纳/逐出操作被阻止" + +#: ../../source/controller/freeze_mode.rst:87 +msgid "Write operations to storage backends are prevented" +msgstr "对存储后端的写入操作被阻止" + +#: ../../source/controller/freeze_mode.rst:88 +msgid "Monitoring and health checks continue normally" +msgstr "监控和健康检查正常进行" + +#: ../../source/controller/freeze_mode.rst:91 +msgid "Related Documentation" +msgstr "相关文档" + +#: ../../source/controller/freeze_mode.rst:93 +msgid ":doc:`index` - Controller WebUI overview" +msgstr ":doc:`index` - 控制器 WebUI 概述" + +#: ../../source/controller/freeze_mode.rst:94 +msgid ":doc:`../api_reference/configurations` - API and configuration reference" +msgstr ":doc:`../api_reference/configurations` - API 和配置参考" + +#: ../../source/controller/freeze_mode.rst:95 +msgid ":doc:`../storage_backend/index` - Storage backend architecture" +msgstr ":doc:`../storage_backend/index` - 存储后端架构" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/controller/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/controller/index.po new file mode 100644 index 00000000000..c017b0573a4 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/controller/index.po @@ -0,0 +1,118 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/controller/index.rst:2 +msgid "Controller WebUI" +msgstr "控制器 WebUI" + +#: ../../source/controller/index.rst:5 +msgid "Overview" +msgstr "概述" + +#: ../../source/controller/index.rst:7 +msgid "" +"The LMCache Controller provides a web-based dashboard for monitoring your" +" LMCache instances. This web interface allows you to monitor system " +"status, instances, workers, and performance metrics." +msgstr "LMCache 控制器提供了一个基于网页的仪表板，用于监控您的 LMCache 实例。该网页界面允许您监控系统状态、实例、工作进程和性能指标。" + +#: ../../source/controller/index.rst:10 +msgid "Quick Start" +msgstr "快速开始" + +#: ../../source/controller/index.rst:12 +msgid "" +"To enable the Controller WebUI, start your LMCache instance with the " +"following command:" +msgstr "要启用 Controller WebUI，请使用以下命令启动您的 LMCache 实例：" + +#: ../../source/controller/index.rst:23 +msgid "After starting the controller, access the WebUI at:" +msgstr "启动控制器后，可以通过以下地址访问 WebUI：" + +#: ../../source/controller/index.rst:30 +msgid "Configuration Options" +msgstr "配置选项" + +#: ../../source/controller/index.rst:32 +msgid "``--host``: Bind address for the API server (default: 0.0.0.0)" +msgstr "``--host``: API 服务器的绑定地址（默认: 0.0.0.0）" + +#: ../../source/controller/index.rst:33 +msgid "``--port``: Port for the API server (default: 9000)" +msgstr "``--port``: API 服务器的端口（默认: 9000）" + +#: ../../source/controller/index.rst:34 +msgid "``--monitor-ports``: ZMQ ports for controller communication" +msgstr "``--monitor-ports``: 控制器通信的 ZMQ 端口" + +#: ../../source/controller/index.rst:35 +msgid "``--lmcache-worker-timeout``: Worker timeout in seconds" +msgstr "``--lmcache-worker-timeout``: 工作线程超时时间（秒）" + +#: ../../source/controller/index.rst:36 +msgid "``--health-check-interval``: Health check interval in seconds" +msgstr "``--health-check-interval``: 健康检查间隔（秒）" + +#: ../../source/controller/index.rst:39 +msgid "Dashboard Features" +msgstr "仪表板功能" + +#: ../../source/controller/index.rst:41 +msgid "The Controller Dashboard provides:" +msgstr "控制器仪表板提供：" + +#: ../../source/controller/index.rst:43 +msgid "System overview and health monitoring" +msgstr "系统概览和健康监控" + +#: ../../source/controller/index.rst:44 +msgid "Instance and worker management" +msgstr "实例和工作者管理" + +#: ../../source/controller/index.rst:45 +msgid "Performance metrics" +msgstr "性能指标" + +#: ../../source/controller/index.rst:46 +msgid "Thread information" +msgstr "线程信息" + +#: ../../source/controller/index.rst:47 +msgid "Environment variables inspection" +msgstr "环境变量检查" + +#: ../../source/controller/index.rst:50 +msgid "Related Documentation" +msgstr "相关文档" + +#: ../../source/controller/index.rst:52 +msgid ":doc:`../api_reference/configurations` - Complete configuration reference" +msgstr ":doc:`../api_reference/configurations` - 完整的配置参考" + +#: ../../source/controller/index.rst:53 +msgid ":doc:`../kv_cache_management/index` - KV cache management guide" +msgstr ":doc:`../kv_cache_management/index` - KV Cache 管理指南" + +#: ../../source/controller/index.rst:54 +msgid ":doc:`freeze_mode` - Freeze Mode safety mechanism guide" +msgstr ":doc:`freeze_mode` - 冻结模式安全机制指南" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/architecture.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/architecture.po new file mode 100644 index 00000000000..fea64ffb718 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/architecture.po @@ -0,0 +1,260 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/architecture.rst:2 +msgid "Architecture Overview" +msgstr "架构概述" + +#: ../../source/developer_guide/architecture.rst:5 +msgid "High-Level System Architecture" +msgstr "高层系统架构" + +#: ../../source/developer_guide/architecture.rst:8 +msgid "" +"LMCache extends an LLM inference engine (e.g., vLLM) with a multi-tier KV" +" cache storage system spanning GPU memory, CPU memory, and disk/remote " +"backends. The diagram below illustrates how KV cache blocks move across " +"these layers." +msgstr "LMCache 扩展了一个 LLM 推理引擎（例如，vLLM），并配备了一个跨越显存、CPU 内存和磁盘/远程后端的多层 KV Cache 存储系统。下面的图示说明了 KV Cache 块如何在这些层之间移动。" + +#: ../../source/developer_guide/architecture.rst:10 +msgid "**Multi-Tier Storage Architecture**" +msgstr "**多层存储架构**" + +#: ../../source/developer_guide/architecture.rst:12 +msgid "" +"LMCache implements a hierarchical storage system with three distinct " +"tiers:" +msgstr "LMCache 实现了一个具有三个不同层次的分层存储系统：" + +#: ../../source/developer_guide/architecture.rst:14 +msgid "" +"**GPU Memory**: Holds the active working set of KV caches that are " +"currently being used by the model" +msgstr "**显存**：保存当前模型使用的 KV Cache 的活动工作集" + +#: ../../source/developer_guide/architecture.rst:15 +msgid "" +"**CPU DRAM**: Acts as a \"hot cache\" for recently used KV chunks, using " +"pinned memory for efficient GPU-CPU transfers" +msgstr "**CPU DRAM**: 作为最近使用的 KV 块的“热缓存”，使用固定内存以实现高效的 GPU-CPU 传输" + +#: ../../source/developer_guide/architecture.rst:16 +msgid "" +"**Local storage (e.g., local disk, NVMe GDS)**: Provides a large tier for" +" local KV caching (e.g. for long documents)" +msgstr "**本地存储（例如，本地磁盘、NVMe GDS）**：为本地 KV 缓存提供了一个大容量层（例如，用于长文档）。" + +#: ../../source/developer_guide/architecture.rst:17 +msgid "" +"**Remote storage (e.g., Redis, Mooncake, InfiniStore)**: Persistent " +"storage for KV caches. Reliable but not as performant as previous tiers." +msgstr "**远程存储（例如，Redis、Mooncake、InfiniStore）**：KV 缓存的持久存储。可靠，但性能不如前面的层。" + +#: ../../source/developer_guide/architecture.rst:19 +msgid "**Data Flow and Operations**" +msgstr "**数据流和操作**" + +#: ../../source/developer_guide/architecture.rst:21 +msgid "" +"When the model generates new key-value (KV) cache chunks on the GPU, " +"LMCache can:" +msgstr "当模型在 GPU 上生成新的键值 (KV) 缓存块时，LMCache 可以：" + +#: ../../source/developer_guide/architecture.rst:23 +msgid "" +"**Offload overflow KV caches** from GPU to CPU DRAM, freeing precious GPU" +" memory" +msgstr "**将溢出的 KV 缓存** 从 GPU 卸载到 CPU DRAM，释放宝贵的显存" + +#: ../../source/developer_guide/architecture.rst:24 +msgid "" +"**Asynchronously write** KV caches from CPU to disk or remote storage " +"using LRU eviction policies" +msgstr "**异步写入** KV Cache 从 CPU 到磁盘或远程存储，使用 LRU 逐出策略" + +#: ../../source/developer_guide/architecture.rst:25 +msgid "" +"**Prefetch hot KV caches** from disk/remote storage back to CPU when " +"needed" +msgstr "**按需预取热 KV 缓存** 从磁盘/远程存储回到 CPU。" + +#: ../../source/developer_guide/architecture.rst:26 +msgid "**On-demand reuse** of cached segments by moving them from CPU back to GPU" +msgstr "**按需重用** 缓存段，通过将它们从 CPU 移动回 GPU" + +#: ../../source/developer_guide/architecture.rst:28 +msgid "" +"This architecture enables LMCache to significantly reduce prefill delays " +"and GPU memory pressure while maintaining high performance through " +"intelligent cache management." +msgstr "这种架构使 LMCache 能够显著减少 Prefill 延迟和显存压力，同时通过智能缓存管理保持高性能。" + +#: ../../source/developer_guide/architecture.rst:49 +msgid "Two modes" +msgstr "两种模式" + +#: ../../source/developer_guide/architecture.rst:51 +msgid "**Storage Mode (KV cache offloading)**" +msgstr "**存储模式 (KV Cache 卸载)**" + +#: ../../source/developer_guide/architecture.rst:52 +msgid "" +"LMCache acts as a persistent KV store, optimizing for high reuse across " +"queries or sessions. It offloads infrequently used KV blocks from GPU " +"memory and persists popular caches across sessions, boosting cache hit " +"rates for \"hot\" content. KV caches survive beyond single inference " +"calls and even process restarts when backed by disk or external storage." +msgstr "LMCache 充当持久化的 KV 存储，优化查询或会话之间的高重用性。它将不常用的 KV 块从显存中卸载，并在会话之间持久化热门缓存，提高“热”内容的缓存命中率。KV 缓存超越单次推理调用，甚至在磁盘或外部存储支持下，能够在进程重启后继续存在。" + +#: ../../source/developer_guide/architecture.rst:73 +msgid "**Transport Mode (Prefill-decode disaggregation)**" +msgstr "**传输模式（Prefill-解码分离）**" + +#: ../../source/developer_guide/architecture.rst:74 +msgid "" +"Focuses on accelerating distributed inference by routing KV cache data " +"between nodes in real-time. Enables prefill-decode disaggregation where " +"one server computes KV for prompts and delivers them to another server " +"for generation without recomputation. Uses peer-to-peer channels with " +"communication libraries like NIXL for low-latency, high-bandwidth " +"transfers." +msgstr "专注于通过实时路由 KV Cache 数据在节点之间加速分布式推理。实现分离式 Prefill-解码，其中一台服务器为提示计算 KV，并将其传递给另一台服务器进行生成，而无需重计算。使用点对点通道和 NIXL 等通信库进行低延迟、高带宽的传输。" + +#: ../../source/developer_guide/architecture.rst:78 +msgid "Core Components" +msgstr "核心组件" + +#: ../../source/developer_guide/architecture.rst:80 +msgid "**LLM Inference Engine Integration Module (Connector)**" +msgstr "**LLM 推理引擎集成模块 (连接器)**" + +#: ../../source/developer_guide/architecture.rst:81 +msgid "" +"Integrated into the LLM engine (vLLM), the Connector taps into the paged " +"KV memory manager. During prompt processing, it checks if token sequences" +" were seen before:" +msgstr "集成到 LLM 引擎 (vLLM) 中，连接器利用分页 KV 内存管理器。在处理提示时，它检查令牌序列是否之前见过：" + +#: ../../source/developer_guide/architecture.rst:83 +msgid "" +"**Cache hit**: Fetches precomputed KV cache chunks from LMCache, " +"bypassing computation" +msgstr "**缓存命中**：从 LMCache 中获取预计算的 KV Cache 块，绕过计算" + +#: ../../source/developer_guide/architecture.rst:84 +msgid "" +"**Cache miss**: Model computes KV as usual, then Connector hands newly-" +"generated KV data to LMCache for storage" +msgstr "**缓存未命中**：模型按常规计算 KV，然后连接器将新生成的 KV 数据交给 LMCache 进行存储" + +#: ../../source/developer_guide/architecture.rst:86 +msgid "**Cache Index (Token Database)**" +msgstr "**缓存索引（令牌数据库）**" + +#: ../../source/developer_guide/architecture.rst:87 +msgid "" +"Maintains an internal index mapping token sequences to cached KV entries " +"and their locations. Enables cross-request and cross-instance cache " +"lookups with configurable chunking strategy (default 256 tokens) and " +"hashing scheme." +msgstr "维护一个内部索引，将令牌序列映射到缓存的 KV 条目及其位置。支持跨请求和跨实例的缓存查找，具有可配置的块策略（默认 256 个令牌）和哈希方案。" + +#: ../../source/developer_guide/architecture.rst:89 +msgid "**Memory Object & Allocator**" +msgstr "**内存对象与分配器**" + +#: ../../source/developer_guide/architecture.rst:90 +msgid "" +"Manages KV cache entries as MemoryObj instances using a custom memory " +"allocator within LocalCPUBackend. Ensures pinned memory for fast GPU↔CPU " +"transfers, NUMA-aware allocation, and interfaces with eviction policies " +"(LRU by default)." +msgstr "在 LocalCPUBackend 中使用自定义内存分配器将 KV Cache 条目管理为 MemoryObj 实例。确保固定内存以实现快速的 GPU↔CPU 传输，支持 NUMA 感知分配，并与逐出策略（默认使用 LRU）接口。" + +#: ../../source/developer_guide/architecture.rst:92 +msgid "**Asynchronous Offloading**" +msgstr "**异步卸载**" + +#: ../../source/developer_guide/architecture.rst:93 +msgid "" +"Offloading / loading the KV cache chunks in an asynchronous manner to " +"avoid blocking inference threads and GPU cycles." +msgstr "以异步方式卸载/加载 KV Cache 块，以避免阻塞推理线程和 GPU 周期。" + +#: ../../source/developer_guide/architecture.rst:95 +msgid "**Remote Connectors**" +msgstr "**远程连接器**" + +#: ../../source/developer_guide/architecture.rst:96 +msgid "" +"Plugin-based system for remote backends (Redis, Mooncake, NiXL). Uses " +"generic RemoteBackend wrapper that delegates operations to connector " +"implementations, supporting dynamic loading of custom backends." +msgstr "基于插件的远程后端系统（Redis、Mooncake、NiXL）。使用通用的 RemoteBackend 包装器，将操作委托给连接器实现，支持动态加载自定义后端。" + +#: ../../source/developer_guide/architecture.rst:99 +msgid "LMCache Controller" +msgstr "LMCache 控制器" + +#: ../../source/developer_guide/architecture.rst:101 +msgid "The Controller provides a management API for runtime cache operations:" +msgstr "控制器提供了一个用于运行时缓存操作的管理 API：" + +#: ../../source/developer_guide/architecture.rst:103 +msgid "" +"**Lookup**: Query cache entries for given token sequences and their " +"locations" +msgstr "**查找**：查询给定令牌序列及其位置的缓存条目" + +#: ../../source/developer_guide/architecture.rst:104 +msgid "**Clear**: Purge KV cache entirely or for specific entries" +msgstr "**清除**: 完全清除 KV Cache 或特定条目" + +#: ../../source/developer_guide/architecture.rst:105 +msgid "" +"**Compress/Decompress**: On-demand compression using CacheGen or " +"decompression to full precision" +msgstr "**压缩/解压缩**：按需使用 CacheGen 进行压缩或解压缩到全精度" + +#: ../../source/developer_guide/architecture.rst:106 +msgid "" +"**Move**: Migrate caches to specified locations for cache warming or " +"optimization" +msgstr "**移动**：将缓存迁移到指定位置以进行缓存预热或优化" + +#: ../../source/developer_guide/architecture.rst:107 +msgid "**Pin/Unpin**: Mark cache entries as persistent to prevent eviction" +msgstr "**固定/解固定**：将缓存条目标记为持久，以防止逐出" + +#: ../../source/developer_guide/architecture.rst:108 +msgid "" +"**Health & Finish Checks**: Report worker health and confirm completion " +"of async operations" +msgstr "**健康与完成检查**：报告工作者健康状态并确认异步操作的完成" + +#: ../../source/developer_guide/architecture.rst:110 +msgid "" +"The Controller coordinates with all LMCache workers in the system, " +"providing centralized management for both single-instance and distributed" +" deployments." +msgstr "控制器与系统中的所有 LMCache 工作者协调，提供单实例和分布式部署的集中管理。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/cli.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/cli.po new file mode 100644 index 00000000000..af41611b65c --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/cli.po @@ -0,0 +1,116 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/cli.rst:2 +msgid "Extending the CLI" +msgstr "扩展 CLI" + +#: ../../source/developer_guide/cli.rst:4 +msgid "This guide explains how to add new subcommands to the ``lmcache`` CLI." +msgstr "本指南解释了如何向 ``lmcache`` CLI 添加新的子命令。" + +#: ../../source/developer_guide/cli.rst:7 +msgid "Architecture Overview" +msgstr "架构概述" + +#: ../../source/developer_guide/cli.rst:9 +msgid "The CLI uses explicit command registration:" +msgstr "CLI 使用显式命令注册：" + +#: ../../source/developer_guide/cli.rst:11 +msgid "" +"Each command is a class inheriting from ``BaseCommand`` in " +"``lmcache/cli/commands/base.py``." +msgstr "每个命令都是一个继承自 ``BaseCommand`` 的类，位于 ``lmcache/cli/commands/base.py`` 中。" + +#: ../../source/developer_guide/cli.rst:13 +msgid "" +"Commands are instantiated and listed in ``ALL_COMMANDS`` in " +"``lmcache/cli/commands/__init__.py``." +msgstr "命令在 ``lmcache/cli/commands/__init__.py`` 中的 ``ALL_COMMANDS`` 中被实例化和列出。" + +#: ../../source/developer_guide/cli.rst:15 +msgid "" +"At startup, ``main.py`` iterates ``ALL_COMMANDS`` and calls " +"``cmd.register(subparsers)`` to wire up argparse." +msgstr "在启动时，``main.py`` 迭代 ``ALL_COMMANDS`` 并调用 ``cmd.register(subparsers)`` 来连接 argparse。" + +#: ../../source/developer_guide/cli.rst:18 +msgid "" +"``BaseCommand`` is an abstract class with four required methods. " +"Forgetting any of them raises ``TypeError`` at instantiation time." +msgstr "``BaseCommand`` 是一个抽象类，具有四个必需的方法。忘记其中任何一个都会在实例化时引发 ``TypeError``。" + +#: ../../source/developer_guide/cli.rst:22 +msgid "File Layout" +msgstr "文件布局" + +#: ../../source/developer_guide/cli.rst:49 +msgid "Step-by-Step: Adding a New Command" +msgstr "逐步指南：添加新命令" + +#: ../../source/developer_guide/cli.rst:51 +msgid "**Step 1.** Create ``lmcache/cli/commands/describe.py``:" +msgstr "**第 1 步。** 创建 ``lmcache/cli/commands/describe.py``：" + +#: ../../source/developer_guide/cli.rst:79 +msgid "**Step 2.** Register it in ``lmcache/cli/commands/__init__.py``:" +msgstr "**步骤 2.** 在 ``lmcache/cli/commands/__init__.py`` 中注册它：" + +#: ../../source/developer_guide/cli.rst:94 +msgid "" +"That's it --- ``lmcache describe --url http://localhost:8000`` is now " +"available." +msgstr "就是这样 --- ``lmcache describe --url http://localhost:8000`` 现在可以使用了。" + +#: ../../source/developer_guide/cli.rst:98 +msgid "Using the Metrics System" +msgstr "使用指标系统" + +#: ../../source/developer_guide/cli.rst:100 +msgid "The metrics system uses a **handler + formatter** architecture:" +msgstr "指标系统使用 **处理器 + 格式化器** 架构：" + +#: ../../source/developer_guide/cli.rst:102 +msgid "**Metrics** — the collector. Holds sections and entries." +msgstr "**指标** — 收集器。包含部分和条目。" + +#: ../../source/developer_guide/cli.rst:103 +msgid "**Handler** — the destination (stdout, file, etc.)." +msgstr "**处理器** — 目标（标准输出、文件等）。" + +#: ../../source/developer_guide/cli.rst:104 +msgid "**Formatter** — the rendering (ASCII table, JSON, etc.)." +msgstr "**格式化器** — 渲染（ASCII 表格、JSON 等）。" + +#: ../../source/developer_guide/cli.rst:106 +msgid "" +"``BaseCommand.create_metrics()`` sets up default handlers automatically, " +"so command authors just build metrics and call ``emit()``:" +msgstr "``BaseCommand.create_metrics()`` 自动设置默认处理程序，因此命令作者只需构建指标并调用 ``emit()``：" + +#: ../../source/developer_guide/cli.rst:128 +msgid "" +"The ``--format`` and ``--output`` flags are added automatically by " +"``BaseCommand.register()`` — subcommands do not need to add them " +"manually." +msgstr "``--format`` 和 ``--output`` 标志由 ``BaseCommand.register()`` 自动添加 — 子命令不需要手动添加它们。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/contributing.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/contributing.po new file mode 100644 index 00000000000..8dff10e7250 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/contributing.po @@ -0,0 +1,497 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/contributing.rst:2 +msgid "Contributing Guide" +msgstr "贡献指南" + +#: ../../source/developer_guide/contributing.rst:4 +msgid "" +"Thank you for your interest in contributing to LMCache! We welcome and " +"accept all kinds of contributions, no matter how small or large. There " +"are several ways you can contribute to the project:" +msgstr "感谢您对为 LMCache 贡献的兴趣！我们欢迎并接受各种形式的贡献，无论大小。您可以通过几种方式为项目做出贡献：" + +#: ../../source/developer_guide/contributing.rst:6 +msgid "Identify and report any issues or bugs" +msgstr "识别并报告任何问题或错误" + +#: ../../source/developer_guide/contributing.rst:7 +msgid "Request or add support for a new model" +msgstr "请求或添加对新模型的支持" + +#: ../../source/developer_guide/contributing.rst:8 +msgid "Suggest or implement new features" +msgstr "建议或实现新功能" + +#: ../../source/developer_guide/contributing.rst:9 +msgid "Improve documentation or contribute a how-to guide" +msgstr "改进文档或贡献一份操作指南" + +#: ../../source/developer_guide/contributing.rst:11 +msgid "" +"A comprehensive list of good first issues can be found in the issue " +"`[Onboarding][Q4] Welcoming contributors with good first issues! " +"`_." +msgstr "可以在问题 `[Onboarding][Q4] 欢迎贡献者的好第一问题！ `_ 中找到一个全面的好第一问题列表。" + +#: ../../source/developer_guide/contributing.rst:13 +msgid "" +"If you'd like to support our community further, then answering queries, " +"offering PR reviews, and assisting others are also impactful ways to " +"contribute and take LMCache further." +msgstr "如果您想进一步支持我们的社区，回答问题、提供 PR 评审和协助他人也是对 LMCache 产生影响的贡献方式。" + +#: ../../source/developer_guide/contributing.rst:15 +msgid "" +"Finally, you can support us by raising awareness about LMCache. Feel free" +" to share our blog posts, check out our handle on X at `LMCache " +"`_ and see the latest of what we are up to. If " +"using LMCache helped your project or product in any way, you can simply " +"offer your appreciation by starring our repository!" +msgstr "最后，您可以通过提高对 LMCache 的认识来支持我们。欢迎分享我们的博客文章，查看我们在 X 上的账号 `LMCache `_，了解我们最新的动态。如果使用 LMCache 在某种程度上帮助了您的项目或产品，您可以通过给我们的代码库加星来表达您的感谢！" + +#: ../../source/developer_guide/contributing.rst:18 +msgid "License" +msgstr "许可证" + +#: ../../source/developer_guide/contributing.rst:20 +msgid "" +"See the `LICENSE `_ " +"file for details." +msgstr "请参阅 `LICENSE `_ 文件以获取详细信息。" + +#: ../../source/developer_guide/contributing.rst:23 +msgid "Code of Conduct" +msgstr "行为准则" + +#: ../../source/developer_guide/contributing.rst:25 +msgid "" +"This project adheres to the `Code of Conduct " +"`_. By " +"participating, you are expected to uphold this code." +msgstr "本项目遵循 `行为准则 `_。参与时，您需要遵守此准则。" + +#: ../../source/developer_guide/contributing.rst:28 +msgid "Contribution Guidelines" +msgstr "贡献指南" + +#: ../../source/developer_guide/contributing.rst:30 +msgid "" +"Help on open source projects is always welcome and there is always " +"something that can be improved. For example, documentation (like the text" +" you are reading now) can always use improvement, code can always be " +"clarified, variables or functions can always be renamed or commented on, " +"and there is always a need for more test coverage. If you see something " +"that you think should be fixed, take ownership! Here is how you get " +"started." +msgstr "对开源项目的帮助总是受欢迎的，总有一些可以改进的地方。例如，文档（就像您现在正在阅读的文本）总是可以改进，代码总是可以更清晰，变量或函数总是可以重命名或添加注释，并且总是需要更多的测试覆盖率。如果您看到认为应该修复的内容，请主动承担责任！以下是您如何开始的指南。" + +#: ../../source/developer_guide/contributing.rst:33 +msgid "How Can I Contribute?" +msgstr "我该如何贡献？" + +#: ../../source/developer_guide/contributing.rst:35 +msgid "" +"When contributing, it's useful to start by looking at `issues " +"`_. After picking up an issue," +" writing code, or updating a document, make a pull request and your work " +"will be reviewed and merged. If you're adding a new feature or find a " +"bug, it's best to `write an issue " +"`_ first to discuss it " +"with maintainers." +msgstr "在贡献时，首先查看 `issues `_ 是很有帮助的。在选择一个问题后，编写代码或更新文档，然后提交一个拉取请求，你的工作将会被审查和合并。如果你要添加新功能或发现错误，最好先 `写一个问题 `_ 与维护者讨论。" + +#: ../../source/developer_guide/contributing.rst:37 +msgid "" +"If you discover a security vulnerability, please follow the instructions " +"in the `Security doc " +"`_." +msgstr "如果您发现安全漏洞，请按照 `安全文档 `_ 中的说明进行操作。" + +#: ../../source/developer_guide/contributing.rst:39 +msgid "" +"To contribute to this repo, you'll use the Fork and Pull model common in " +"many open source repositories. For details on this process, check out " +"`The GitHub Workflow Guide " +"`_ from Kubernetes. In short:" +msgstr "要为这个仓库贡献代码，您将使用许多开源仓库中常见的分叉和拉取模型。有关此过程的详细信息，请查看 Kubernetes 的 `GitHub 工作流程指南 `_。简而言之：" + +#: ../../source/developer_guide/contributing.rst:41 +msgid "Fork the repository" +msgstr "分叉仓库" + +#: ../../source/developer_guide/contributing.rst:42 +msgid "Create a branch" +msgstr "创建一个分支" + +#: ../../source/developer_guide/contributing.rst:43 +msgid "Run code style checks and fix any issues" +msgstr "运行代码风格检查并修复任何问题" + +#: ../../source/developer_guide/contributing.rst:44 +msgid "Run unit tests and fix any broken tests" +msgstr "运行单元测试并修复任何损坏的测试" + +#: ../../source/developer_guide/contributing.rst:45 +msgid "Submit a pull request with detailed descriptions" +msgstr "提交包含详细描述的拉取请求" + +#: ../../source/developer_guide/contributing.rst:47 +msgid "" +"When your contribution is ready, you can create a pull request. Pull " +"requests are often referred to as \"PRs\". In general, we follow the " +"standard `GitHub pull request `_ process. Follow the template to provide details about " +"your pull request to the maintainers." +msgstr "当您的贡献准备好后，您可以创建一个拉取请求。拉取请求通常被称为“PR”。一般来说，我们遵循标准的 `GitHub 拉取请求 `_ 流程。请按照模板提供有关您的拉取请求的详细信息给维护者。" + +#: ../../source/developer_guide/contributing.rst:49 +msgid "" +"Please try to classify PRs for easy understanding of the type of changes." +" The PR title is prefixed appropriately to indicate the type of change. " +"Please use one of the following:" +msgstr "请尽量对 PR 进行分类，以便于理解更改类型。PR 标题前缀应适当地指示更改类型。请使用以下之一：" + +#: ../../source/developer_guide/contributing.rst:51 +msgid "[Bugfix] for bug fixes" +msgstr "[Bugfix] 用于错误修复" + +#: ../../source/developer_guide/contributing.rst:52 +msgid "[Build] for build fixes and improvements" +msgstr "[构建] 用于构建修复和改进" + +#: ../../source/developer_guide/contributing.rst:53 +msgid "[CI] for continuous integration fixes and iimprovements" +msgstr "[CI] 用于持续集成的修复和改进" + +#: ../../source/developer_guide/contributing.rst:54 +msgid "" +"[Core] for changes in the core LMCache logic (e.g., ``LMCacheEngine``, " +"``Backend`` etc.)" +msgstr "[核心] 用于核心 LMCache 逻辑的更改（例如，``LMCacheEngine``，``Backend`` 等）" + +#: ../../source/developer_guide/contributing.rst:55 +msgid "[Doc] for documentation fixes and improvements" +msgstr "[文档] 用于文档修复和改进" + +#: ../../source/developer_guide/contributing.rst:56 +msgid "" +"[Misc] for PRs that do not fit the above categories. Please use this " +"sparingly" +msgstr "[杂项] 用于不符合上述类别的 PR。请谨慎使用。" + +#: ../../source/developer_guide/contributing.rst:57 +msgid "" +"[Model] for adding a new model or improving an existing model. Model name" +" should appear in the title" +msgstr "[模型] 用于添加新模型或改进现有模型。模型名称应出现在标题中" + +#: ../../source/developer_guide/contributing.rst:58 +msgid "[Test] for unit tests" +msgstr "[Test] 用于单元测试" + +#: ../../source/developer_guide/contributing.rst:62 +msgid "" +"If the PR spans more than one category, please include all relevant " +"prefixes" +msgstr "如果 PR 涉及多个类别，请包含所有相关的前缀。" + +#: ../../source/developer_guide/contributing.rst:64 +msgid "" +"It's best to break your contribution into smaller PRs with incremental " +"changes, and include a good description of the changes. We require new " +"unit tests to be contributed with any new functionality added and docs if" +" user facing changes." +msgstr "最好将您的贡献分解为较小的 PR，进行增量更改，并包含对更改的良好描述。我们要求在添加任何新功能时提供新的单元测试，并在用户可见的更改时提供文档。" + +#: ../../source/developer_guide/contributing.rst:66 +msgid "" +"Before sending pull requests, make sure your changes pass code quality " +"checks and unit tests. These checks will run when the pull request " +"builds. Alternatively, you can run the checks manually on your local " +"machine `as specified in Development <#development>`_ ." +msgstr "在发送拉取请求之前，请确保您的更改通过了代码质量检查和单元测试。这些检查将在拉取请求构建时运行。或者，您可以按照 `开发 <#development>`_ 中的说明在本地机器上手动运行检查。" + +#: ../../source/developer_guide/contributing.rst:69 +msgid "DCO and Signed-off-by" +msgstr "DCO 和签署确认" + +#: ../../source/developer_guide/contributing.rst:71 +msgid "" +"When contributing changes to the project, you must agree to the `DCO " +"`_. Commits must include" +" a :code:`Signed-off-by` header which certifies agreement with the terms " +"of the `DCO `_." +msgstr "在向项目贡献更改时，您必须同意 `DCO `_。提交必须包含一个 :code:`Signed-off-by` 头部，以证明您同意 `DCO `_ 的条款。" + +#: ../../source/developer_guide/contributing.rst:75 +msgid "" +"Using :code:`-s` or :code:`--signoff` flag with :code:`git commit` will " +"automatically add this header. The flags can also be used with " +":code:`--amend` if you forget to sign your last commit." +msgstr "使用 :code:`-s` 或 :code:`--signoff` 标志与 :code:`git commit` 一起将自动添加此标题。如果您忘记为上一个提交签名，这些标志也可以与 :code:`--amend` 一起使用。" + +#: ../../source/developer_guide/contributing.rst:78 +msgid "Code Review" +msgstr "代码审查" + +#: ../../source/developer_guide/contributing.rst:80 +msgid "" +"Once you've created a pull request, maintainers will review your code and" +" may make suggestions to fix before merging. It will be easier for your " +"pull request to receive reviews if you consider the criteria the " +"reviewers follow while working. Remember to:" +msgstr "一旦您创建了拉取请求，维护者将审查您的代码，并可能提出建议以在合并之前进行修复。如果您在工作时考虑审查者遵循的标准，您的拉取请求将更容易获得审查。请记住：" + +#: ../../source/developer_guide/contributing.rst:82 +msgid "Document the code well to help future contributors and maintainers" +msgstr "对代码进行良好的文档记录，以帮助未来的贡献者和维护者。" + +#: ../../source/developer_guide/contributing.rst:83 +msgid "Follow the project coding conventions for consistency and best practices" +msgstr "遵循项目编码规范，以确保一致性和最佳实践" + +#: ../../source/developer_guide/contributing.rst:84 +msgid "Include sufficient tests to maintain robustness of the code" +msgstr "包含足够的测试以保持代码的健壮性" + +#: ../../source/developer_guide/contributing.rst:85 +msgid "" +"Add or update documentation in :code:`/docs` if PR modifies user facing " +"behavior to help people using LMCache" +msgstr "如果 PR 修改了面向用户的行为，请在 :code:`/docs` 中添加或更新文档，以帮助使用 LMCache 的人。" + +#: ../../source/developer_guide/contributing.rst:86 +msgid "Run coding style checks to ensure consistency and correctness" +msgstr "运行编码风格检查以确保一致性和正确性" + +#: ../../source/developer_guide/contributing.rst:87 +msgid "" +"Run tests locally and ensure they pass and don't break the existing code " +"base" +msgstr "在本地运行测试，确保它们通过并且不会破坏现有代码库。" + +#: ../../source/developer_guide/contributing.rst:88 +msgid "Write detailed commit messages to help future contributors and maintainers" +msgstr "撰写详细的提交信息，以帮助未来的贡献者和维护者。" + +#: ../../source/developer_guide/contributing.rst:89 +msgid "" +"Break large changes into a logical series of smaller patches, which are " +"easy to understand individually and combine to solve a broader issue" +msgstr "将大型更改拆分为一系列逻辑上较小的补丁，这些补丁单独易于理解，并结合起来解决更广泛的问题。" + +#: ../../source/developer_guide/contributing.rst:93 +msgid "" +"Maintainers will perform \"squash and merge\" actions on PRs in this " +"repo, so it doesn't matter how many commits your PR has, as they will end" +" up being a single commit after merging." +msgstr "维护者将在此仓库的 PR 上执行“压缩并合并”操作，因此您的 PR 有多少个提交并不重要，因为它们在合并后将变成一个单一的提交。" + +#: ../../source/developer_guide/contributing.rst:96 +msgid "Development" +msgstr "开发" + +#: ../../source/developer_guide/contributing.rst:99 +msgid "Set up your dev environment" +msgstr "设置您的开发环境" + +#: ../../source/developer_guide/contributing.rst:101 +msgid "The following prerequisites are required:" +msgstr "以下先决条件是必需的：" + +#: ../../source/developer_guide/contributing.rst:103 +msgid "OS: Linux" +msgstr "操作系统：Linux" + +#: ../../source/developer_guide/contributing.rst:104 +msgid "" +"GPU: NVIDIA compute capability 7.0+ (e.g., V100, T4, RTX20xx, A100, L4, " +"H100, etc.)" +msgstr "GPU: NVIDIA 计算能力 7.0+（例如，V100、T4、RTX20xx、A100、L4、H100 等）" + +#: ../../source/developer_guide/contributing.rst:105 +msgid "CUDA 12.8+" +msgstr "CUDA 12.8+" + +#: ../../source/developer_guide/contributing.rst:107 +msgid "The following tools are required:" +msgstr "以下工具是必需的：" + +#: ../../source/developer_guide/contributing.rst:109 +msgid "`git `_" +msgstr "`git `_" + +#: ../../source/developer_guide/contributing.rst:110 +msgid "`python `_ (v3.10 -- v3.13)" +msgstr "`python `_ (v3.10 -- v3.13)" + +#: ../../source/developer_guide/contributing.rst:111 +msgid "`pip `_ (v23.0+)" +msgstr "`pip `_ (v23.0+)" + +#: ../../source/developer_guide/contributing.rst:113 +msgid "" +"The first step is to install the necessary Python packages required for " +"development. The commands to do this are as follows:" +msgstr "第一步是安装开发所需的必要 Python 包。执行此操作的命令如下：" + +#: ../../source/developer_guide/contributing.rst:123 +msgid "" +"Before pushing changes to GitHub, you need to run the coding style checks" +" and unit tests as shown below." +msgstr "在将更改推送到 GitHub 之前，您需要运行如下所示的编码风格检查和单元测试。" + +#: ../../source/developer_guide/contributing.rst:126 +msgid "Coding style" +msgstr "编码风格" + +#: ../../source/developer_guide/contributing.rst:128 +msgid "" +"LMCache follows the Python `pep8 `_ " +"coding style for Python and `Google C++ style guide " +"`_ for C++. We use the" +" following tools to enforce the coding style:" +msgstr "LMCache 遵循 Python 的 `pep8 `_ 编码风格和 C++ 的 `Google C++ style guide `_。我们使用以下工具来强制执行编码风格：" + +#: ../../source/developer_guide/contributing.rst:130 +msgid "" +"Python linting and formatting: `Ruff `_, " +"and `isort `_" +msgstr "Python 代码检查和格式化： `Ruff `_ 和 `isort `_" + +#: ../../source/developer_guide/contributing.rst:131 +msgid "Python static code checking: `mypy `_" +msgstr "Python 静态代码检查: `mypy `_" + +#: ../../source/developer_guide/contributing.rst:132 +msgid "" +"Spell checking: `codespell `_" +msgstr "拼写检查: `codespell `_" + +#: ../../source/developer_guide/contributing.rst:133 +msgid "" +"C++ formatting: `clang-format " +"`_" +msgstr "C++ 格式化: `clang-format `_" + +#: ../../source/developer_guide/contributing.rst:135 +msgid "" +"The tools are managed by `pre-commit `_. It is " +"installed as follows:" +msgstr "这些工具由 `pre-commit `_ 管理。安装方法如下：" + +#: ../../source/developer_guide/contributing.rst:142 +msgid "" +"It will run automatically when you add a commit. You can also run it " +"manually on all files with the following command:" +msgstr "它将在您添加提交时自动运行。您也可以使用以下命令手动在所有文件上运行它：" + +#: ../../source/developer_guide/contributing.rst:150 +msgid "" +"For all new code added, please write docstrings in `sphinx-doc format " +"`_." +msgstr "对于所有新增的代码，请使用 `sphinx-doc 格式 `_ 编写文档字符串。" + +#: ../../source/developer_guide/contributing.rst:153 +msgid "Unit tests" +msgstr "单元测试" + +#: ../../source/developer_guide/contributing.rst:155 +msgid "" +"When making changes, run the tests before pushing the changes. Running " +"unit tests ensures your contributions do not break existing code. We use " +"the `pytest `_ framework to run unit tests. The" +" framework is setup to run all files in the `tests " +"`_ directory which " +"have a prefix or posfix of \"test\"." +msgstr "在进行更改时，请在推送更改之前运行测试。运行单元测试可以确保您的贡献不会破坏现有代码。我们使用 `pytest `_ 框架来运行单元测试。该框架已设置为运行 `tests `_ 目录中所有以“test”开头或结尾的文件。" + +#: ../../source/developer_guide/contributing.rst:157 +msgid "Running unit tests is as simple as:" +msgstr "运行单元测试非常简单：" + +#: ../../source/developer_guide/contributing.rst:165 +msgid "" +"``vLLM`` (``pip install vllm``) and the dependencies in " +"``requirements/test.txt`` need to be installed prior to running unit " +"tests." +msgstr "在运行单元测试之前，需要安装 ``vLLM`` (``pip install vllm``) 及 ``requirements/test.txt`` 中的依赖项。" + +#: ../../source/developer_guide/contributing.rst:167 +msgid "" +"By default, all tests found within the tests directory are run. However, " +"specific unit tests can run by passing filenames, classes and/or methods " +"to `pytest`. The following example invokes a single test method " +"\"test_lm_connector\" that is declared in the \"tests/test_connector.py\"" +" file:" +msgstr "默认情况下，将运行测试目录中找到的所有测试。然而，可以通过将文件名、类和/或方法传递给 `pytest` 来运行特定的单元测试。以下示例调用在 \\\"tests/test_connector.py\\\" 文件中声明的单个测试方法 \\\"test_lm_connector\\\"：" + +#: ../../source/developer_guide/contributing.rst:175 +msgid "" +"Some unit tests require a NVIDIA GPU. This means that on a non Linux " +"NVIDIA GPU system, the full suite of tests will not be run (tests " +"requiring CUDA will be skipped). The Buildkite continuous integration " +"(CI) system executes a full run of all the tests." +msgstr "某些单元测试需要 NVIDIA GPU。这意味着在非 Linux NVIDIA GPU 系统上，所有测试将不会全部运行（需要 CUDA 的测试将被跳过）。Buildkite 持续集成（CI）系统会执行所有测试的完整运行。" + +#: ../../source/developer_guide/contributing.rst:179 +msgid "" +"The `NVIDIA Inference Xfer Library (NIXL) `_ unit tests require NIXL to be to be installed. This is not" +" installed by LMCache by and therefore requires you to install it " +"separately. Please follow the details in the NIXL GitHub repo to install." +" If the NIXL package is not installed, the NIXL unit tests are skipped." +msgstr "`NVIDIA Inference Xfer Library (NIXL) `_ 单元测试需要安装 NIXL。LMCache 不会安装它，因此需要您单独安装。请按照 NIXL GitHub 仓库中的详细信息进行安装。如果未安装 NIXL 包，则会跳过 NIXL 单元测试。" + +#: ../../source/developer_guide/contributing.rst:183 +msgid "Building the docs" +msgstr "构建文档" + +#: ../../source/developer_guide/contributing.rst:185 +msgid "Install the dependencies:" +msgstr "安装依赖项：" + +#: ../../source/developer_guide/contributing.rst:191 +msgid "" +"After that, you can build the docs (from :code:`docs/` directory) using " +"`make`:" +msgstr "之后，您可以使用 `make` 从 :code:`docs/` 目录构建文档：" + +#: ../../source/developer_guide/contributing.rst:198 +msgid "" +"Serve docs page locally at http://localhost:8000: :code:`python -m " +"http.server -d build/html/`" +msgstr "在 http://localhost:8000 本地服务文档页面： :code:`python -m http.server -d build/html/`" + +#: ../../source/developer_guide/contributing.rst:201 +msgid "Thank You" +msgstr "谢谢你" + +#: ../../source/developer_guide/contributing.rst:203 +msgid "" +"Thank you for your contribution to LMCache and making it a better, and " +"accessible framework for all." +msgstr "感谢您为 LMCache 的贡献，使其成为一个更好、更易于访问的框架。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/docker_file.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/docker_file.po new file mode 100644 index 00000000000..4e97f3abc23 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/docker_file.po @@ -0,0 +1,58 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/docker_file.rst:2 +msgid "Dockerfile" +msgstr "Dockerfile" + +#: ../../source/developer_guide/docker_file.rst:4 +msgid "" +"We provide a Dockerfile to help you build a container image for LMCache " +"integrated with vLLM. More information about deploying LMCache image " +"using Docker can be found here - :ref:`Docker deployment guide " +"`." +msgstr "我们提供了一个 Dockerfile，帮助您构建一个与 vLLM 集成的 LMCache 容器镜像。有关使用 Docker 部署 LMCache 镜像的更多信息，请参见这里 - :ref:`Docker deployment guide `。" + +#: ../../source/developer_guide/docker_file.rst:8 +msgid "Building the container image" +msgstr "构建容器镜像" + +#: ../../source/developer_guide/docker_file.rst:10 +msgid "" +"You can build the LMCache (integrated with vLLM) image using Docker from " +"source via the provided Dockerfile. The Dockerfile is located at `docker " +"`_." +msgstr "您可以通过提供的 Dockerfile 使用 Docker 从源代码构建集成了 vLLM 的 LMCache 镜像。Dockerfile 位于 `docker `_。" + +#: ../../source/developer_guide/docker_file.rst:13 +msgid "" +"To build the container image, run the following command from the root " +"directory of the LMCache repository:" +msgstr "要构建容器镜像，请在 LMCache 仓库的根目录下运行以下命令：" + +#: ../../source/developer_guide/docker_file.rst:19 +msgid "" +"Replace `` and `` with your desired image name and tag. " +"See example build file in `docker " +"`_ for explanation of" +" all arguments." +msgstr "将 `` 和 `` 替换为您想要的镜像名称和标签。有关所有参数的说明，请参见 `docker `_ 中的示例构建文件。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/index.po new file mode 100644 index 00000000000..ddcf83f11aa --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/index.po @@ -0,0 +1,267 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/extending_lmcache/index.rst:2 +#: ../../source/developer_guide/extending_lmcache/index.rst:121 +msgid "Extending LMCache" +msgstr "扩展 LMCache" + +#: ../../source/developer_guide/extending_lmcache/index.rst:4 +msgid "" +"LMCache is designed to be extensible, allowing integration of custom " +"functionality without modifying the core. The main extension mechanisms " +"are:" +msgstr "LMCache 旨在可扩展，允许在不修改核心的情况下集成自定义功能。主要的扩展机制包括：" + +#: ../../source/developer_guide/extending_lmcache/index.rst:6 +msgid "" +"**Native Connector Framework** – build high-performance C++ storage " +"connectors with pybind11 that work in both non-MP and MP modes. Provides " +"multi-threaded I/O, batched tiling, and eventfd-based completions." +msgstr "**原生连接器框架** – 使用 pybind11 构建高性能 C++ 存储连接器，支持非多进程和多进程模式。提供多线程 I/O、批量切片和基于 eventfd 的完成。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:7 +msgid "" +"**Storage Plugin Framework** – integrate new storage backends (custom " +"cache storage modules) via a standardized interface." +msgstr "**存储插件框架** – 通过标准化接口集成新的存储后端（自定义缓存存储模块）。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:8 +msgid "" +"**External Remote Connector Framework** – integrate new remote KV store " +"connectors for external/distributed storage systems. This is still " +"supported but has a planned deprecation by v0.5.0. It is replaced by " +"**Remote Storage Plugin Framework**." +msgstr "**外部远程连接器框架** – 通过标准化接口集成新的远程 KV 存储连接器，用于外部/分布式存储系统。该功能仍然支持，但计划在 v0.5.0 中弃用。它被 **远程存储插件框架** 替代。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:9 +msgid "" +"**Remote Storage Plugin Framework** – integrate new remote storage " +"connectors for remote/distributed KV storage systems." +msgstr "**远程存储插件框架** – 集成新的远程存储连接器，用于远程/分布式 KV 存储系统。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:10 +msgid "" +"**Runtime Plugin Framework** – run custom scripts as separate processes " +"alongside LMCache for added functionality." +msgstr "**运行时插件框架** – 作为独立进程运行自定义脚本，以增强 LMCache 的功能。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:40 +msgid "" +"**Storage Plugin Framework** enable LMCache to interface with new " +"storage or transport systems. Developers can implement the standardized " +"``StoragePluginInterface`` to create a custom storage backend module. " +"Such backends are loaded by LMCache at runtime (via configuration) in " +"addition to the built-in backends that ship with LMCache. Built-in " +"backends include in-memory CPU caching, local disk storage, NVIDIA NIXL " +"(GPU peer-to-peer), and remote stores like Redis, InfiniStore, Mooncake, " +"etc." +msgstr "**存储插件框架** 使 LMCache 能够与新的存储或传输系统接口。开发者可以实现标准化的 ``StoragePluginInterface`` 来创建自定义存储后端模块。这些后端在运行时由 LMCache 加载（通过配置），除了与 LMCache 一起提供的内置后端。内置后端包括内存 CPU 缓存、本地磁盘存储、NVIDIA NIXL（GPU 点对点）以及 Redis、InfiniStore、Mooncake 等远程存储。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:42 +msgid "To add a custom storage backend, you need to:" +msgstr "要添加自定义存储后端，您需要：" + +#: ../../source/developer_guide/extending_lmcache/index.rst:44 +msgid "" +"**Implement** a Python class inheriting from the LMCache " +"``StoragePluginInterface``, overriding all required methods." +msgstr "**实现**一个继承自 LMCache ``StoragePluginInterface`` 的 Python 类，重写所有必需的方法。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:45 +msgid "" +"**Install** this backend package in the LMCache environment (so that " +"LMCache can import it)." +msgstr "**在** LMCache 环境中安装此后端包（以便 LMCache 可以导入它）。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:46 +msgid "" +"**Configure** LMCache to use it by adding an entry to the " +"`storage_plugins` list and specifying the module path and class name in " +"the configuration’s `extra_config`. For example:" +msgstr "**配置** LMCache 以使用它，通过在 `storage_plugins` 列表中添加一个条目，并在配置的 `extra_config` 中指定模块路径和类名。例如：" + +#: ../../source/developer_guide/extending_lmcache/index.rst:55 +msgid "" +"Multiple storage backends can be enabled simultaneously. They are " +"initialized during LMCache startup. Note that the order of backends can " +"matter: if multiple backends are used, earlier listed backends have " +"higher priority for cache lookups (i.e. LMCache will check those backends" +" first when retrieving KV entries)." +msgstr "可以同时启用多个存储后端。它们在 LMCache 启动时初始化。请注意，后端的顺序可能很重要：如果使用多个后端，较早列出的后端在缓存查找中具有更高的优先级（即 LMCache 在检索 KV 条目时会首先检查这些后端）。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:57 +msgid "" +"**Remote Storage Plugin Framework** allows LMCache's storage layer to " +"connect to new remote KV storage systems. The LMCache `RemoteBackend` " +"uses connector implementations to communicate with various remote stores." +" Built-in connectors exist for Redis, InfiniStore, MooncakeStore, S3, and" +" more. The plugin system provides the ability to add custom remote " +"storage connectors through dynamic loading, extending remote storage " +"capabilities without modifying core code." +msgstr "**远程存储插件框架** 允许 LMCache 的存储层连接到新的远程 KV 存储系统。LMCache 的 `RemoteBackend` 使用连接器实现与各种远程存储进行通信。内置的连接器支持 Redis、InfiniStore、MooncakeStore、S3 等。插件系统提供了通过动态加载添加自定义远程存储连接器的能力，从而扩展远程存储功能，而无需修改核心代码。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:59 +msgid "To add a custom remote storage connector, you need to:" +msgstr "要添加自定义远程存储连接器，您需要：" + +#: ../../source/developer_guide/extending_lmcache/index.rst:61 +msgid "**Implement** two classes:" +msgstr "**实现** 两个类：" + +#: ../../source/developer_guide/extending_lmcache/index.rst:63 +msgid "" +"A ``ConnectorAdapter`` subclass that handles URL parsing and connector " +"instantiation for your scheme" +msgstr "一个 ``ConnectorAdapter`` 子类，用于处理您的方案的 URL 解析和连接器实例化。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:64 +msgid "" +"A ``RemoteConnector`` subclass that implements the actual storage " +"operations (get, put, exists, etc.)" +msgstr "一个 ``RemoteConnector`` 子类，实现实际的存储操作（获取、放置、存在等）。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:66 +msgid "" +"**Package** as an installable Python module (so that LMCache can import " +"it)." +msgstr "**作为可安装的 Python 模块（以便 LMCache 可以导入它）。**" + +#: ../../source/developer_guide/extending_lmcache/index.rst:67 +msgid "" +"**Configure** LMCache to use it by adding an entry to the " +"``remote_storage_plugins`` list and specifying the module path and class " +"name in the configuration's ``extra_config``. For example:" +msgstr "**配置** LMCache 以使用它，通过将条目添加到 ``remote_storage_plugins`` 列表中，并在配置的 ``extra_config`` 中指定模块路径和类名。例如：" + +#: ../../source/developer_guide/extending_lmcache/index.rst:76 +msgid "" +"With this configuration, when LMCache sees a ``remote_url`` matching your" +" adapter's scheme (e.g., ``mystore://...``), it will import and use your " +"connector for remote operations." +msgstr "使用此配置，当 LMCache 看到与您的适配器方案匹配的 ``remote_url``（例如，``mystore://...``）时，它将导入并使用您的连接器进行远程操作。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:78 +msgid "" +"Multiple remote storage plugins can be enabled simultaneously. By " +"implementing a custom connector and configuring it as above, you can " +"integrate new remote backends (databases, distributed KV stores, etc.) " +"without changing LMCache's core code." +msgstr "可以同时启用多个远程存储插件。通过实现自定义连接器并按上述方式进行配置，您可以集成新的远程后端（数据库、分布式 KV 存储等），而无需更改 LMCache 的核心代码。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:80 +msgid "" +"**Runtime Plugin Framework** allows running custom scripts alongside " +"LMCache processes. A runtime plugin is launched as a separate subprocess " +"by LMCache’s runtime plugin launcher. Runtime plugins can target specific" +" LMCache roles – the scheduler (controller), worker processes, or all " +"nodes – depending on their filename. They can be written in Python, Bash," +" or other scripting languages, and are useful for tasks such as logging " +"and metrics, custom cache management policies, health checks, or " +"integration with external systems." +msgstr "**运行时插件框架** 允许在 LMCache 进程旁边运行自定义脚本。运行时插件由 LMCache 的运行时插件启动器作为单独的子进程启动。运行时插件可以根据其文件名针对特定的 LMCache 角色——调度器（控制器）、工作进程或所有节点。它们可以用 Python、Bash 或其他脚本语言编写，并且对于日志记录和指标、自定义缓存管理策略、健康检查或与外部系统的集成等任务非常有用。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:82 +msgid "Key points and usage of the runtime plugin system:" +msgstr "运行时插件系统的关键点和使用方法：" + +#: ../../source/developer_guide/extending_lmcache/index.rst:84 +msgid "" +"**Configuration:** You can enable runtime plugins via environment " +"variables and the LMCache config file. Set the `runtime_plugin_locations`" +" in your YAML config to point to directories containing plugin scripts. " +"For example:" +msgstr "**配置：** 您可以通过环境变量和 LMCache 配置文件启用运行时插件。在您的 YAML 配置中设置 `runtime_plugin_locations`，指向包含插件脚本的目录。例如：" + +#: ../../source/developer_guide/extending_lmcache/index.rst:92 +msgid "At runtime, LMCache will scan these directories for plugin files." +msgstr "在运行时，LMCache 将扫描这些目录以查找插件文件。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:94 +msgid "" +"**Environment Variables:** LMCache provides context to plugins through " +"env vars: - `LMCACHE_RUNTIME_PLUGIN_ROLE`: the process role (e.g. " +"`SCHEDULER` or `WORKER`) in which the plugin is running. - " +"`LMCACHE_RUNTIME_PLUGIN_CONFIG`: a JSON string for any plugin " +"configuration passed through LMCache. - " +"`LMCACHE_RUNTIME_PLUGIN_WORKER_ID`: the ID of the current worker (if " +"running on a worker). - `LMCACHE_RUNTIME_PLUGIN_WORKER_COUNT`: total " +"number of worker processes in the LMCache cluster." +msgstr "**环境变量：** LMCache 通过环境变量为插件提供上下文：- `LMCACHE_RUNTIME_PLUGIN_ROLE`：插件运行的进程角色（例如 `SCHEDULER` 或 `WORKER`）。- `LMCACHE_RUNTIME_PLUGIN_CONFIG`：通过 LMCache 传递的任何插件配置的 JSON 字符串。- `LMCACHE_RUNTIME_PLUGIN_WORKER_ID`：当前工作进程的 ID（如果在工作进程上运行）。- `LMCACHE_RUNTIME_PLUGIN_WORKER_COUNT`：LMCache 集群中工作进程的总数。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:100 +msgid "" +"**Naming Conventions:** Runtime plugin filenames determine where they " +"execute: - Files prefixed with **`scheduler_`** run only on the scheduler" +" process. *(Example: `scheduler_metrics.py` runs on the scheduler only.)*" +" - Files prefixed with **`worker_`** run on worker processes. If a " +"numeric worker ID is included (e.g. `worker_0_health.sh`), the plugin " +"runs only on that specific worker. If no ID is included (e.g. " +"`worker_logcollector.py`), the plugin will run on **all** workers. - " +"Files prefixed with **`all_`** (or any file without a role prefix) run on" +" all LMCache processes (both the scheduler and every worker). *(Example: " +"`all_monitor.sh` runs on every LMCache node.)* - Role names in filenames " +"are case-insensitive. Ensure worker ID (if specified) is numeric and part" +" of the filename (with underscores separating, e.g. `worker_2_custom.py` " +"has three parts, which targets worker ID 2 specifically)." +msgstr "**命名约定：** 运行时插件文件名决定它们的执行位置：- 以 **`scheduler_`** 开头的文件仅在调度程序进程上运行。*(示例：`scheduler_metrics.py` 仅在调度程序上运行。)* - 以 **`worker_`** 开头的文件在工作进程上运行。如果包含数字工作 ID（例如 `worker_0_health.sh`），则插件仅在该特定工作进程上运行。如果不包含 ID（例如 `worker_logcollector.py`），则插件将在 **所有** 工作进程上运行。- 以 **`all_`** 开头的文件（或任何没有角色前缀的文件）在所有 LMCache 进程上运行（包括调度程序和每个工作进程）。*(示例：`all_monitor.sh` 在每个 LMCache 节点上运行。)* - 文件名中的角色名称不区分大小写。确保工作 ID（如果指定）为数字，并且是文件名的一部分（用下划线分隔，例如 `worker_2_custom.py` 有三个部分，专门针对工作 ID 2）。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:106 +msgid "" +"**Execution Model:** When LMCache starts up, the `RuntiumePluginLauncher`" +" locates and starts each plugin as a subprocess: 1. **Interpreter " +"Selection:** The runtime plugin launcher checks the script’s shebang " +"(e.g. `#!...`) to decide which interpreter to use. If no shebang is " +"provided, it falls back on the file extension (``.py`` uses the default " +"Python interpreter, ``.sh`` uses Bash, etc.). 2. **Output Handling:** " +"Stdout and stderr from the plugin processes are captured by LMCache and " +"logged with a prefix (the plugin name), so plugin output appears in " +"LMCache logs for easy debugging/monitoring. 3. **Lifecycle:** Runtime " +"plugins are launched when the LMCache process starts (if their naming " +"indicates they should run in that process). They will be automatically " +"terminated when the parent LMCache process exits, ensuring no orphan " +"processes." +msgstr "**执行模型：** 当 LMCache 启动时，`RuntiumePluginLauncher` 会定位并启动每个插件作为子进程：1. **解释器选择：** 运行时插件启动器检查脚本的 shebang（例如 `#!...`）以决定使用哪个解释器。如果没有提供 shebang，它将回退到文件扩展名（``.py`` 使用默认的 Python 解释器，``.sh`` 使用 Bash 等）。2. **输出处理：** 插件进程的 stdout 和 stderr 被 LMCache 捕获并带有前缀（插件名称）记录，因此插件输出会出现在 LMCache 日志中，便于调试/监控。3. **生命周期：** 当 LMCache 进程启动时，运行时插件会被启动（如果它们的命名指示它们应该在该进程中运行）。当父 LMCache 进程退出时，它们会被自动终止，确保没有孤儿进程。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:111 +msgid "" +"**Best Practices:** When writing runtime plugins, consider the following " +"guidelines to ensure they work smoothly with LMCache: - Keep plugins " +"lightweight in terms of resource usage and startup time, so they don’t " +"slow down the LMCache process. - Use clear and descriptive filenames to " +"reflect their purpose. - Include proper error handling within your plugin" +" script to avoid unhandled exceptions causing issues. - Use a shebang " +"line at the top of the script for portability (so the correct interpreter" +" is invoked). - Validate any configuration input (from " +"`LMCACHE_RUNTIME_PLUGIN_CONFIG` or elsewhere) before use. - If a plugin " +"performs lengthy operations, implement timeouts or periodic logging so " +"you can detect if it hangs, and ensure it does not block LMCache’s normal" +" operation." +msgstr "**最佳实践：** 在编写运行时插件时，请考虑以下指南，以确保它们与 LMCache 平稳协作：- 保持插件在资源使用和启动时间方面轻量，以免减慢 LMCache 进程。- 使用清晰且描述性的文件名来反映其目的。- 在插件脚本中包含适当的错误处理，以避免未处理的异常导致问题。- 在脚本顶部使用 shebang 行以提高可移植性（以便调用正确的解释器）。- 在使用之前验证任何配置输入（来自 `LMCACHE_RUNTIME_PLUGIN_CONFIG` 或其他地方）。- 如果插件执行耗时操作，请实现超时或定期日志记录，以便您可以检测是否挂起，并确保它不会阻塞 LMCache 的正常操作。" + +#: ../../source/developer_guide/extending_lmcache/index.rst:119 +msgid "" +"Together, these extension points – custom storage backends, remote " +"storage connectors, and runtime plugin scripts – let users tailor " +"LMCache's functionality and integrate with external systems in a modular," +" maintainable way." +msgstr "这些扩展点——自定义存储后端、远程存储连接器和运行时插件脚本——使用户能够以模块化和可维护的方式定制 LMCache 的功能并与外部系统集成。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/native_connectors.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/native_connectors.po new file mode 100644 index 00000000000..da33ad29a9a --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/native_connectors.po @@ -0,0 +1,729 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:2 +msgid "Adding Native Connectors" +msgstr "添加本地连接器" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:7 +msgid "Overview" +msgstr "概述" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:9 +msgid "" +"Native connectors are high-performance C++ storage backends that " +"integrate with LMCache through pybind11. They work in **both** LMCache " +"operating modes:" +msgstr "本地连接器是高性能的 C++ 存储后端，通过 pybind11 与 LMCache 集成。它们在 **两种** LMCache 操作模式下均可工作：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:12 +msgid "" +"**Non-MP mode** (single process): via ``ConnectorClientBase`` (asyncio " +"integration)" +msgstr "**非 MP 模式**（单进程）：通过 ``ConnectorClientBase``（asyncio 集成）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:13 +msgid "" +"**MP mode** (multiprocess): via ``NativeConnectorL2Adapter`` (L2 adapter " +"interface)" +msgstr "**多进程模式** (multiprocess): 通过 ``NativeConnectorL2Adapter`` (L2 适配器接口)" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:15 +msgid "Write the connector once, get both modes for free." +msgstr "一次编写连接器，免费获得两种模式。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:17 +msgid "" +"The framework lives in ``csrc/storage_backends/`` with the Redis RESP " +"connector as the reference implementation." +msgstr "该框架位于 ``csrc/storage_backends/``，Redis RESP 连接器作为参考实现。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:21 +msgid "Architecture" +msgstr "架构" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:41 +msgid "**Design principles:**" +msgstr "**设计原则：**" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:43 +msgid "" +"**GIL release** at the pybind layer for true concurrency between native " +"threads" +msgstr "**在 pybind 层释放 GIL 以实现本地线程之间的真正并发**" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:44 +msgid "" +"**Batching with tiling**: work for a batched request is split evenly " +"among threads" +msgstr "**批处理与分块**：批量请求的工作在线程之间均匀分配" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:45 +msgid "**eventfd-based completions**: the kernel wakes Python -- no polling" +msgstr "**基于 eventfd 的完成**：内核唤醒 Python -- 无需轮询" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:46 +msgid "" +"**Non-blocking submission**: submission queue / completion queue " +"architecture" +msgstr "**非阻塞提交**：提交队列 / 完成队列架构" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:50 +msgid "Step 1: C++ Connector" +msgstr "步骤 1：C++ 连接器" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:52 +msgid "" +"Create your connector directory (e.g., " +"``csrc/storage_backends/mybackend/``) and inherit from " +"``ConnectorBase``. You need to override 4 required " +"methods (and optionally ``do_single_delete`` to support eviction)." +msgstr "创建你的连接器目录（例如，``csrc/storage_backends/mybackend/``），并从 ``ConnectorBase`` 继承。你需要重写 4 个必需的方法（并可选地重写 ``do_single_delete`` 以支持逐出）。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:56 +msgid "**connector.h:**" +msgstr "**connector.h:**" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:127 +msgid "**What ConnectorBase gives you for free:**" +msgstr "**ConnectorBase 为您免费提供的内容：**" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:129 +msgid "Worker thread pool with per-thread connections" +msgstr "每个线程连接的工作线程池" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:130 +msgid "Submission queue (lock-free enqueue) and completion queue" +msgstr "提交队列（无锁入队）和完成队列" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:131 +msgid "Automatic tiling: batch operations are split across workers" +msgstr "自动平铺：批量操作在工作节点之间拆分" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:132 +msgid "eventfd signaling on completion (kernel wakes Python)" +msgstr "完成时的 eventfd 信号（内核唤醒 Python）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:133 +msgid "Graceful shutdown (stop flag, drain, join)" +msgstr "优雅关闭（停止标志，排空，加入）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:136 +msgid "" +"Always call ``start_workers()`` at the **end** of your derived " +"constructor, after all member variables are initialized. Worker threads " +"call ``create_connection()`` immediately, so the object must be fully " +"constructed." +msgstr "始终在派生构造函数的**末尾**调用 ``start_workers()``，在所有成员变量初始化之后。工作线程会立即调用 ``create_connection()``，因此对象必须完全构造。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:140 +msgid "" +"**Reference:** ``csrc/storage_backends/redis/connector.h`` and " +"``connector.cpp``" +msgstr "**参考:** ``csrc/storage_backends/redis/connector.h`` 和 ``connector.cpp``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:144 +msgid "Step 2: Pybind Module" +msgstr "步骤 2：Pybind 模块" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:146 +msgid "" +"Use the ``LMCACHE_BIND_CONNECTOR_METHODS`` macro, which binds all 7 " +"methods (``event_fd``, ``submit_batch_get/set/exists/delete``, " +"``drain_completions``, ``close``) with proper GIL release and Python " +"buffer protocol handling." +msgstr "使用 ``LMCACHE_BIND_CONNECTOR_METHODS`` 宏，该宏绑定所有 7 个方法（``event_fd``、``submit_batch_get/set/exists/delete``、``drain_completions``、``close``），并正确处理 GIL 释放和 Python 缓冲区协议。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:168 +msgid "The pybind utilities automatically:" +msgstr "pybind 工具自动：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:170 +msgid "Extract buffer pointers from Python ``memoryview`` objects under the GIL" +msgstr "在 GIL 下从 Python ``memoryview`` 对象中提取缓冲区指针" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:171 +msgid "Release the GIL before calling into C++" +msgstr "在调用 C++ 之前释放 GIL" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:172 +msgid "" +"Convert C++ ``Completion`` structs to Python tuples ``(future_id, ok, " +"error, result_bools)``" +msgstr "将 C++ ``Completion`` 结构转换为 Python 元组 ``(future_id, ok, error, result_bools)``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:174 +msgid "**Reference:** ``csrc/storage_backends/redis/pybind.cpp``" +msgstr "**参考:** ``csrc/storage_backends/redis/pybind.cpp``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:178 +msgid "Step 3: Build System" +msgstr "步骤 3：构建系统" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:180 +msgid "" +"Register your C++ sources in ``setup.py`` alongside the existing Redis " +"extension:" +msgstr "在 ``setup.py`` 中注册您的 C++ 源文件，与现有的 Redis 扩展一起：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:201 +msgid "Then rebuild:" +msgstr "然后重建：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:209 +msgid "Step 4: Python Client (Non-MP Mode)" +msgstr "步骤 4：Python 客户端（非 MP 模式）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:211 +msgid "" +"Inherit from ``ConnectorClientBase`` which provides asyncio event loop " +"integration, future management, and both sync and async methods." +msgstr "从 ``ConnectorClientBase`` 继承，该类提供 asyncio 事件循环集成、未来管理以及同步和异步方法。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:226 +msgid "" +"This gives you ``batch_get``, ``batch_set``, ``batch_exists`` (async), " +"and their synchronous variants, all with automatic eventfd-driven " +"completion handling." +msgstr "这为您提供了 ``batch_get``、``batch_set``、``batch_exists``（异步）及其同步变体，所有这些都具有自动的基于 eventfd 的完成处理。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:229 +msgid "" +"**Reference:** " +"``lmcache/v1/storage_backend/native_clients/resp_client.py``" +msgstr "**参考:** ``lmcache/v1/storage_backend/native_clients/resp_client.py``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:233 +msgid "Step 5: L2 Adapter (MP Mode)" +msgstr "步骤 5：L2 适配器（MP 模式）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:235 +msgid "" +"To use your connector as an L2 adapter in MP mode, create a single Python" +" module that defines the config class, factory function, and self-" +"registers both. The ``NativeConnectorL2Adapter`` bridge handles all the " +"complexity (eventfd demuxing, key serialization, locking)." +msgstr "要将您的连接器用作 MP 模式下的 L2 适配器，请创建一个单独的 Python 模块，该模块定义配置类、工厂函数，并自我注册这两者。``NativeConnectorL2Adapter`` 桥接处理所有复杂性（eventfd 解复用、键序列化、锁定）。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:240 +msgid "Create a new file in the L2 adapters package:" +msgstr "在 L2 适配器包中创建一个新文件：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:322 +msgid "" +"The L2 adapter package uses ``pkgutil.iter_modules`` to auto-discover all" +" modules in ``lmcache/v1/distributed/l2_adapters/``. Simply creating the " +"file above is sufficient -- no changes to ``__init__.py`` or any other " +"existing file are needed." +msgstr "L2 适配器包使用 ``pkgutil.iter_modules`` 自动发现 ``lmcache/v1/distributed/l2_adapters/`` 中的所有模块。仅仅创建上述文件就足够了——不需要对 ``__init__.py`` 或任何其他现有文件进行更改。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:326 +msgid "**Usage from the command line:**" +msgstr "**命令行使用：**" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:336 +msgid "How NativeConnectorL2Adapter Bridges the Gap" +msgstr "原生连接器 L2Adapter 如何弥合差距" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:338 +msgid "" +"The C++ connector has 1 eventfd and mixed completions. MP mode's " +"``L2AdapterInterface`` requires 3 separate eventfds and typed results. " +"The bridge handles this transparently:" +msgstr "C++ 连接器有 1 个 eventfd 和混合完成。MP 模式的 ``L2AdapterInterface`` 需要 3 个独立的 eventfd 和类型化结果。桥接器对此进行了透明处理：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:345 +msgid "L2 Adapter Method" +msgstr "L2 适配器方法" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:346 +msgid "Native Call" +msgstr "本地调用" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:347 +msgid "Extra Logic" +msgstr "额外逻辑" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:348 +msgid "``submit_store_task(keys, objs)``" +msgstr "``submit_store_task(keys, objs)``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:349 +msgid "``submit_batch_set``" +msgstr "``submit_batch_set``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:350 +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:356 +msgid "ObjectKey to string, MemoryObj to memoryview" +msgstr "ObjectKey 转换为字符串，MemoryObj 转换为 memoryview" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:351 +msgid "``submit_lookup_and_lock_task(keys)``" +msgstr "``submit_lookup_and_lock_task(keys)``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:352 +msgid "``submit_batch_exists``" +msgstr "``submit_batch_exists``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:353 +msgid "client-side lock refcount" +msgstr "客户端锁引用计数" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:354 +msgid "``submit_load_task(keys, objs)``" +msgstr "``submit_load_task(keys, objs)``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:355 +msgid "``submit_batch_get``" +msgstr "``submit_batch_get``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:357 +msgid "``submit_unlock(keys)``" +msgstr "``submit_unlock(keys)``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:358 +msgid "*(none)*" +msgstr "*(无)*" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:359 +msgid "client-side lock decrement" +msgstr "客户端锁减少" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:360 +msgid "``pop_completed_store_tasks()``" +msgstr "``pop_completed_store_tasks()``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:361 +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:364 +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:367 +msgid "via ``drain_completions``" +msgstr "通过 ``drain_completions``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:362 +msgid "demux by op type" +msgstr "按操作类型解复用" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:363 +msgid "``query_lookup_and_lock_result()``" +msgstr "``query_lookup_and_lock_result()``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:365 +msgid "exists results to Bitmap, apply locks" +msgstr "存在结果到位图，应用锁定" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:366 +msgid "``query_load_result()``" +msgstr "``query_load_result()``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:368 +msgid "ok/fail to Bitmap" +msgstr "成功/失败到位图" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:370 +msgid "" +"A background demux thread polls the native eventfd, calls " +"``drain_completions()``, looks up each ``future_id`` to determine its " +"operation type, routes the result to the correct completion dict, and " +"signals the corresponding Python eventfd." +msgstr "一个后台解复用线程轮询本地 eventfd，调用 ``drain_completions()``，查找每个 ``future_id`` 以确定其操作类型，将结果路由到正确的完成字典，并发出相应的 Python eventfd 信号。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:376 +msgid "Third-Party Native Connector Plugins (``native_plugin``)" +msgstr "第三方原生连接器插件（``native_plugin``）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:380 +msgid "" +"The steps above describe how to add a native connector **inside** the " +"LMCache source tree. If you want to ship a connector as a **separate, " +"pip-installable package** (e.g. a proprietary storage backend), use the " +"``native_plugin`` L2 adapter type instead. It dynamically loads your " +"connector at runtime -- no LMCache source modifications required." +msgstr "上述步骤描述了如何在 LMCache 源代码树中**内部**添加一个原生连接器。如果您想将连接器作为**单独的、可通过 pip 安装的包**（例如，专有存储后端）进行发布，请改用 ``native_plugin`` L2 适配器类型。它在运行时动态加载您的连接器——无需修改 LMCache 源代码。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:386 +msgid "How It Works" +msgstr "如何工作" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:388 +msgid "" +"The ``native_plugin`` adapter type loads a third-party **connector " +"object** (pybind-wrapped C++ or pure Python) and wraps it with the built-" +"in ``NativeConnectorL2Adapter`` bridge. This means you only need to " +"implement the 6 connector methods -- the Python-side demux/lock bridging " +"logic is reused from LMCache." +msgstr "``native_plugin`` 适配器类型加载一个第三方 **连接器对象**（pybind 封装的 C++ 或纯 Python），并用内置的 ``NativeConnectorL2Adapter`` 桥接器将其包装。这意味着您只需实现 6 个连接器方法——Python 端的解复用/锁定桥接逻辑是从 LMCache 重用的。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:393 +msgid "``plugin`` vs ``native_plugin``" +msgstr "``plugin`` 与 ``native_plugin``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:397 +msgid "Aspect" +msgstr "方面" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:398 +msgid "``plugin``" +msgstr "``plugin``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:399 +msgid "``native_plugin``" +msgstr "``native_plugin``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:400 +msgid "What is loaded" +msgstr "加载的内容是什么" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:401 +msgid "A full ``L2AdapterInterface`` subclass" +msgstr "一个完整的 ``L2AdapterInterface`` 子类" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:402 +msgid "A **connector** object (lower level)" +msgstr "一个 **连接器** 对象（较低级别）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:403 +msgid "Bridging logic" +msgstr "桥接逻辑" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:404 +msgid "Provided by the plugin itself" +msgstr "由插件本身提供" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:405 +msgid "Reused from ``NativeConnectorL2Adapter``" +msgstr "从 ``NativeConnectorL2Adapter`` 重用" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:406 +msgid "Third-party effort" +msgstr "第三方努力" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:407 +msgid "Must implement all abstract methods + 3 eventfds" +msgstr "必须实现所有抽象方法 + 3 个 eventfds" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:408 +msgid "Only 6 connector methods" +msgstr "仅需 6 个连接器方法" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:411 +msgid "Required Connector Interface" +msgstr "必需的连接器接口" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:413 +msgid "" +"The dynamically loaded connector instance must expose the following " +"methods (identical to the pybind ``LMCACHE_BIND_CONNECTOR_METHODS`` " +"contract):" +msgstr "动态加载的连接器实例必须暴露以下方法（与 pybind ``LMCACHE_BIND_CONNECTOR_METHODS`` 合同相同）：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:443 +msgid "" +"The factory validates the first 6 methods at creation time and raises " +"``TypeError`` if any are missing. ``submit_batch_delete`` is **optional**" +" -- if absent, the adapter's ``delete()`` method will be a no-op " +"(eviction will not remove keys from the backend)." +msgstr "工厂在创建时验证前 6 个方法，如果缺少任何方法则引发 ``TypeError``。``submit_batch_delete`` 是 **可选** 的——如果缺失，适配器的 ``delete()`` 方法将不执行任何操作（逐出不会从后端移除键）。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:448 +msgid "Configuration" +msgstr "配置" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:462 +msgid "``NativePluginL2AdapterConfig`` fields" +msgstr "``NativePluginL2AdapterConfig`` 字段" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:466 +msgid "Field" +msgstr "字段" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:467 +msgid "Type" +msgstr "类型" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:468 +msgid "Required" +msgstr "必需" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:469 +msgid "Description" +msgstr "描述" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:470 +msgid "``module_path``" +msgstr "``module_path``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:471 +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:475 +msgid "``str``" +msgstr "``str``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:472 +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:476 +msgid "yes" +msgstr "是" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:473 +msgid "Dotted Python import path of the module containing the connector class." +msgstr "连接器类所在模块的点分 Python 导入路径。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:474 +msgid "``class_name``" +msgstr "``class_name``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:477 +msgid "Name of the connector class inside ``module_path``." +msgstr "``module_path`` 中连接器类的名称。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:478 +msgid "``adapter_params``" +msgstr "``adapter_params``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:479 +msgid "``dict``" +msgstr "``dict``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:480 +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:484 +msgid "no" +msgstr "不" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:481 +msgid "Forwarded as ``**kwargs`` to the connector class constructor." +msgstr "作为 ``**kwargs`` 转发给连接器类构造函数。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:482 +msgid "``max_capacity_gb``" +msgstr "``max_capacity_gb``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:483 +msgid "``float``" +msgstr "``float``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:485 +msgid "" +"Maximum L2 storage capacity in GB for client-side usage tracking. " +"Required for L2 eviction. Default 0 (disabled)." +msgstr "客户端使用跟踪的最大 L2 存储容量（以 GB 为单位）。这是 L2 逐出的必要条件。默认值为 0（禁用）。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:488 +msgid "Loading Flow" +msgstr "加载流程" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:510 +msgid "Step-by-Step: Building an External Native Connector Plugin" +msgstr "逐步：构建外部原生连接器插件" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:512 +msgid "" +"**Create a Python package** with a C++ pybind11 extension that inherits " +"from ``ConnectorBase`` (same base class as built-in connectors)." +msgstr "**创建一个 Python 包**，其中包含一个继承自 ``ConnectorBase`` 的 C++ pybind11 扩展（与内置连接器相同的基类）。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:515 +msgid "Project layout:" +msgstr "项目布局：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:531 +msgid "" +"**Implement the C++ connector** inheriting from ``ConnectorBase`` and " +"override the 4 required methods (``create_connection``, " +"``do_single_get``, ``do_single_set``, ``do_single_exists``) and " +"optionally ``do_single_delete`` for eviction support." +msgstr "**实现 C++ 连接器**，继承自 ``ConnectorBase``，并重写 4 个必需的方法（``create_connection``、``do_single_get``、``do_single_set``、``do_single_exists``），可选地重写 ``do_single_delete`` 以支持逐出。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:535 +msgid "" +"**Create pybind11 bindings** using the ``LMCACHE_BIND_CONNECTOR_METHODS``" +" macro:" +msgstr "**使用 ``LMCACHE_BIND_CONNECTOR_METHODS`` 宏创建 pybind11 绑定：**" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:553 +msgid "" +"**Write a Python factory class** that selects the backend and returns the" +" native connector instance:" +msgstr "**编写一个 Python 工厂类**，选择后端并返回本地连接器实例：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:568 +msgid "**Build and install** the package:" +msgstr "**构建和安装** 包：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:575 +msgid "**Configure LMCache** to use it:" +msgstr "**配置 LMCache** 以使用它：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:590 +msgid "Reference Implementation" +msgstr "参考实现" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:592 +msgid "" +"See ``examples/lmc_external_native_connector/`` for a complete, pip-" +"installable example connector plugin that demonstrates:" +msgstr "请参阅 ``examples/lmc_external_native_connector/`` 以获取完整的、可通过 pip 安装的示例连接器插件，该插件演示了：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:595 +msgid "" +"C++ pybind11-wrapped connectors inheriting from ``ConnectorBase`` " +"(same as built-in Redis/FS)." +msgstr "从 ``ConnectorBase`` 继承的 C++ pybind11 封装连接器（与内置的 Redis/FS 相同）。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:597 +msgid "" +"Two backends: filesystem (``ExampleFSConnector``) and in-memory " +"(``ExampleMemoryConnector``), both in C++." +msgstr "两个后端：文件系统（``ExampleFSConnector``）和内存（``ExampleMemoryConnector``），均为 C++ 实现。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:599 +msgid "" +"A thin Python factory class (``ExampleNativeConnector``) that selects the" +" backend via a ``\"backend\"`` parameter." +msgstr "一个薄的 Python 工厂类 (``ExampleNativeConnector``)，通过 ``\"backend\"`` 参数选择后端。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:601 +msgid "" +"Worker thread pool with eventfd notification (inherited from " +"``ConnectorBase``)." +msgstr "带有 eventfd 通知的工作线程池（继承自 ``ConnectorBase``）。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:602 +msgid "Build via ``pip install -e .`` using pybind11 + setuptools." +msgstr "通过 ``pip install -e .`` 使用 pybind11 + setuptools 构建。" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:606 +msgid "Checklist" +msgstr "清单" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:608 +msgid "Use this checklist when adding a new native connector:" +msgstr "在添加新的本地连接器时，请使用此检查清单：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:610 +msgid "" +"C++ connector inheriting ``ConnectorBase`` with 4 required + 1 " +"optional (``do_single_delete``) method overrides" +msgstr "继承 ``ConnectorBase`` 的 C++ 连接器，具有 4 个必需的方法重写和 1 个可选的方法重写（``do_single_delete``）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:611 +msgid "Pybind module using ``LMCACHE_BIND_CONNECTOR_METHODS``" +msgstr "使用 ``LMCACHE_BIND_CONNECTOR_METHODS`` 的 Pybind 模块" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:612 +msgid "``setup.py`` entry for the new ``CppExtension``" +msgstr "``setup.py`` 中用于新的 ``CppExtension`` 的条目" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:613 +msgid "Python client inheriting ``ConnectorClientBase`` (non-MP mode)" +msgstr "继承 ``ConnectorClientBase`` 的 Python 客户端（非 MP 模式）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:614 +msgid "L2 adapter module with config class + factory self-registration (MP mode)" +msgstr "带有配置类和工厂自注册的 L2 适配器模块（多进程模式）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:615 +msgid "" +"Unit tests (see " +"``tests/v1/distributed/test_native_connector_l2_adapter.py``)" +msgstr "单元测试（请参见 ``tests/v1/distributed/test_native_connector_l2_adapter.py``）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:616 +msgid "Rebuild with ``pip install -e .`` and verify both modes work" +msgstr "使用 ``pip install -e .`` 重新构建并验证两种模式均正常工作" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:618 +msgid "For **external** native connector plugins (``native_plugin``):" +msgstr "对于 **外部** 原生连接器插件 (``native_plugin``)：" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:620 +msgid "Separate pip-installable package with C++ pybind11 extension" +msgstr "带有 C++ pybind11 扩展的独立 pip 可安装包" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:621 +msgid "" +"Connector class exposing the 6 required methods (+ optional " +"``submit_batch_delete`` for eviction)" +msgstr "连接器类，暴露 6 个必需的方法（+ 可选的 ``submit_batch_delete`` 用于逐出）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:622 +msgid "Python factory class for backend selection" +msgstr "后端选择的 Python 工厂类" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:623 +msgid "``pip install -e .`` and configure via ``--l2-adapter`` JSON" +msgstr "``pip install -e .`` 并通过 ``--l2-adapter`` JSON 进行配置" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:624 +msgid "Unit tests (see ``examples/lmc_external_native_connector/tests/``)" +msgstr "单元测试（参见 ``examples/lmc_external_native_connector/tests/``）" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:628 +msgid "Additional Resources" +msgstr "附加资源" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:630 +msgid "Framework source: ``csrc/storage_backends/``" +msgstr "框架源代码：``csrc/storage_backends/``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:631 +msgid "``ConnectorBase`` template: ``csrc/storage_backends/connector_base.h``" +msgstr "``ConnectorBase`` 模板: ``csrc/storage_backends/connector_base.h``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:632 +msgid "" +"``IStorageConnector`` interface: " +"``csrc/storage_backends/connector_interface.h``" +msgstr "``IStorageConnector`` 接口: ``csrc/storage_backends/connector_interface.h``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:633 +msgid "Pybind utilities: ``csrc/storage_backends/connector_pybind_utils.h``" +msgstr "Pybind 工具: ``csrc/storage_backends/connector_pybind_utils.h``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:634 +msgid "Redis reference implementation: ``csrc/storage_backends/redis/``" +msgstr "Redis 参考实现: ``csrc/storage_backends/redis/``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:635 +msgid "Architecture README: ``csrc/storage_backends/README.md``" +msgstr "架构自述文件: ``csrc/storage_backends/README.md``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:636 +msgid "" +"External native connector example: " +"``examples/lmc_external_native_connector/``" +msgstr "外部原生连接器示例：``examples/lmc_external_native_connector/``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:638 +msgid "" +"Native plugin adapter source: " +"``lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py``" +msgstr "原生插件适配器源: ``lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:640 +msgid "" +"Design document: " +"``lmcache/v1/distributed/l2_adapters/design_docs/plugin.md``" +msgstr "设计文档：``lmcache/v1/distributed/l2_adapters/design_docs/plugin.md``" + +#: ../../source/developer_guide/extending_lmcache/native_connectors.rst:642 +msgid "" +"RESP backend user guide: :doc:`RESP (Native Redis/Valkey) " +"<../../kv_cache/storage_backends/resp>`" +msgstr "RESP 后端用户指南： :doc:`RESP (原生 Redis/Valkey) <../../kv_cache/storage_backends/resp>`" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/remote_storage_plugins.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/remote_storage_plugins.po new file mode 100644 index 00000000000..8e0ca7a17fe --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/remote_storage_plugins.po @@ -0,0 +1,267 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:2 +msgid "Remote Storage Plugins" +msgstr "远程存储插件" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:4 +msgid "" +"LMCache supports built-in remote storage connectors for Redis, " +"InfiniStore, MooncakeStore, S3, Hugging Face Buckets, and more. The " +"remote storage plugin system provides the ability to add custom storage " +"connectors through dynamic loading. This enables extending remote storage" +" capabilities without modifying core code." +msgstr "LMCache 支持内置的远程存储连接器，包括 Redis、InfiniStore、MooncakeStore、S3、Hugging Face Buckets 等。远程存储插件系统提供了通过动态加载添加自定义存储连接器的能力。这使得在不修改核心代码的情况下扩展远程存储功能成为可能。" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:10 +msgid "" +"The ``remote_url`` configuration is **deprecated** and will be removed in" +" a future release. Please use ``remote_storage_plugins`` instead." +msgstr "``remote_url`` 配置已**弃用**，并将在未来的版本中移除。请改用 ``remote_storage_plugins``。" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:14 +msgid "Connector Definition Requirements" +msgstr "连接器定义要求" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:15 +msgid "A custom remote storage connector requires two classes:" +msgstr "自定义远程存储连接器需要两个类：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:17 +msgid "" +"**ConnectorAdapter**: Handles URL scheme matching and connector " +"instantiation" +msgstr "**ConnectorAdapter**: 处理 URL 方案匹配和连接器实例化" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:19 +msgid "Inherit from ``ConnectorAdapter``" +msgstr "从 ``ConnectorAdapter`` 继承" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:20 +msgid "Set the URL scheme in the constructor (e.g., ``mystore://``)" +msgstr "在构造函数中设置 URL 方案（例如，``mystore://``）" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:21 +msgid "Implement the ``create_connector`` method" +msgstr "实现 ``create_connector`` 方法" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:23 +msgid "**RemoteConnector**: Implements the actual storage operations" +msgstr "**RemoteConnector**: 实现实际的存储操作" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:25 +msgid "Inherit from ``RemoteConnector``" +msgstr "从 ``RemoteConnector`` 继承" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:26 +msgid "" +"Implement all abstract methods: ``exists``, ``exists_sync``, ``get``, " +"``put``, ``list``, ``close``" +msgstr "实现所有抽象方法：``exists``、``exists_sync``、``get``、``put``、``list``、``close``" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:30 +msgid "" +"The ``ConnectorAdapter`` constructor receives no arguments from LMCache. " +"The scheme should be set by calling the parent constructor with the " +"scheme string." +msgstr "``ConnectorAdapter`` 构造函数不接收来自 LMCache 的任何参数。应通过调用父构造函数并传入方案字符串来设置方案。" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:32 +msgid "" +"The ``create_connector`` method receives a ``ConnectorContext`` object " +"containing the URL, event loop, local CPU backend, config, metadata, and " +"``plugin_name``." +msgstr "``create_connector`` 方法接收一个 ``ConnectorContext`` 对象，其中包含 URL、事件循环、本地 CPU 后端、配置、元数据和 ``plugin_name``。" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:35 +msgid "Plugin Naming Convention" +msgstr "插件命名约定" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:36 +#, python-brace-format +msgid "Plugin names follow the format ``{type}`` or ``{type}.{instance}``:" +msgstr "插件名称遵循格式 ``{type}`` 或 ``{type}.{instance}``：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:38 +#, python-brace-format +msgid "" +"``{type}`` — a single instance of that connector type (e.g. ``fs``, " +"``mooncakestore``)" +msgstr "``{type}`` — 该连接器类型的单个实例（例如 ``fs``, ``mooncakestore``）" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:39 +#, python-brace-format +msgid "" +"``{type}.{instance}`` — a named instance, allowing **multiple instances " +"of the same type** (e.g. ``fs.primary``, ``fs.backup``)" +msgstr "``{type}.{instance}`` — 一个命名实例，允许 **同类型的多个实例**（例如 ``fs.primary``, ``fs.backup``）" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:41 +msgid "" +"The framework extracts the *type* portion (everything before the first " +"``.``) to locate the matching ``ConnectorAdapter``. The full plugin name " +"is used as the configuration key prefix." +msgstr "框架提取 *type* 部分（在第一个 ``.`` 之前的所有内容）以定位匹配的 ``ConnectorAdapter``。完整的插件名称用作配置键前缀。" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:44 +msgid "Using Built-in Connectors via Plugins" +msgstr "通过插件使用内置连接器" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:45 +msgid "" +"Built-in connectors (``fs``, ``mooncakestore``, ``hfbucket``, etc.) can " +"be used directly via ``remote_storage_plugins`` without specifying " +"``module_path`` or ``class_name``. Their configuration is placed under " +"``extra_config``:" +msgstr "内置连接器（``fs``, ``mooncakestore``, ``hfbucket`` 等）可以通过 ``remote_storage_plugins`` 直接使用，而无需指定 ``module_path`` 或 ``class_name``。它们的配置放在 ``extra_config`` 下：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:58 +msgid "Multiple instances of the same connector type:" +msgstr "同一连接器类型的多个实例：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:67 +msgid "Mixing different connector types:" +msgstr "混合不同的连接器类型：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:76 +msgid "Built-in Hugging Face Buckets example:" +msgstr "内置 Hugging Face Buckets 示例：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:86 +msgid "How to Integrate Custom Remote Storage with LMCache" +msgstr "如何将自定义远程存储与 LMCache 集成" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:87 +msgid "Install your connector package in the LMCache environment" +msgstr "在 LMCache 环境中安装您的连接器包" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:88 +msgid "" +"Add ``remote_storage_plugins`` and its related ``module_path`` and " +"``class_name`` to the ``extra_config`` section of LMCache configuration " +"as follows:" +msgstr "将 ``remote_storage_plugins`` 及其相关的 ``module_path`` 和 ``class_name`` 添加到 LMCache 配置的 ``extra_config`` 部分，如下所示：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:100 +msgid "An example configuration for a custom remote storage connector:" +msgstr "自定义远程存储连接器的示例配置：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:112 +msgid "Multiple instances of a custom connector:" +msgstr "多个自定义连接器的实例：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:125 +msgid "``remote_url`` is **deprecated**; use ``remote_storage_plugins`` instead" +msgstr "``remote_url`` 已被 **弃用**；请改用 ``remote_storage_plugins``。" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:126 +msgid "" +"``remote_storage_plugin.`` uses the full plugin name " +"(including instance suffix) as the key prefix" +msgstr "``remote_storage_plugin.`` 使用完整的插件名称（包括实例后缀）作为键前缀" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:127 +msgid "Multiple remote storage plugins can be loaded simultaneously" +msgstr "可以同时加载多个远程存储插件" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:128 +msgid "Built-in connectors do not require ``module_path`` / ``class_name``" +msgstr "内置连接器不需要 ``module_path`` / ``class_name``" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:131 +msgid "ConnectorAdapter Implementation" +msgstr "连接器适配器实现" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:132 +msgid "The ``ConnectorAdapter`` class is responsible for:" +msgstr "``ConnectorAdapter`` 类负责：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:134 +msgid "Defining the URL scheme it handles" +msgstr "定义它处理的 URL 方案" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:135 +msgid "Creating the appropriate ``RemoteConnector`` instance" +msgstr "创建适当的 ``RemoteConnector`` 实例" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:181 +msgid "RemoteConnector Implementation" +msgstr "RemoteConnector 实现" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:182 +msgid "" +"The ``RemoteConnector`` class defines the interface for remote storage " +"operations. Your implementation must provide the following abstract " +"methods:" +msgstr "``RemoteConnector`` 类定义了远程存储操作的接口。您的实现必须提供以下抽象方法：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:229 +msgid "Optional Methods" +msgstr "可选方法" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:230 +msgid "" +"The ``RemoteConnector`` base class also provides optional methods that " +"can be overridden for enhanced functionality:" +msgstr "``RemoteConnector`` 基类还提供了可选的方法，可以被重写以增强功能：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:232 +msgid "``support_ping()`` / ``ping()``: Health check support" +msgstr "``support_ping()`` / ``ping()``: 健康检查支持" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:233 +msgid "``support_batched_get()`` / ``batched_get()``: Batch retrieval operations" +msgstr "``support_batched_get()`` / ``batched_get()``: 批量检索操作" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:234 +msgid "``support_batched_put()`` / ``batched_put()``: Batch storage operations" +msgstr "``support_batched_put()`` / ``batched_put()``: 批量存储操作" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:235 +msgid "" +"``support_batched_contains()`` / ``batched_contains()``: Batch existence " +"checks" +msgstr "``support_batched_contains()`` / ``batched_contains()``: 批量存在性检查" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:236 +msgid "``remove_sync()``: Synchronous key removal" +msgstr "``remove_sync()``: 同步键移除" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:239 +msgid "Implementation Example" +msgstr "实现示例" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:240 +msgid "" +"A complete remote storage connector implementation would include both the" +" adapter and connector classes in a single module or package. Here's a " +"minimal working example structure:" +msgstr "一个完整的远程存储连接器实现将包括适配器和连接器类在一个模块或包中。以下是一个最小的工作示例结构：" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:249 +msgid "The adapter module (``adapter.py``):" +msgstr "适配器模块 (``adapter.py``):" + +#: ../../source/developer_guide/extending_lmcache/remote_storage_plugins.rst:282 +msgid "Configuration would then reference the adapter:" +msgstr "配置将引用适配器：" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/runtime_plugins.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/runtime_plugins.po new file mode 100644 index 00000000000..45398b978ec --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/runtime_plugins.po @@ -0,0 +1,185 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:2 +msgid "Runtime Plugins" +msgstr "运行时插件" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:4 +msgid "" +"The LMCache runtime plugin system provides the ability to extend " +"functionality by running custom scripts alongside LMCache processes. " +"Plugins can be written in Python and Bash for now, and are managed by the" +" ``RuntimePluginLauncher`` class." +msgstr "LMCache 运行时插件系统提供了通过在 LMCache 进程旁边运行自定义脚本来扩展功能的能力。目前，插件可以用 Python 和 Bash 编写，并由 ``RuntimePluginLauncher`` 类管理。" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:7 +msgid "Key Use Cases" +msgstr "关键使用案例" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:8 +msgid "Start metric reporters for centralized monitoring" +msgstr "启动度量报告器以进行集中监控" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:9 +msgid "Implement log reporters for log collection systems" +msgstr "实现日志报告器以用于日志收集系统" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:10 +msgid "Report process-level metrics to alerting systems" +msgstr "将进程级别的指标报告给警报系统" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:11 +msgid "Implement health checks and service discovery" +msgstr "实现健康检查和服务发现" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:12 +msgid "Custom cache management operations" +msgstr "自定义缓存管理操作" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:15 +msgid "Configuration" +msgstr "配置" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:16 +msgid "" +"Runtime plugins are configured through environment variables and " +"configuration files:" +msgstr "运行时插件通过环境变量和配置文件进行配置：" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:18 +msgid "" +"Environment Variables: - ``LMCACHE_RUNTIME_PLUGIN_ROLE``: Process role " +"(e.g., ``SCHEDULER``, ``WORKER``) - ``LMCACHE_RUNTIME_PLUGIN_CONFIG``: " +"JSON string containing plugin configuration - " +"``LMCACHE_RUNTIME_PLUGIN_WORKER_ID``: Current worker ID - " +"``LMCACHE_RUNTIME_PLUGIN_WORKER_COUNT``: Total worker count in cluster" +msgstr "环境变量： - ``LMCACHE_RUNTIME_PLUGIN_ROLE``：进程角色（例如，``SCHEDULER``，``WORKER``） - ``LMCACHE_RUNTIME_PLUGIN_CONFIG``：包含插件配置的 JSON 字符串 - ``LMCACHE_RUNTIME_PLUGIN_WORKER_ID``：当前工作线程 ID - ``LMCACHE_RUNTIME_PLUGIN_WORKER_COUNT``：集群中的总工作线程数" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:24 +msgid "Configuration File (``lmcache.yaml``):" +msgstr "配置文件 (``lmcache.yaml``):" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:35 +msgid "Runtime Plugin Naming Convention" +msgstr "运行时插件命名约定" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:36 +msgid "Plugin filenames determine execution targets:" +msgstr "插件文件名决定执行目标：" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:38 +msgid "" +"Role-Specific Plugins: - Format: " +"``[_][_].`` - Examples:" +msgstr "角色特定插件： - 格式：``[_][_].`` - 示例：" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:42 +msgid "``scheduler_foo_plugin.py``: Runs only on ``SCHEDULER``" +msgstr "``scheduler_foo_plugin.py``：仅在 ``SCHEDULER`` 上运行" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:43 +msgid "``worker_0_test.sh``: Runs only on worker ID 0" +msgstr "``worker_0_test.sh``: 仅在工作节点 ID 0 上运行" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:44 +msgid "``all_plugin.sh``: Runs on all workers" +msgstr "``all_plugin.sh``: 在所有工作节点上运行" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:46 +msgid "" +"Notes: - Role names are case-insensitive - Worker ID must be numeric when" +" specified - To target a specific worker ID, the filename must have at " +"least three parts separated by underscores (e.g., " +"`worker__.ext`). A file named `worker_.ext`" +" will run on all workers." +msgstr "注意：- 角色名称不区分大小写 - 指定时工作节点 ID 必须为数字 - 要针对特定的工作节点 ID，文件名必须至少包含三个由下划线分隔的部分（例如，`worker__.ext`）。名为 `worker_.ext` 的文件将在所有工作节点上运行。" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:52 +msgid "Execution Model" +msgstr "执行模型" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:53 +msgid "" +"**Interpreter Detection**: - Uses shebang line (e.g., " +"``#!/opt/venv/bin/python``) - Fallback interpreters:" +msgstr "**解释器检测**： - 使用 shebang 行（例如，``#!/opt/venv/bin/python``） - 备用解释器：" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:57 +msgid "``.py`` → ``python``" +msgstr "``.py`` → ``python``" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:58 +msgid "``.sh`` → ``bash``" +msgstr "``.sh`` → ``bash``" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:60 +msgid "" +"**Output Handling**: - Stdout/stderr captured continuously - Logged with " +"plugin name prefix" +msgstr "**输出处理**：- 持续捕获标准输出/标准错误 - 以插件名称前缀记录" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:64 +msgid "" +"**Process Management**: - Launched as subprocesses - Terminated when " +"parent process exits" +msgstr "**进程管理**： - 作为子进程启动 - 当父进程退出时终止" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:69 +msgid "Example Runtime Plugins" +msgstr "示例运行时插件" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:70 +msgid "Python Plugin (``scheduler_foo_plugin.py``):" +msgstr "Python 插件 (``scheduler_foo_plugin.py``):" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:76 +msgid "Bash Plugin (``all_plugin.sh``):" +msgstr "Bash 插件 (``all_plugin.sh``):" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:83 +msgid "Best Practices" +msgstr "最佳实践" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:84 +msgid "Keep runtime plugins lightweight and efficient" +msgstr "保持运行时插件轻量和高效" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:85 +msgid "Use descriptive naming conventions" +msgstr "使用描述性命名约定" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:86 +msgid "Implement graceful error handling" +msgstr "实现优雅的错误处理" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:87 +msgid "Include shebang for portability" +msgstr "包含 shebang 以提高可移植性" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:88 +msgid "Validate configuration inputs" +msgstr "验证配置输入" + +#: ../../source/developer_guide/extending_lmcache/runtime_plugins.rst:89 +msgid "Add timeout mechanisms for long operations" +msgstr "为长时间操作添加超时机制" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/storage_plugins.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/storage_plugins.po new file mode 100644 index 00000000000..02765d644f7 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/storage_plugins.po @@ -0,0 +1,379 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:2 +msgid "Storage Plugins" +msgstr "存储插件" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:4 +msgid "" +"LMCache supports out of the box storage backends like Mooncake, S3 and " +"NIXL. The LMCache storage plugin system provides the ability to add " +"custom storage backends through dynamic loading or plug and play " +"capability. In other words, extending cache storage capabilities without " +"modifying core code." +msgstr "LMCache 支持开箱即用的存储后端，如 Mooncake、S3 和 NIXL。LMCache 存储插件系统提供通过动态加载或即插即用能力添加自定义存储后端的功能。换句话说，可以在不修改核心代码的情况下扩展缓存存储能力。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:8 +msgid "Backend Definition Requirements" +msgstr "后端定义要求" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:9 +msgid "Inherit from ``StoragePluginInterface``" +msgstr "继承自 ``StoragePluginInterface``" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:10 +msgid "" +"Implement all the abstract methods of the parent interface of " +"``StoragePluginInterface``- ``StorageBackendInterface``" +msgstr "实现 ``StoragePluginInterface`` 的父接口 ``StorageBackendInterface`` 的所有抽象方法。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:11 +msgid "Package as an installable Python module" +msgstr "打包为可安装的 Python 模块" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:15 +msgid "" +"The interface constructor is the instantiation contract that the LMCache " +"loading system will use when loading custom storage backends. If you wish" +" to implement a constructor, it should have the same parameter signature " +"and call the interface constructor." +msgstr "接口构造函数是 LMCache 加载系统在加载自定义存储后端时使用的实例化契约。如果您希望实现一个构造函数，它应该具有相同的参数签名并调用接口构造函数。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:19 +msgid "How to Integrate the Backend with LMCache" +msgstr "如何将后端与 LMCache 集成" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:20 +msgid "Install your backend package in the LMCache environment" +msgstr "在 LMCache 环境中安装您的后端包" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:21 +msgid "" +"Add ``storage_plugins`` and its related ``module_path`` and " +"``class_name`` to ``extra_config`` section of LMCache configuration as " +"follows:" +msgstr "将 ``storage_plugins`` 及其相关的 ``module_path`` 和 ``class_name`` 添加到 LMCache 配置的 ``extra_config`` 部分，如下所示：" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:33 +msgid "An example configuration for a logging backend is as follows:" +msgstr "日志后端的示例配置如下：" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:47 +msgid "" +"Storage backends are initialized in order during LMCache startup - " +"earlier backends have higher priority during cache lookups" +msgstr "存储后端在 LMCache 启动时按顺序初始化 - 较早的后端在缓存查找中具有更高的优先级" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:48 +msgid "" +"``storage_plugin.`` distinguishes the different dynamic " +"loaded backends" +msgstr "``storage_plugin.`` 区分不同的动态加载后端" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:51 +msgid "Backend Implementation Example" +msgstr "后端实现示例" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:52 +msgid "" +"A sample custom backend implementation can be viewed at " +"https://github.com/opendataio/lmc_external_log_backend/" +msgstr "可以在 https://github.com/opendataio/lmc_external_log_backend/ 查看一个示例自定义后端实现。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:56 +msgid "MP-Mode L2 Adapter Plugins (``plugin``)" +msgstr "MP-模式 L2 适配器插件 (``plugin``)" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:60 +msgid "" +"The storage plugin system described above applies to **non-MP mode** " +"(single-process). For **MP mode** (multiprocess), LMCache provides the " +"``plugin`` L2 adapter type, which dynamically loads a third-party " +"``L2AdapterInterface`` implementation at runtime." +msgstr "上述描述的存储插件系统适用于 **非 MP 模式**（单进程）。对于 **MP 模式**（多进程），LMCache 提供了 ``plugin`` L2 适配器类型，该类型在运行时动态加载第三方 ``L2AdapterInterface`` 实现。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:65 +msgid "Overview" +msgstr "概述" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:67 +msgid "" +"The ``plugin`` adapter type lets you ship a full L2 adapter as a " +"**separate, pip-installable package**. At startup, LMCache imports your " +"module, instantiates your adapter class, and uses it just like a built-in" +" adapter -- no LMCache source modifications required." +msgstr "``plugin`` 适配器类型允许您将完整的 L2 适配器作为 **单独的、可通过 pip 安装的包** 进行发布。在启动时，LMCache 导入您的模块，实例化您的适配器类，并像使用内置适配器一样使用它 -- 无需对 LMCache 源代码进行修改。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:73 +msgid "" +"If your storage backend is a native C++ connector (pybind-wrapped), " +"consider using the ``native_plugin`` type instead (see " +":doc:`native_connectors`). It reuses the built-in bridging logic so you " +"only need to implement 6 connector methods rather than the full " +"``L2AdapterInterface``." +msgstr "如果您的存储后端是一个原生的 C++ 连接器（使用 pybind 封装），请考虑使用 ``native_plugin`` 类型（请参见 :doc:`native_connectors`）。它重用了内置的桥接逻辑，因此您只需实现 6 个连接器方法，而不是完整的 ``L2AdapterInterface``。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:79 +msgid "Configuration" +msgstr "配置" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:93 +msgid "``PluginL2AdapterConfig`` fields" +msgstr "``PluginL2AdapterConfig`` 字段" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:97 +msgid "Field" +msgstr "字段" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:98 +msgid "Type" +msgstr "类型" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:99 +msgid "Required" +msgstr "必需的" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:100 +msgid "Description" +msgstr "描述" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:101 +msgid "``module_path``" +msgstr "``module_path``" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:102 +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:106 +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:114 +msgid "``str``" +msgstr "``str``" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:103 +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:107 +msgid "yes" +msgstr "是" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:104 +msgid "Dotted Python import path of the module containing the adapter class." +msgstr "适配器类所在模块的点分 Python 导入路径。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:105 +msgid "``class_name``" +msgstr "``class_name``" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:108 +msgid "" +"Name of the class inside ``module_path`` that implements " +"``L2AdapterInterface``." +msgstr "``module_path`` 中实现 ``L2AdapterInterface`` 的类的名称。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:109 +msgid "``adapter_params``" +msgstr "``adapter_params``" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:110 +msgid "``dict``" +msgstr "``dict``" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:111 +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:115 +msgid "no" +msgstr "不" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:112 +msgid "" +"Forwarded to the adapter class constructor (as a typed config or raw " +"dict)." +msgstr "转发到适配器类构造函数（作为类型化配置或原始字典）。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:113 +msgid "``config_class_name``" +msgstr "``config_class_name``" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:116 +msgid "Explicit config class name; when omitted the factory auto-discovers it." +msgstr "显式配置类名称；当省略时，工厂会自动发现它。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:119 +msgid "Config Class Auto-Discovery" +msgstr "配置类自动发现" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:121 +msgid "" +"The factory automatically resolves the adapter's config class using the " +"following priority chain (first match wins):" +msgstr "工厂会自动使用以下优先级链解析适配器的配置类（第一个匹配的优先）：" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:124 +msgid "**Explicit** ``config_class_name`` field in JSON config." +msgstr "**显式** ``config_class_name`` 字段在 JSON 配置中。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:125 +msgid "" +"**Convention**: adapter class name + ``\"Config\"`` suffix (e.g. " +"``MyL2Adapter`` looks for ``MyL2AdapterConfig``)." +msgstr "**约定**：适配器类名 + ``\"Config\"`` 后缀（例如 ``MyL2Adapter`` 查找 ``MyL2AdapterConfig``）。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:127 +msgid "" +"**Attribute**: ``config_class_name`` attribute on the adapter class " +"itself." +msgstr "**属性**: ``config_class_name`` 属性在适配器类本身上。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:128 +msgid "**Fallback**: no config class found -- pass raw ``adapter_params`` dict." +msgstr "**后备**: 未找到配置类 -- 传递原始 ``adapter_params`` 字典。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:130 +msgid "" +"Each candidate is looked up in the loaded module and validated as an " +"``L2AdapterConfigBase`` subclass. This means most plugins that follow the" +" naming convention will **automatically** receive a typed config instance" +" without any extra configuration." +msgstr "每个候选项都会在加载的模块中查找，并验证为 ``L2AdapterConfigBase`` 子类。这意味着大多数遵循命名约定的插件将 **自动** 接收一个类型化的配置实例，而无需任何额外的配置。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:135 +msgid "Plugin Contract" +msgstr "插件合同" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:137 +msgid "A plugin adapter class **must**:" +msgstr "插件适配器类 **必须**：" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:139 +msgid "" +"Subclass ``L2AdapterInterface`` from " +"``lmcache.v1.distributed.l2_adapters.base``." +msgstr "从 ``lmcache.v1.distributed.l2_adapters.base`` 子类化 ``L2AdapterInterface``。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:140 +msgid "" +"Implement all abstract methods: ``submit_store_task``, " +"``pop_completed_store_tasks``, ``submit_lookup_and_lock_task``, " +"``query_lookup_and_lock_result``, ``submit_unlock``, " +"``submit_load_task``, ``query_load_result``, ``close``, and all three " +"event-fd getters." +msgstr "实现所有抽象方法：``submit_store_task``、``pop_completed_store_tasks``、``submit_lookup_and_lock_task``、``query_lookup_and_lock_result``、``submit_unlock``、``submit_load_task``、``query_load_result``、``close``，以及所有三个事件文件描述符获取器。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:143 +msgid "" +"Provide **three distinct event fds** (store / lookup / load). The " +"controllers build ``fd -> adapter`` maps; duplicates will misroute " +"events." +msgstr "提供 **三个不同的事件 fd**（存储 / 查找 / 加载）。控制器构建 ``fd -> 适配器`` 映射；重复会导致事件错误路由。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:145 +msgid "" +"Be **thread-safe**: ``StoreController`` and ``PrefetchController`` call " +"adapter methods from different threads concurrently." +msgstr "确保 **线程安全**：``StoreController`` 和 ``PrefetchController`` 从不同线程并发调用适配器方法。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:147 +msgid "Accept ``**kwargs`` in ``__init__`` to stay forward-compatible." +msgstr "在``__init__``中接受``**kwargs``以保持向前兼容。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:149 +msgid "A plugin adapter class **should**:" +msgstr "插件适配器类 **应该**：" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:151 +msgid "" +"Create its own asyncio event loop and background thread if it needs async" +" I/O." +msgstr "如果需要异步 I/O，则应创建自己的 asyncio 事件循环和后台线程。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:152 +msgid "" +"Use ``create_event_notifier()`` from ``lmcache.v1.platform`` for the " +"three event fds (cross-platform: ``os.eventfd`` on Linux, ``os.pipe`` " +"fallback elsewhere)." +msgstr "使用 ``lmcache.v1.platform`` 中的 ``create_event_notifier()`` 来处理三个事件文件描述符（跨平台：在 Linux 上使用 ``os.eventfd``，在其他地方使用 ``os.pipe`` 作为后备）。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:154 +msgid "Clean up all resources (event fds, threads, connections) in ``close()``." +msgstr "在 ``close()`` 中清理所有资源（事件文件描述符、线程、连接）。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:157 +msgid "Loading Flow" +msgstr "加载流程" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:190 +msgid "Minimal Example" +msgstr "最小示例" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:220 +msgid "Launch via CLI:" +msgstr "通过 CLI 启动：" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:232 +msgid "Reference Implementation" +msgstr "参考实现" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:234 +msgid "" +"See ``examples/lmc_external_l2_adapter/`` for a complete, pip-installable" +" example plugin (``InMemoryL2Adapter``) that demonstrates:" +msgstr "请参阅 ``examples/lmc_external_l2_adapter/`` 以获取完整的、可通过 pip 安装的示例插件 (``InMemoryL2Adapter``)，该插件演示了：" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:237 +msgid "FIFO eviction with configurable capacity." +msgstr "可配置容量的 FIFO 逐出。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:238 +msgid "Simulated bandwidth delay for realistic testing." +msgstr "用于现实测试的模拟带宽延迟。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:239 +msgid "Background asyncio event loop with proper shutdown." +msgstr "后台 asyncio 事件循环及其正确关闭。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:240 +msgid "" +"Full test suite covering store, lookup, load, batch operations, and " +"eviction behavior." +msgstr "全面的测试套件，涵盖存储、查找、加载、批量操作和逐出行为。" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:243 +msgid "Additional Resources" +msgstr "附加资源" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:245 +msgid "" +"Plugin adapter source: " +"``lmcache/v1/distributed/l2_adapters/plugin_l2_adapter.py``" +msgstr "插件适配器源代码：``lmcache/v1/distributed/l2_adapters/plugin_l2_adapter.py``" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:246 +msgid "" +"Native plugin adapter: " +"``lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py``" +msgstr "原生插件适配器: ``lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py``" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:247 +msgid "" +"Design document: " +"``lmcache/v1/distributed/l2_adapters/design_docs/plugin.md``" +msgstr "设计文档：``lmcache/v1/distributed/l2_adapters/design_docs/plugin.md``" + +#: ../../source/developer_guide/extending_lmcache/storage_plugins.rst:248 +msgid "L2 adapter base interface: ``lmcache/v1/distributed/l2_adapters/base.py``" +msgstr "L2 适配器基础接口: ``lmcache/v1/distributed/l2_adapters/base.py``" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/integration.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/integration.po new file mode 100644 index 00000000000..6976c694457 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/integration.po @@ -0,0 +1,114 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/integration.rst:2 +msgid "Integration" +msgstr "集成" + +#: ../../source/developer_guide/integration.rst:4 +msgid "" +"LMCache acts as a caching middleware that sits between LLM inference " +"engines and storage systems, enabling:" +msgstr "LMCache充当一个缓存中间件，位于LLM推理引擎和存储系统之间，支持：" + +#: ../../source/developer_guide/integration.rst:6 +msgid "" +"**Transparent Cache Reuse**: Automatic detection and reuse of previously " +"computed KV caches" +msgstr "**透明缓存重用**：自动检测和重用先前计算的 KV 缓存" + +#: ../../source/developer_guide/integration.rst:7 +msgid "" +"**Performance Optimization**: 3×–10× reduction in time-to-first-token " +"(TTFT) for multi-round conversation and RAG." +msgstr "**性能优化**：在多轮对话和 RAG 中，首次令牌的时间（TTFT）减少 3 倍至 10 倍。" + +#: ../../source/developer_guide/integration.rst:8 +msgid "**Resource Efficiency**: Significant GPU cycle savings through cache reuse" +msgstr "**资源效率**：通过缓存重用显著节省显存周期" + +#: ../../source/developer_guide/integration.rst:9 +msgid "" +"**Scalable Architecture**: Support for single-node and multi-node " +"deployments" +msgstr "**可扩展架构**：支持单节点和多节点部署" + +#: ../../source/developer_guide/integration.rst:12 +msgid "Supported Engines" +msgstr "支持的引擎" + +#: ../../source/developer_guide/integration.rst:14 +msgid "LMCache currently supports integration with:" +msgstr "LMCache 目前支持与以下内容集成：" + +#: ../../source/developer_guide/integration.rst:16 +msgid "**vLLM**" +msgstr "**vLLM**" + +#: ../../source/developer_guide/integration.rst:17 +msgid "**SGLang**" +msgstr "**SGLang**" + +#: ../../source/developer_guide/integration.rst:18 +msgid "**TRT-LLM** (coming soon)" +msgstr "**TRT-LLM**（即将推出）" + +#: ../../source/developer_guide/integration.rst:21 +msgid "Integration with vLLM" +msgstr "与 vLLM 的集成" + +#: ../../source/developer_guide/integration.rst:23 +msgid "" +"When LMCache is integrated with vLLM, the inference pipeline is augmented" +" to lookup and inject cached KV chunks for any reused input content. The " +"sequence diagram below shows how a prompt request flows through vLLM with" +" LMCache:" +msgstr "当 LMCache 与 vLLM 集成时，推理管道会增强以查找并注入缓存的 KV 块，以便处理任何重复使用的输入内容。下面的序列图展示了提示请求如何通过 vLLM 和 LMCache 流动：" + +#: ../../source/developer_guide/integration.rst:44 +msgid "" +"**Cache Lookup**: Upon receiving a new prompt, vLLM (via the LMCache " +"connector) computes identifiers (e.g. hashes of token sequences) and " +"queries LMCache for matching KV cache chunks. If a cache hit occurs, " +"LMCache retrieves the KV chunk (potentially from CPU or disk) and returns" +" it to vLLM. vLLM then injects these KV tensors into the model's " +"attention cache instead of recomputing them from scratch." +msgstr "**缓存查找**：在接收到新的提示后，vLLM（通过 LMCache 连接器）计算标识符（例如，令牌序列的哈希值）并查询 LMCache 以获取匹配的 KV 缓存块。如果发生缓存命中，LMCache 将检索 KV 块（可能来自 CPU 或磁盘）并将其返回给 vLLM。然后，vLLM 将这些 KV 张量注入到模型的注意力缓存中，而不是从头开始重新计算它们。" + +#: ../../source/developer_guide/integration.rst:46 +msgid "" +"**Inference with Cache**: vLLM proceeds with inference. For any parts of " +"the prompt where no cache was available (cache miss), the model computes " +"new KV values as usual. Cached segments (e.g. previously seen text) are " +"skipped in computation, significantly reducing the time-to-first-token " +"(TTFT) and saving GPU cycles." +msgstr "**使用缓存的推理**：vLLM 继续进行推理。对于没有可用缓存的提示部分（缓存未命中），模型会像往常一样计算新的 KV 值。缓存的片段（例如，之前见过的文本）在计算中被跳过，显著减少了首次令牌时间（TTFT），并节省了显存周期。" + +#: ../../source/developer_guide/integration.rst:48 +msgid "" +"**Async Cache Write**: After processing the prompt, any newly generated " +"KV cache chunks (corresponding to this prompt's content) are handed off " +"to LMCache for storage. This put operation is done asynchronously (in the" +" background) so it doesn't delay the response. The response is returned " +"to the user promptly, and LMCache's background tasks will offload the new" +" KV data to CPU, disk, or other backends for future reuse." +msgstr "**异步缓存写入**：在处理完提示后，任何新生成的 KV 缓存块（对应于该提示的内容）都被交给 LMCache 进行存储。此放置操作是异步进行的（在后台），因此不会延迟响应。响应会及时返回给用户，而 LMCache 的后台任务将把新的 KV 数据卸载到 CPU、磁盘或其他后端以供将来重用。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/basic_check.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/basic_check.po new file mode 100644 index 00000000000..698830a73e1 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/basic_check.po @@ -0,0 +1,209 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/usage/basic_check.rst:2 +msgid "Basic Check Tool" +msgstr "基本检查工具" + +#: ../../source/developer_guide/usage/basic_check.rst:4 +msgid "" +"The LMCache Basic Check Tool is a testing and validation utility that " +"helps you verify your LMCache installation, configuration, and " +"functionality. It provides multiple testing modes to validate different " +"components of the LMCache system." +msgstr "LMCache 基本检查工具是一个测试和验证工具，帮助您验证 LMCache 的安装、配置和功能。它提供多种测试模式以验证 LMCache 系统的不同组件。" + +#: ../../source/developer_guide/usage/basic_check.rst:7 +msgid "Overview" +msgstr "概述" + +#: ../../source/developer_guide/usage/basic_check.rst:9 +msgid "The basic check tool (``lmcache.v1.basic_check``) is designed to:" +msgstr "基本检查工具（``lmcache.v1.basic_check``）旨在：" + +#: ../../source/developer_guide/usage/basic_check.rst:11 +msgid "Test remote backend connectivity and functionality" +msgstr "测试远程后端的连接性和功能性" + +#: ../../source/developer_guide/usage/basic_check.rst:12 +msgid "Validate storage manager operations" +msgstr "验证存储管理器操作" + +#: ../../source/developer_guide/usage/basic_check.rst:13 +msgid "Generate test keys for performance testing" +msgstr "生成用于性能测试的测试键" + +#: ../../source/developer_guide/usage/basic_check.rst:14 +msgid "Verify configuration settings" +msgstr "验证配置设置" + +#: ../../source/developer_guide/usage/basic_check.rst:15 +msgid "Provide diagnostic information for troubleshooting" +msgstr "提供故障排除的诊断信息" + +#: ../../source/developer_guide/usage/basic_check.rst:18 +msgid "Available Check Modes" +msgstr "可用检查模式" + +#: ../../source/developer_guide/usage/basic_check.rst:20 +msgid "" +"The tool supports several check modes, each targeting specific " +"functionality:" +msgstr "该工具支持几种检查模式，每种模式针对特定功能：" + +#: ../../source/developer_guide/usage/basic_check.rst:23 +msgid "test_remote" +msgstr "test_remote" + +#: ../../source/developer_guide/usage/basic_check.rst:25 +msgid "Tests the remote backend functionality including:" +msgstr "测试远程后端功能，包括：" + +#: ../../source/developer_guide/usage/basic_check.rst:27 +msgid "Connection establishment to remote backends (fs, etc.)" +msgstr "与远程后端（文件系统等）的连接建立" + +#: ../../source/developer_guide/usage/basic_check.rst:28 +msgid "put/get operations with data integrity validation" +msgstr "带数据完整性验证的 put/get 操作" + +#: ../../source/developer_guide/usage/basic_check.rst:29 +msgid "put/get/exists operations with performance reports" +msgstr "带性能报告的 put/get/exists 操作" + +#: ../../source/developer_guide/usage/basic_check.rst:31 +#: ../../source/developer_guide/usage/basic_check.rst:46 +#: ../../source/developer_guide/usage/basic_check.rst:62 +msgid "**Usage:**" +msgstr "**用法：**" + +#: ../../source/developer_guide/usage/basic_check.rst:38 +msgid "test_storage_manager" +msgstr "test_storage_manager" + +#: ../../source/developer_guide/usage/basic_check.rst:40 +msgid "Tests the storage manager operations including:" +msgstr "测试存储管理器操作，包括：" + +#: ../../source/developer_guide/usage/basic_check.rst:42 +msgid "Configuration validation" +msgstr "配置验证" + +#: ../../source/developer_guide/usage/basic_check.rst:43 +msgid "batched_put/get operations with data integrity validation" +msgstr "带数据完整性验证的批量放入/获取操作" + +#: ../../source/developer_guide/usage/basic_check.rst:44 +msgid "batched_put/get/contains operations with performance reports" +msgstr "带批量放入/获取/包含操作的性能报告" + +#: ../../source/developer_guide/usage/basic_check.rst:53 +msgid "gen (Key Generation)" +msgstr "键生成 (Key Generation)" + +#: ../../source/developer_guide/usage/basic_check.rst:55 +msgid "Generates test keys for performance testing and benchmarking:" +msgstr "生成用于性能测试和基准测试的测试密钥：" + +#: ../../source/developer_guide/usage/basic_check.rst:57 +msgid "Configurable number of keys and concurrency levels" +msgstr "可配置的键数量和并发级别" + +#: ../../source/developer_guide/usage/basic_check.rst:58 +msgid "Memory-efficient batch processing" +msgstr "内存高效的批处理" + +#: ../../source/developer_guide/usage/basic_check.rst:59 +msgid "Progress tracking and performance metrics" +msgstr "进度跟踪和性能指标" + +#: ../../source/developer_guide/usage/basic_check.rst:60 +msgid "Offset support for distributed testing" +msgstr "分布式测试的偏移支持" + +#: ../../source/developer_guide/usage/basic_check.rst:69 +msgid "Command Line Interface" +msgstr "命令行接口" + +#: ../../source/developer_guide/usage/basic_check.rst:72 +msgid "Basic Usage" +msgstr "基本用法" + +#: ../../source/developer_guide/usage/basic_check.rst:79 +msgid "List Available Modes" +msgstr "列出可用模式" + +#: ../../source/developer_guide/usage/basic_check.rst:86 +msgid "Command Line Options" +msgstr "命令行选项" + +#: ../../source/developer_guide/usage/basic_check.rst:90 +msgid "**Required.** Operation mode to run. Use ``list`` to see available modes." +msgstr "**必需。** 操作模式。使用 ``list`` 查看可用模式。" + +#: ../../source/developer_guide/usage/basic_check.rst:94 +msgid "" +"Model name for testing, just a part of key of persist kv-cache. Default: " +"``/lmcache_test_model/``" +msgstr "用于测试的模型名称，仅为持久 KV Cache 的键的一部分。默认值：``/lmcache_test_model/``" + +#: ../../source/developer_guide/usage/basic_check.rst:98 +msgid "Number of keys to generate (gen mode only). Default: 100" +msgstr "生成的键的数量（仅限生成模式）。默认值：100" + +#: ../../source/developer_guide/usage/basic_check.rst:102 +msgid "Concurrency level for operations (gen mode only). Default: 16" +msgstr "操作的并发级别（仅限生成模式）。默认值：16" + +#: ../../source/developer_guide/usage/basic_check.rst:106 +msgid "Offset for key generation (gen mode only). Default: 0" +msgstr "密钥生成的偏移量（仅限生成模式）。默认值：0" + +#: ../../source/developer_guide/usage/basic_check.rst:109 +msgid "Configuration" +msgstr "配置" + +#: ../../source/developer_guide/usage/basic_check.rst:111 +msgid "" +"The basic check tool uses your existing LMCache configuration. You can " +"specify configuration in several ways:" +msgstr "基本检查工具使用您现有的 LMCache 配置。您可以通过几种方式指定配置：" + +#: ../../source/developer_guide/usage/basic_check.rst:114 +msgid "Environment Variable" +msgstr "环境变量" + +#: ../../source/developer_guide/usage/basic_check.rst:122 +msgid "Example Configuration" +msgstr "示例配置" + +#: ../../source/developer_guide/usage/basic_check.rst:124 +msgid "Here's an example configuration optimized for basic checks:" +msgstr "这是一个针对基本检查优化的示例配置：" + +#: ../../source/developer_guide/usage/basic_check.rst:137 +msgid "Examples" +msgstr "示例" + +#: ../../source/developer_guide/usage/basic_check.rst:139 +msgid "The ``examples/basic_check/`` directory contains comprehensive examples:" +msgstr "``examples/basic_check/`` 目录包含全面的示例：" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/index.po new file mode 100644 index 00000000000..589b6ef1863 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/index.po @@ -0,0 +1,25 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/usage/index.rst:2 +msgid "Usage Data Module" +msgstr "使用数据模块" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/usage_stats_collection.po b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/usage_stats_collection.po new file mode 100644 index 00000000000..9dad95f97ca --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/usage_stats_collection.po @@ -0,0 +1,123 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:4 +msgid "Usage Stats Collection" +msgstr "使用统计收集" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:6 +msgid "" +"LMCache collects anonymous usage data by default to help the engineering " +"team understand real-world workloads, prioritize optimizations, and " +"improve reliability. All collected data is aggregated and contains no " +"sensitive user information." +msgstr "LMCache 默认收集匿名使用数据，以帮助工程团队了解实际工作负载，优先优化并提高可靠性。所有收集的数据都是汇总的，并不包含敏感用户信息。" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:8 +msgid "" +"A sanitized subset of the aggregated data may be publicly released for " +"the community’s benefit (for example, see a daily usage report `here " +"`_)." +msgstr "聚合数据的一个经过清洗的子集可能会公开发布，以惠及社区（例如，查看每日使用报告 `here `_）。" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:11 +msgid "What data is collected?" +msgstr "收集了哪些数据？" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:13 +msgid "" +"Usage stats are emitted as three message types, implemented in " +"``usage_context.py``:" +msgstr "使用统计信息以三种消息类型发出，具体实现位于 ``usage_context.py``：" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:15 +msgid "" +"**EnvMessage** Captures environment details such as cloud provider, CPU " +"info, total memory, architecture, GPU count/type, and execution source." +msgstr "**EnvMessage** 捕获环境细节，例如云提供商、CPU 信息、总内存、架构、GPU 数量/类型和执行源。" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:18 +msgid "" +"**EngineMessage** Records engine configuration and metadata, including " +"cache settings (chunk size, local device, cache limits), remote backend " +"parameters, blending settings, model name, world size, and KV-cache " +"dtype/shape." +msgstr "**EngineMessage** 记录引擎配置和元数据，包括缓存设置（块大小、本地设备、缓存限制）、远程后端参数、混合设置、模型名称、全局大小，以及 KV Cache 的数据类型/形状。" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:21 +msgid "" +"**MetadataMessage** Reports execution metadata: the timestamp when the " +"run started and total duration in seconds." +msgstr "**MetadataMessage** 报告执行元数据：运行开始的时间戳和总持续时间（以秒为单位）。" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:24 +msgid "" +"These messages are serialized to JSON and POSTed to the LMCache usage " +"server." +msgstr "这些消息被序列化为 JSON 并通过 POST 发送到 LMCache 使用服务器。" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:27 +msgid "Example JSON payload" +msgstr "示例 JSON 有效负载" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:47 +msgid "Previewing collected data" +msgstr "预览收集的数据" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:49 +msgid "" +"If you enable **local logging**, usage messages are appended to your " +"specified log file. To inspect the most recent entries:" +msgstr "如果您启用 **本地日志记录**，使用消息将附加到您指定的日志文件中。要检查最近的条目：" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:56 +msgid "Configuration & Opt-out" +msgstr "配置与选择退出" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:58 +msgid "" +"By default, usage tracking is **enabled**. To disable all usage stats " +"collection, set the environment variable:" +msgstr "默认情况下，使用情况跟踪是 **启用** 的。要禁用所有使用统计信息收集，请设置环境变量：" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:64 +msgid "" +"When ``LMCACHE_TRACK_USAGE`` is set to ``false``, " +"``InitializeUsageContext`` will return ``None`` and no data will be sent " +"or logged." +msgstr "当 ``LMCACHE_TRACK_USAGE`` 设置为 ``false`` 时，``InitializeUsageContext`` 将返回 ``None``，并且不会发送或记录任何数据。" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:67 +msgid "Local logging" +msgstr "本地日志记录" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:69 +msgid "" +"If you would like to log to a file in addition to (or instead of) sending" +" data to the server, pass a local-log path when initializing:" +msgstr "如果您希望在将数据发送到服务器的同时（或替代）记录到文件中，请在初始化时传递一个本地日志路径：" + +#: ../../source/developer_guide/usage/usage_stats_collection.rst:81 +msgid "" +"Omitting the ``local_log`` argument (or passing ``None``) disables local " +"file logging." +msgstr "省略 ``local_log`` 参数（或传递 ``None``）将禁用本地文件日志记录。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/1p1d.po b/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/1p1d.po new file mode 100644 index 00000000000..d1604fc87ab --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/1p1d.po @@ -0,0 +1,346 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:2 +msgid "1p1d" +msgstr "1p1d" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:5 +msgid "One Prefiller, One Decoder (1p1d) Example" +msgstr "一个 Prefill 组件，一个解码器 (1p1d) 示例" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:7 +msgid "" +"This example demonstrates how to run LMCache with disaggregated prefill " +"using NIXL on a single node with a 1 prefiller + 1 decoder setup. This " +"configuration separates the compute-intensive prefill operations from the" +" decode operations, allowing for better resource utilization and " +"performance optimization." +msgstr "此示例演示了如何在单个节点上使用 NIXL 以分离式 Prefill 运行 LMCache，配置为 1 个 Prefiller + 1 个 Decoder。此配置将计算密集型的 Prefill 操作与解码操作分离，从而实现更好的资源利用和性能优化。" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:10 +msgid "Architecture Overview" +msgstr "架构概述" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:12 +msgid "The 1p1d setup consists of three main components:" +msgstr "1p1d 设置由三个主要组件组成：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:14 +msgid "" +"**Prefiller Server** - Handles the prefill phase of inference (initial " +"prompt processing)" +msgstr "**Prefill 服务器** - 处理推理的预填充阶段（初始提示处理）" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:15 +msgid "" +"**Decoder Server** - Handles the decode phase of inference (token " +"generation)" +msgstr "**解码服务器** - 处理推理的解码阶段（令牌生成）" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:16 +msgid "**Proxy Server** - Coordinates requests between the prefiller and decoder" +msgstr "**代理服务器** - 协调预填充器和解码器之间的请求" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:40 +msgid "Prerequisites" +msgstr "先决条件" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:42 +msgid "**LMCache**: Install with ``pip install lmcache``" +msgstr "**LMCache**: 使用 ``pip install lmcache`` 安装" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:43 +msgid "" +"**NIXL**: Install from `NIXL GitHub repository `_" +msgstr "**NIXL**: 从 `NIXL GitHub 仓库 `_ 安装" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:44 +msgid "**Hardware**: At least 2 GPUs" +msgstr "**硬件**：至少需要 2 个 GPU" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:45 +msgid "" +"**Model Access**: Valid Hugging Face token (HF_TOKEN) for Llama 3.1 8B " +"Instruct" +msgstr "**模型访问**：有效的 Hugging Face 令牌 (HF_TOKEN) 用于 Llama 3.1 8B Instruct" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:48 +msgid "Quick Start" +msgstr "快速开始" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:50 +msgid "**Set your Hugging Face token**:" +msgstr "**设置您的 Hugging Face 令牌**：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:56 +msgid "**Navigate to the example directory**:" +msgstr "**导航到示例目录**：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:62 +msgid "**Run the example**:" +msgstr "**运行示例**:" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:68 +msgid "The script will automatically:" +msgstr "该脚本将自动：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:70 +msgid "Launch a prefiller instance on port 7100 (GPU 0)" +msgstr "在端口 7100（GPU 0）上启动一个 Prefill 实例" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:71 +msgid "Launch a decoder instance on port 7200 (GPU 1)" +msgstr "在端口 7200 上启动解码器实例（GPU 1）" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:72 +msgid "Launch a proxy server on port 9100" +msgstr "在 9100 端口启动一个代理服务器" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:73 +msgid "Wait for all servers to be ready" +msgstr "等待所有服务器准备就绪" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:75 +msgid "Press ``Ctrl+C`` to stop all servers." +msgstr "按 ``Ctrl+C`` 停止所有服务器。" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:78 +msgid "Configuration" +msgstr "配置" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:80 +msgid "" +"**Important**: For correct KV cache transfer, ensure all processes use " +"the same ``PYTHONHASHSEED`` to keep the hash of the KV cache consistent " +"across processes:" +msgstr "**重要**：为了正确的 KV Cache 传输，请确保所有进程使用相同的 ``PYTHONHASHSEED`` 以保持 KV Cache 在进程间的一致性：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:87 +msgid "Prefiller Configuration" +msgstr "分离式 Prefill 配置" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:89 +msgid "The prefiller is configured via ``configs/lmcache-prefiller-config.yaml``:" +msgstr "预填充器通过 ``configs/lmcache-prefiller-config.yaml`` 进行配置：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:105 +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:131 +msgid "Key settings:" +msgstr "关键设置：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:107 +msgid "``pd_role: \"sender\"`` - Configures this instance to send KV cache data" +msgstr "``pd_role: \"sender\"`` - 配置此实例以发送 KV Cache 数据" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:108 +msgid "``pd_buffer_size: 1073741824 # 1GB`` - Buffer size for transfers" +msgstr "``pd_buffer_size: 1073741824 # 1GB`` - 传输的缓冲区大小" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:109 +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:135 +msgid "``pd_buffer_device: \"cuda\"`` - Uses GPU memory for buffering" +msgstr "``pd_buffer_device: \"cuda\"`` - 使用显存进行缓冲" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:112 +msgid "Decoder Configuration" +msgstr "解码器配置" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:114 +msgid "The decoder is configured via ``configs/lmcache-decoder-config.yaml``:" +msgstr "解码器通过 ``configs/lmcache-decoder-config.yaml`` 进行配置：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:133 +msgid "" +"``pd_role: \"receiver\"`` - Configures this instance to receive KV cache " +"data" +msgstr "``pd_role: \"receiver\"`` - 配置此实例以接收 KV Cache 数据" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:134 +msgid "``pd_buffer_size: 2147483648 # 2GB`` - Buffer size for transfers" +msgstr "``pd_buffer_size: 2147483648 # 2GB`` - 传输的缓冲区大小" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:138 +msgid "Components Deep Dive" +msgstr "组件深入分析" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:141 +msgid "Proxy Server (disagg_proxy_server.py)" +msgstr "代理服务器 (disagg_proxy_server.py)" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:143 +msgid "The proxy server coordinates the disaggregated prefill workflow:" +msgstr "代理服务器协调分离式 Prefill 工作流：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:145 +msgid "**Request Handling**: Receives client requests on port 9100" +msgstr "**请求处理**：在 9100 端口接收客户端请求" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:146 +msgid "" +"**Prefill Coordination**: Sends requests to the prefiller with " +"``max_tokens=1``" +msgstr "**分离式 Prefill 协调**: 发送请求给 Prefill 模块，``max_tokens=1``" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:147 +msgid "**Prefill Response**: Receives prefiller that says nixl transfer is done" +msgstr "**Prefill Response**: 接收预填充器，表示 nixl 转移已完成" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:148 +msgid "**Response Streaming**: Streams the full response from the decoder" +msgstr "**响应流**：从解码器流式传输完整响应" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:149 +msgid "**Performance Monitoring**: Tracks Time-To-First-Token (TTFT) statistics" +msgstr "**性能监控**：跟踪首次令牌时间（TTFT）统计信息" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:151 +msgid "Supported endpoints:" +msgstr "支持的端点：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:153 +msgid "``/v1/completions``" +msgstr "``/v1/completions``" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:154 +msgid "``/v1/chat/completions``" +msgstr "``/v1/chat/completions``" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:157 +msgid "vLLM Server Launcher (disagg_vllm_launcher.sh)" +msgstr "vLLM 服务器启动器 (disagg_vllm_launcher.sh)" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:159 +msgid "" +"This script launches individual vLLM servers with appropriate " +"configurations:" +msgstr "此脚本启动具有适当配置的单个 vLLM 服务器：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:161 +msgid "**Prefiller Launch Command**:" +msgstr "**Prefill 启动命令**:" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:178 +msgid "**Decoder Launch Command**:" +msgstr "**解码器启动命令**:" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:196 +msgid "Testing and Benchmarking" +msgstr "测试与基准测试" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:199 +msgid "Basic Test" +msgstr "基本测试" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:201 +msgid "Once all servers are running, you can test with a simple curl command:" +msgstr "一旦所有服务器都在运行，您可以使用简单的 curl 命令进行测试：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:213 +msgid "Performance Benchmarking" +msgstr "性能基准测试" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:215 +msgid "For comprehensive performance testing, use vLLM's benchmark tool:" +msgstr "要进行全面的性能测试，请使用 vLLM 的基准测试工具：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:224 +msgid "" +"This benchmark: - Sends requests to port 9100 (proxy server) - Uses " +"random prompts with 7500 input tokens - Generates 200 output tokens per " +"request - Tests with 30 total prompts at 1 request/second" +msgstr "此基准测试： - 向 9100 端口（代理服务器）发送请求 - 使用 7500 个输入 token 的随机提示 - 每个请求生成 200 个输出 token - 使用 30 个总提示以 1 请求/秒的速度进行测试" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:231 +msgid "Log Files and Monitoring" +msgstr "日志文件和监控" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:233 +msgid "The example generates three log files for monitoring:" +msgstr "该示例生成三个用于监控的日志文件：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:235 +msgid "``prefiller.log`` - Prefiller server logs and errors" +msgstr "``prefiller.log`` - Prefill 服务器日志和错误" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:236 +msgid "``decoder.log`` - Decoder server logs and errors" +msgstr "``decoder.log`` - 解码器服务器日志和错误" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:237 +msgid "``proxy.log`` - Proxy server logs and TTFT statistics" +msgstr "``proxy.log`` - 代理服务器日志和 TTFT 统计信息" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:239 +msgid "" +"The proxy server automatically calculates and displays TTFT statistics " +"every 5 seconds:" +msgstr "代理服务器每 5 秒自动计算并显示 TTFT 统计信息：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:252 +msgid "Troubleshooting" +msgstr "故障排除" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:255 +msgid "Common Issues" +msgstr "常见问题" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:257 +msgid "**GPU Memory**: Ensure each GPU has sufficient memory for the model" +msgstr "**显存**：确保每个 GPU 具有足够的内存来支持模型" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:258 +msgid "**NIXL Installation**: Verify NIXL is properly installed and accessible" +msgstr "**NIXL 安装**: 验证 NIXL 是否正确安装并可访问" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:259 +msgid "**Port Conflicts**: Check that ports 7100, 7200, and 9000 are available" +msgstr "**端口冲突**：检查端口 7100、7200 和 9000 是否可用" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:260 +msgid "**HF Token**: Ensure your Hugging Face token has access to Llama models" +msgstr "**HF Token**: 确保您的 Hugging Face 令牌可以访问 Llama 模型" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:263 +msgid "Error Recovery" +msgstr "错误恢复" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:265 +msgid "If any server fails to start:" +msgstr "如果任何服务器无法启动：" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:267 +msgid "Check the corresponding log file for error details" +msgstr "检查相应的日志文件以获取错误详情" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:268 +msgid "Verify GPU availability with ``nvidia-smi``" +msgstr "使用 ``nvidia-smi`` 验证 GPU 可用性" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:269 +msgid "Ensure all dependencies are installed" +msgstr "确保所有依赖项已安装" + +#: ../../source/disaggregated_prefill/nixl/1p1d.rst:270 +msgid "Try restarting with ``Ctrl+C`` followed by re-running the script" +msgstr "尝试使用 ``Ctrl+C`` 重启，然后重新运行脚本。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/index.po new file mode 100644 index 00000000000..4592af04bd7 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/index.po @@ -0,0 +1,52 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/disaggregated_prefill/nixl/index.rst:2 +msgid "Using NIXL" +msgstr "使用 NIXL" + +#: ../../source/disaggregated_prefill/nixl/index.rst:4 +msgid "" +"NIXL (NVIDIA Inference Xfer Library) is a high-performance library " +"designed for accelerating point to point communications in AI inference " +"frameworks. It provides an abstraction over various types of memory (CPU " +"and GPU) and storage through a modular plug-in architecture, enabling " +"efficient data transfer and coordination between different components of " +"the inference pipeline." +msgstr "NIXL（NVIDIA 推理传输库）是一个高性能库，旨在加速 AI 推理框架中的点对点通信。它通过模块化插件架构提供对各种类型内存（CPU 和 GPU）和存储的抽象，使得推理管道中不同组件之间的数据传输和协调更加高效。" + +#: ../../source/disaggregated_prefill/nixl/index.rst:7 +msgid "" +"LMCache supports using NIXL as the underlying communication library for " +"prefill-decode disaggregation." +msgstr "LMCache 支持将 NIXL 作为分离式 Prefill-解码的底层通信库。" + +#: ../../source/disaggregated_prefill/nixl/index.rst:9 +msgid "" +"For detailed installation instructions of LMCache with NIXL, please refer" +" to our `installation guide " +"`_." +msgstr "有关使用 NIXL 安装 LMCache 的详细说明，请参阅我们的 `安装指南 `_。" + +#: ../../source/disaggregated_prefill/nixl/index.rst:12 +msgid "Examples" +msgstr "示例" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/xpyd.po b/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/xpyd.po new file mode 100644 index 00000000000..aac7a14ea3b --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/xpyd.po @@ -0,0 +1,403 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:2 +msgid "XpYd" +msgstr "XpYd" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:5 +msgid "X Prefiller, Y Decoder (XpYd) Example" +msgstr "X 预填充器，Y 解码器 (XpYd) 示例" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:7 +msgid "" +"This example demonstrates how to run LMCache with disaggregated prefill " +"using NIXL on a single node with multiple prefiller and decoder " +"instances. This configuration allows for horizontal scaling of both the " +"compute-intensive prefill operations and the decode operations, enabling " +"better resource utilization and higher throughput." +msgstr "此示例演示了如何在单个节点上使用 NIXL 运行带有分离式 Prefill 的 LMCache，使用多个 Prefiller 和 Decoder 实例。此配置允许对计算密集型的 Prefill 操作和解码操作进行横向扩展，从而实现更好的资源利用和更高的吞吐量。" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:10 +msgid "Architecture Overview" +msgstr "架构概述" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:12 +msgid "" +"The XpYd setup consists of multiple components that can be scaled " +"independently:" +msgstr "XpYd 设置由多个可以独立扩展的组件组成：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:14 +msgid "" +"**Multiple Prefiller Servers** - Handle the prefill phase of inference " +"(initial prompt processing)" +msgstr "**多个 Prefiller 服务器** - 处理推理的预填充阶段（初始提示处理）" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:15 +msgid "" +"**Multiple Decoder Servers** - Handle the decode phase of inference " +"(token generation)" +msgstr "**多个解码器服务器** - 处理推理的解码阶段（令牌生成）" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:16 +msgid "" +"**Proxy Server** - Coordinates requests between prefillers and decoders " +"using round-robin load balancing" +msgstr "**代理服务器** - 使用轮询负载均衡协调预填充器和解码器之间的请求" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:18 +msgid "Example 2p2d Architecture:" +msgstr "示例 2p2d 架构：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:49 +msgid "Prerequisites" +msgstr "先决条件" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:51 +msgid "**LMCache**: Install with ``pip install lmcache``" +msgstr "**LMCache**: 使用 ``pip install lmcache`` 安装" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:52 +msgid "" +"**NIXL**: Install from `NIXL GitHub repository `_" +msgstr "**NIXL**: 从 `NIXL GitHub 仓库 `_ 安装" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:53 +msgid "" +"**Hardware**: At least 4 GPUs (2 for prefillers + 2 for decoders in 2p2d " +"setup)" +msgstr "**硬件**：至少需要 4 个 GPU（2 个用于 Prefillers + 2 个用于 2p2d 设置中的解码器）" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:54 +msgid "" +"**Model Access**: Valid Hugging Face token (HF_TOKEN) for Llama 3.1 8B " +"Instruct" +msgstr "**模型访问**：有效的 Hugging Face 令牌 (HF_TOKEN) 用于 Llama 3.1 8B Instruct" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:57 +msgid "Quick Start (2p2d Example)" +msgstr "快速开始 (2p2d 示例)" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:59 +msgid "**Set your Hugging Face token**:" +msgstr "**设置你的 Hugging Face 令牌**：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:65 +msgid "**Navigate to the example directory**:" +msgstr "**导航到示例目录**：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:71 +msgid "**Run the example**:" +msgstr "**运行示例**:" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:77 +msgid "The script will automatically:" +msgstr "该脚本将自动：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:79 +msgid "Launch two decoder instances on port 8200 and 8201 (GPU 2 and GPU 3)" +msgstr "在端口 8200 和 8201 启动两个解码器实例（GPU 2 和 GPU 3）" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:80 +msgid "Launch two prefiller instances on ports 7100 and 7101 (GPU 0 and GPU 1)" +msgstr "在端口 7100 和 7101 上启动两个 Prefill 实例（GPU 0 和 GPU 1）" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:81 +msgid "Launch a proxy server on port 9100 with round-robin load balancing" +msgstr "在9100端口启动一个代理服务器，并使用轮询负载均衡。" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:82 +msgid "Wait for all servers to be ready" +msgstr "等待所有服务器准备就绪" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:84 +msgid "Press ``Ctrl+C`` to stop all servers." +msgstr "按 ``Ctrl+C`` 停止所有服务器。" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:87 +msgid "Configuration" +msgstr "配置" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:89 +msgid "" +"**Important**: For correct KV cache transfer, ensure all processes use " +"the same ``PYTHONHASHSEED`` to keep the hash of the KV cache consistent " +"across processes:" +msgstr "**重要**：为了正确的 KV Cache 传输，确保所有进程使用相同的 ``PYTHONHASHSEED`` 以保持 KV Cache 在进程间的一致性：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:96 +msgid "Prefiller Configuration" +msgstr "分离式 Prefill 配置" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:98 +msgid "" +"All prefillers share the same configuration via ``configs/lmcache-" +"prefiller-config.yaml``:" +msgstr "所有 Prefiller 通过 ``configs/lmcache-prefiller-config.yaml`` 共享相同的配置：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:114 +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:140 +msgid "Key settings:" +msgstr "关键设置：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:116 +msgid "``pd_role: \"sender\"`` - Configures these instances to send KV cache data" +msgstr "``pd_role: \"sender\"`` - 配置这些实例以发送 KV Cache 数据" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:117 +msgid "" +"``pd_buffer_size: 1073741824 # 1GB`` - Upper bound of PD transport buffer" +" size (in bytes), aligned to chunk size" +msgstr "``pd_buffer_size: 1073741824 # 1GB`` - PD 传输缓冲区大小的上限（以字节为单位），与块大小对齐" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:118 +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:144 +msgid "``pd_buffer_device: \"cuda\"`` - Uses GPU memory for buffering" +msgstr "``pd_buffer_device: \"cuda\"`` - 使用显存进行缓冲" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:121 +msgid "Decoder Configuration" +msgstr "解码器配置" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:123 +msgid "" +"The decoder(s) are configured via ``configs/lmcache-" +"decoder-x-config.yaml``:" +msgstr "解码器通过 ``configs/lmcache-decoder-x-config.yaml`` 进行配置：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:142 +msgid "" +"``pd_role: \"receiver\"`` - Configures these instances to receive KV " +"cache data" +msgstr "``pd_role: \"receiver\"`` - 配置这些实例以接收 KV Cache 数据" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:143 +msgid "" +"``pd_buffer_size: 2147483648 # 2GB`` - Upper bound of PD transport buffer" +" size (in bytes), aligned to chunk size" +msgstr "``pd_buffer_size: 2147483648 # 2GB`` - PD 传输缓冲区大小的上限（以字节为单位），与块大小对齐" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:147 +msgid "Components Deep Dive" +msgstr "组件深入分析" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:150 +msgid "Proxy Server (disagg_proxy_server.py)" +msgstr "代理服务器 (disagg_proxy_server.py)" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:152 +msgid "The proxy server coordinates the multi-prefiller disaggregated workflow:" +msgstr "代理服务器协调多预填充器的分离式工作流程：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:154 +msgid "**Request Handling**: Receives client requests on port 9000" +msgstr "**请求处理**：在 9000 端口接收客户端请求" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:155 +msgid "" +"**Load Balancing**: Distributes requests across multiple prefillers using" +" round-robin" +msgstr "**负载均衡**：使用轮询将请求分配到多个预填充器" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:156 +msgid "" +"**Prefill Coordination**: Sends requests to prefillers with " +"``max_tokens=1``" +msgstr "**Prefill Coordination**: 将请求发送到使用 ``max_tokens=1`` 的 Prefill 处理器" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:157 +msgid "**Prefill Response**: Receives prefiller that says nixl transfer is done" +msgstr "**Prefill Response**: 接收预填充器，表示 nixl 转移已完成" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:158 +msgid "**Response Streaming**: Streams the full response from the decoder" +msgstr "**响应流**：从解码器流式传输完整响应" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:159 +msgid "**Performance Monitoring**: Tracks Time-To-First-Token (TTFT) statistics" +msgstr "**性能监控**: 跟踪首次令牌时间（TTFT）统计信息" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:161 +msgid "" +"Key features: - **Round-robin distribution**: Balances load across " +"``--num-prefillers`` instances - **Fault tolerance**: Handles prefiller " +"failures gracefully - **Monitoring**: Provides detailed TTFT statistics " +"for each prefiller" +msgstr "主要特性：- **轮询分配**：在 ``--num-prefillers`` 实例之间平衡负载 - **容错**：优雅地处理预填充器故障 - **监控**：为每个预填充器提供详细的 TTFT 统计信息" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:166 +msgid "Supported endpoints: - ``/v1/completions`` - ``/v1/chat/completions``" +msgstr "支持的端点： - ``/v1/completions`` - ``/v1/chat/completions``" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:171 +msgid "vLLM Server Launcher (disagg_vllm_launcher.sh)" +msgstr "vLLM 服务器启动器 (disagg_vllm_launcher.sh)" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:173 +msgid "" +"This script launches individual vLLM servers with appropriate " +"configurations:" +msgstr "此脚本启动具有适当配置的单个 vLLM 服务器：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:175 +msgid "**Prefiller1 Launch Command**:" +msgstr "**Prefiller1 启动命令**:" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:192 +msgid "**Prefiller2 Launch Command**:" +msgstr "**Prefiller2 启动命令**:" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:209 +msgid "**Decoder1 Launch Command**:" +msgstr "**解码器1 启动命令**:" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:226 +msgid "**Decoder2 Launch Command**:" +msgstr "**Decoder2 启动命令**:" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:243 +msgid "" +"Key differences from 1p1d: - Each prefiller gets a unique " +"``lmcache_rpc_port`` (producer1, producer2, etc.) - Each prefiller runs " +"on a different GPU (CUDA_VISIBLE_DEVICES) - Different ports for each " +"prefiller (7100, 7101, etc.) - Different ports for each decoder (7200, " +"7201, etc.)" +msgstr "与 1p1d 的主要区别： - 每个 Prefiller 获取一个唯一的 ``lmcache_rpc_port``（producer1、producer2 等） - 每个 Prefiller 在不同的 GPU 上运行（CUDA_VISIBLE_DEVICES） - 每个 Prefiller 使用不同的端口（7100、7101 等） - 每个解码器使用不同的端口（7200、7201 等）" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:250 +msgid "Basic Test" +msgstr "基本测试" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:252 +msgid "Once all servers are running, you can test with a simple curl command:" +msgstr "一旦所有服务器都在运行，您可以使用简单的 curl 命令进行测试：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:264 +msgid "Performance Benchmarking" +msgstr "性能基准测试" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:266 +msgid "For comprehensive performance testing, use vLLM's benchmark tool:" +msgstr "要进行全面的性能测试，请使用 vLLM 的基准测试工具：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:275 +msgid "" +"Expected performance improvements with 2p2d: - **Higher throughput**: " +"Multiple prefillers can handle more concurrent requests - **Better " +"TTFT**: Load balancing reduces queuing delays - **Improved utilization**:" +" Better GPU utilization across multiple devices" +msgstr "预期的性能提升与 2p2d 相关：- **更高的吞吐量**：多个预填充器可以处理更多的并发请求 - **更好的 TTFT**：负载均衡减少排队延迟 - **更好的利用率**：多个设备之间更好的 GPU 利用率" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:280 +msgid "Sample benchmark results:" +msgstr "示例基准测试结果：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:307 +msgid "Log Files and Monitoring" +msgstr "日志文件和监控" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:309 +msgid "The example generates multiple log files for comprehensive monitoring:" +msgstr "该示例生成多个日志文件以进行全面监控：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:311 +msgid "``prefiller1.log`` - First prefiller server logs and errors" +msgstr "``prefiller1.log`` - 第一个 Prefill 服务器的日志和错误" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:312 +msgid "``prefiller2.log`` - Second prefiller server logs and errors" +msgstr "``prefiller2.log`` - 第二个 Prefill 服务器日志和错误" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:313 +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:314 +msgid "``decoder1.log`` - First decoder server logs and errors" +msgstr "``decoder1.log`` - 第一个解码器服务器日志和错误" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:315 +msgid "``proxy.log`` - Proxy server logs and TTFT statistics" +msgstr "``proxy.log`` - 代理服务器日志和 TTFT 统计信息" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:317 +msgid "The proxy server provides detailed statistics for each prefiller:" +msgstr "代理服务器为每个 Prefiller 提供详细的统计信息：" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:333 +msgid "" +"This helps identify performance differences between prefiller instances " +"and optimize load balancing." +msgstr "这有助于识别不同 Prefill 实例之间的性能差异并优化负载均衡。" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:336 +msgid "Troubleshooting" +msgstr "故障排除" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:339 +msgid "Common Issues" +msgstr "常见问题" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:341 +msgid "**GPU Memory**: Ensure each GPU has sufficient memory for the model" +msgstr "**显存**：确保每个 GPU 具有足够的内存以支持模型" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:342 +msgid "**NIXL Installation**: Verify NIXL is properly installed and accessible" +msgstr "**NIXL 安装**: 验证 NIXL 是否正确安装并可访问" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:343 +msgid "**Port Conflicts**: Check that all required ports are available" +msgstr "**端口冲突**：检查所有必需的端口是否可用" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:344 +msgid "**HF Token**: Ensure your Hugging Face token has access to Llama models" +msgstr "**HF Token**: 确保您的 Hugging Face 令牌可以访问 Llama 模型" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:345 +msgid "**GPU Assignment**: Verify CUDA_VISIBLE_DEVICES assignments don't conflict" +msgstr "**GPU 分配**: 验证 CUDA_VISIBLE_DEVICES 的分配是否存在冲突" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:348 +msgid "Multi-Instance Specific Issues" +msgstr "多实例特定问题" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:350 +msgid "" +"**Uneven Load**: Monitor prefiller statistics to ensure balanced " +"distribution" +msgstr "**不均匀负载**：监控 Prefill 统计数据以确保负载均衡分配" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:351 +msgid "" +"**Resource Contention**: Watch for GPU memory pressure with multiple " +"instances" +msgstr "**资源争用**：监控多个实例的显存压力" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:352 +msgid "" +"**Network Bottlenecks**: Monitor NIXL transfer performance between " +"instances" +msgstr "**网络瓶颈**：监控实例之间的 NIXL 传输性能" + +#: ../../source/disaggregated_prefill/nixl/xpyd.rst:353 +msgid "**Startup Timing**: Stagger prefiller launches to avoid resource conflicts" +msgstr "**启动时机**：错开 Prefill 启动以避免资源冲突" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/shared_storage.po b/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/shared_storage.po new file mode 100644 index 00000000000..397aa6df354 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/shared_storage.po @@ -0,0 +1,29 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/disaggregated_prefill/shared_storage.rst:2 +msgid "Using shared storage" +msgstr "使用共享存储" + +#: ../../source/disaggregated_prefill/shared_storage.rst:4 +msgid "Coming soon..." +msgstr "敬请期待..." + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/benchmarking.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/benchmarking.po new file mode 100644 index 00000000000..b5c8d0f4a52 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/benchmarking.po @@ -0,0 +1,262 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/benchmarking.rst:2 +msgid "Benchmarking" +msgstr "基准测试" + +#: ../../source/getting_started/benchmarking.rst:4 +msgid "" +"This is a simple tutorial on how to deploy and benchmark LMCache using " +"the ``lmcache bench engine`` CLI." +msgstr "这是一个关于如何使用 ``lmcache bench engine`` CLI 部署和基准测试 LMCache 的简单教程。" + +#: ../../source/getting_started/benchmarking.rst:7 +msgid "" +"The ``lmcache bench engine`` command is a flexible traffic simulator that" +" sends configurable workloads to your inference engine and reports TTFT, " +"decoding speed, and throughput metrics. This tutorial walks through a " +"long-document Q&A benchmark that exercises LMCache's CPU offloading path." +msgstr "``lmcache bench engine`` 命令是一个灵活的流量模拟器，可以向您的推理引擎发送可配置的工作负载，并报告 TTFT、解码速度和吞吐量指标。本教程将介绍一个长文档问答基准测试，测试 LMCache 的 CPU 卸载路径。" + +#: ../../source/getting_started/benchmarking.rst:12 +msgid "" +"For the full CLI reference -- including every flag, every workload type, " +"and config-file usage -- see :doc:`/cli/bench`." +msgstr "有关完整的 CLI 参考 -- 包括每个标志、每种工作负载类型和配置文件用法 -- 请参见 :doc:`/cli/bench`。" + +#: ../../source/getting_started/benchmarking.rst:16 +msgid "Long Doc QA workload" +msgstr "长文档问答工作负载" + +#: ../../source/getting_started/benchmarking.rst:18 +msgid "" +"The ``long-doc-qa`` workload simulates repeated Q&A over long synthetic " +"documents: a warmup round primes the KV cache with each document, then a " +"benchmark round dispatches the questions. The number of documents is " +"derived from ``--kv-cache-volume`` and the model's tokens-per-GB rather " +"than set directly. See :doc:`/cli/bench` for the full flag list." +msgstr "``long-doc-qa`` 工作负载模拟对长的合成文档进行重复的问答：预热轮次用每个文档预热 KV Cache，然后基准轮次分发问题。文档的数量是根据 ``--kv-cache-volume`` 和模型的每 GB 令牌数推导的，而不是直接设置的。有关完整的标志列表，请参见 :doc:`/cli/bench`。" + +#: ../../source/getting_started/benchmarking.rst:25 +msgid "Example" +msgstr "示例" + +#: ../../source/getting_started/benchmarking.rst:27 +msgid "" +"To measure the benefit of LMCache, run the **same benchmark against two " +"setups** and compare the results:" +msgstr "要衡量 LMCache 的好处，请对 **两个设置运行相同的基准测试** 并比较结果：" + +#: ../../source/getting_started/benchmarking.rst:30 +msgid "**Setup A (baseline)** -- vLLM alone." +msgstr "**设置 A（基准）** -- 仅 vLLM。" + +#: ../../source/getting_started/benchmarking.rst:31 +msgid "**Setup B (with LMCache)** -- vLLM plus a standalone LMCache server." +msgstr "**Setup B (使用 LMCache)** -- vLLM 加上一个独立的 LMCache 服务器。" + +#: ../../source/getting_started/benchmarking.rst:33 +msgid "" +"The steps below reproduce both runs on ``Qwen/Qwen3-8B``. Adjust the " +"sizes to match your hardware." +msgstr "以下步骤将在 ``Qwen/Qwen3-8B`` 上重现这两个运行。根据您的硬件调整大小。" + +#: ../../source/getting_started/benchmarking.rst:37 +msgid "Setup A: vLLM alone (baseline)" +msgstr "设置 A：仅 vLLM（基准）" + +#: ../../source/getting_started/benchmarking.rst:44 +msgid "Setup B: vLLM with LMCache" +msgstr "设置 B：vLLM 与 LMCache 一起使用" + +#: ../../source/getting_started/benchmarking.rst:46 +msgid "" +"Run LMCache as a standalone service and point vLLM at it with the " +"``LMCacheMPConnector``. See :doc:`/mp/quickstart` for the full MP-mode " +"walkthrough." +msgstr "将 LMCache 作为独立服务运行，并使用 ``LMCacheMPConnector`` 将 vLLM 指向它。有关完整的 MP 模式操作指南，请参见 :doc:`/mp/quickstart`。" + +#: ../../source/getting_started/benchmarking.rst:50 +msgid "**Start the LMCache server:**" +msgstr "**启动 LMCache 服务器：**" + +#: ../../source/getting_started/benchmarking.rst:57 +msgid "" +"The ZMQ port defaults to **5555** (used by vLLM) and the HTTP frontend " +"defaults to **8080** (used by ``lmcache bench engine --lmcache-url``)." +msgstr "ZMQ 端口默认为 **5555**（由 vLLM 使用），HTTP 前端默认为 **8080**（由 ``lmcache bench engine --lmcache-url`` 使用）。" + +#: ../../source/getting_started/benchmarking.rst:60 +msgid "**Start vLLM with the MP connector in a separate terminal:**" +msgstr "**在单独的终端中启动 vLLM 和 MP 连接器：**" + +#: ../../source/getting_started/benchmarking.rst:69 +msgid "Run the benchmark" +msgstr "运行基准测试" + +#: ../../source/getting_started/benchmarking.rst:71 +msgid "" +"To make the comparison fair, capture the benchmark settings **once** in a" +" config file, then replay the same config against both setups." +msgstr "为了使比较公平，请在配置文件中**一次**捕获基准设置，然后在两个设置上重放相同的配置。" + +#: ../../source/getting_started/benchmarking.rst:74 +msgid "" +"**Step 1 -- export a shared config.** With the LMCache server from Setup " +"B still running, launch ``lmcache bench engine`` in interactive mode:" +msgstr "**步骤 1 -- 导出共享配置。** 在 Setup B 的 LMCache 服务器仍在运行的情况下，以交互模式启动 ``lmcache bench engine``：" + +#: ../../source/getting_started/benchmarking.rst:81 +msgid "" +"Interactive mode triggers because ``--engine-url`` and ``--workload`` are" +" missing, and ``--lmcache-url`` auto-detects ``tokens-per-gb-kvcache`` " +"from the server. Walk through the prompts and pick:" +msgstr "交互模式被触发，因为缺少 ``--engine-url`` 和 ``--workload``，而 ``--lmcache-url`` 从服务器自动检测 ``tokens-per-gb-kvcache``。按照提示进行操作并选择：" + +#: ../../source/getting_started/benchmarking.rst:85 +msgid "Engine URL: ``http://localhost:8000``" +msgstr "引擎 URL: ``http://localhost:8000``" + +#: ../../source/getting_started/benchmarking.rst:86 +msgid "Workload: ``long-doc-qa``" +msgstr "工作负载: ``long-doc-qa``" + +#: ../../source/getting_started/benchmarking.rst:87 +msgid "Model: auto-detected from the engine (or type ``Qwen/Qwen3-8B``)" +msgstr "模型：从引擎自动检测（或类型为 ``Qwen/Qwen3-8B``）" + +#: ../../source/getting_started/benchmarking.rst:88 +msgid "KV cache volume (GB): ``10``" +msgstr "KV cache volume (GB): ``10``" + +#: ../../source/getting_started/benchmarking.rst:89 +msgid "``ldqa-query-per-document``: ``1``" +msgstr "``ldqa-query-per-document``: ``1``" + +#: ../../source/getting_started/benchmarking.rst:90 +msgid "``ldqa-shuffle-policy``: ``tile``" +msgstr "``ldqa-shuffle-policy``: ``tile``" + +#: ../../source/getting_started/benchmarking.rst:91 +msgid "``ldqa-num-inflight-requests``: ``4``" +msgstr "``ldqa-num-inflight-requests``: ``4``" + +#: ../../source/getting_started/benchmarking.rst:92 +msgid "Leave the rest at their defaults." +msgstr "将其余选项保留为默认值。" + +#: ../../source/getting_started/benchmarking.rst:94 +msgid "" +"At the **summary** step, choose **\"Export configuration for later use " +"and exit\"** and save to ``bench_config.json``. The file looks like:" +msgstr "在 **总结** 步骤中，选择 **“导出配置以供后用并退出”** 并保存为 ``bench_config.json``。该文件如下所示：" + +#: ../../source/getting_started/benchmarking.rst:110 +msgid "" +"Note that the exported config stores ``tokens_per_gb_kvcache`` (resolved " +"from ``--lmcache-url``) but **not** the engine URL or the LMCache URL, so" +" the same file is portable across environments." +msgstr "请注意，导出的配置存储了 ``tokens_per_gb_kvcache``（从 ``--lmcache-url`` 解析得出），但**不**包括引擎 URL 或 LMCache URL，因此同一个文件可以在不同环境中使用。" + +#: ../../source/getting_started/benchmarking.rst:114 +msgid "" +"**Step 2 -- replay against each setup.** Point ``--engine-url`` at " +"whichever vLLM you want to benchmark and pass the shared config:" +msgstr "**步骤 2 -- 针对每个设置进行重放。** 将 ``--engine-url`` 指向您想要基准测试的 vLLM，并传递共享配置：" + +#: ../../source/getting_started/benchmarking.rst:123 +msgid "" +"Run this once against Setup A's vLLM and once against Setup B's vLLM-" +"plus-LMCache, recording the metrics from each run." +msgstr "对 Setup A 的 vLLM 运行一次，对 Setup B 的 vLLM-plus-LMCache 运行一次，记录每次运行的指标。" + +#: ../../source/getting_started/benchmarking.rst:127 +msgid "Results" +msgstr "结果" + +#: ../../source/getting_started/benchmarking.rst:129 +msgid "" +"Pull the headline numbers out of each run's ``Engine Benchmark Result " +"(long-doc-qa)`` summary:" +msgstr "从每次运行的 ``Engine Benchmark Result (long-doc-qa)`` 摘要中提取关键数字：" + +#: ../../source/getting_started/benchmarking.rst:136 +msgid "Metric" +msgstr "指标" + +#: ../../source/getting_started/benchmarking.rst:137 +msgid "Setup A (vLLM)" +msgstr "设置 A (vLLM)" + +#: ../../source/getting_started/benchmarking.rst:138 +msgid "Setup B (+ LMCache)" +msgstr "设置 B (+ LMCache)" + +#: ../../source/getting_started/benchmarking.rst:139 +msgid "Successful requests" +msgstr "成功的请求" + +#: ../../source/getting_started/benchmarking.rst:140 +#: ../../source/getting_started/benchmarking.rst:141 +msgid "46" +msgstr "46" + +#: ../../source/getting_started/benchmarking.rst:142 +msgid "Benchmark duration (s)" +msgstr "基准测试持续时间（秒）" + +#: ../../source/getting_started/benchmarking.rst:143 +msgid "23.47" +msgstr "23.47" + +#: ../../source/getting_started/benchmarking.rst:144 +msgid "13.79" +msgstr "13.79" + +#: ../../source/getting_started/benchmarking.rst:145 +msgid "Mean TTFT (ms)" +msgstr "平均 TTFT (毫秒)" + +#: ../../source/getting_started/benchmarking.rst:146 +msgid "757.00" +msgstr "757.00" + +#: ../../source/getting_started/benchmarking.rst:147 +msgid "185.00" +msgstr "185.00" + +#: ../../source/getting_started/benchmarking.rst:149 +msgid "" +"That's a **75%** reduction in Mean TTFT (757 ms → 185 ms) and a **41%** " +"reduction in benchmark duration (23.47 s → 13.79 s) from LMCache " +"offloading." +msgstr "这意味着在 LMCache 卸载的情况下，平均 TTFT 减少了 **75%**（757 毫秒 → 185 毫秒），基准持续时间减少了 **41%**（23.47 秒 → 13.79 秒）。" + +#: ../../source/getting_started/benchmarking.rst:153 +msgid "" +"Without LMCache, once the benchmark's working set overflows the GPU KV " +"cache the second round has to recompute every prefix, so TTFT and " +"throughput don't improve even when content repeats. LMCache keeps the " +"evicted blocks on CPU RAM and restores them on demand -- that's where the" +" speedup comes from." +msgstr "没有 LMCache，一旦基准测试的工作集溢出 GPU KV Cache，第二轮就必须重计算每个前缀，因此即使内容重复，TTFT 和吞吐量也不会改善。LMCache 将逐出的块保留在 CPU 内存中，并按需恢复它们——这就是加速的来源。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/cli.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/cli.po new file mode 100644 index 00000000000..d966d919c01 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/cli.po @@ -0,0 +1,478 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/cli.rst:2 +msgid "CLI Reference" +msgstr "CLI 参考" + +#: ../../source/getting_started/cli.rst:4 +msgid "" +"LMCache provides a unified ``lmcache`` command-line interface for " +"interacting with KV cache servers, running benchmarks, and inspecting " +"cache state." +msgstr "LMCache 提供了一个统一的 ``lmcache`` 命令行接口，用于与 KV Cache 服务器交互、运行基准测试和检查缓存状态。" + +#: ../../source/getting_started/cli.rst:12 +msgid "Installation" +msgstr "安装" + +#: ../../source/getting_started/cli.rst:14 +msgid "The ``lmcache`` CLI is available in two packages:" +msgstr "``lmcache`` CLI 可在两个软件包中使用：" + +#: ../../source/getting_started/cli.rst:20 +msgid "Package" +msgstr "包" + +#: ../../source/getting_started/cli.rst:21 +msgid "Install" +msgstr "安装" + +#: ../../source/getting_started/cli.rst:22 +msgid "When to use" +msgstr "何时使用" + +#: ../../source/getting_started/cli.rst:23 +msgid "``lmcache``" +msgstr "``lmcache``" + +#: ../../source/getting_started/cli.rst:24 +msgid "``pip install lmcache``" +msgstr "``pip install lmcache``" + +#: ../../source/getting_started/cli.rst:25 +msgid "Full install: server, CLI, and CUDA extensions. Requires Linux + GPU." +msgstr "完整安装：服务器、CLI 和 CUDA 扩展。需要 Linux + GPU。" + +#: ../../source/getting_started/cli.rst:26 +msgid "``lmcache-cli``" +msgstr "``lmcache-cli``" + +#: ../../source/getting_started/cli.rst:27 +msgid "``pip install lmcache-cli``" +msgstr "``pip install lmcache-cli``" + +#: ../../source/getting_started/cli.rst:28 +msgid "CLI only: query, ping, bench, describe. No GPU required, any OS." +msgstr "仅限 CLI：查询、ping、基准测试、描述。无需 GPU，任何操作系统均可。" + +#: ../../source/getting_started/cli.rst:32 +msgid "" +"Do not install both packages in the same environment — they both provide " +"the ``lmcache`` entry point." +msgstr "请勿在同一环境中安装这两个软件包 — 它们都提供 ``lmcache`` 入口点。" + +#: ../../source/getting_started/cli.rst:36 +msgid "Quick Start" +msgstr "快速开始" + +#: ../../source/getting_started/cli.rst:38 +msgid "After installing LMCache, the ``lmcache`` command is available:" +msgstr "安装 LMCache 后，可以使用 ``lmcache`` 命令：" + +#: ../../source/getting_started/cli.rst:63 +msgid "Available Commands" +msgstr "可用命令" + +#: ../../source/getting_started/cli.rst:69 +msgid "Command" +msgstr "命令" + +#: ../../source/getting_started/cli.rst:70 +#: ../../source/getting_started/cli.rst:193 +#: ../../source/getting_started/cli.rst:336 +msgid "Description" +msgstr "描述" + +#: ../../source/getting_started/cli.rst:71 +msgid "``describe``" +msgstr "``describe``" + +#: ../../source/getting_started/cli.rst:72 +msgid "" +"Show detailed status of a running LMCache service, including cache " +"health, L1 storage, registered models, and L2 adapters." +msgstr "显示正在运行的 LMCache 服务的详细状态，包括缓存健康状况、L1 存储、注册的模型和 L2 适配器。" + +#: ../../source/getting_started/cli.rst:74 +#: ../../source/getting_started/cli.rst:280 +msgid "``query``" +msgstr "``query``" + +#: ../../source/getting_started/cli.rst:75 +msgid "" +"Single-shot query interface for both the serving engine and KV cache " +"worker." +msgstr "用于服务引擎和 KV Cache 工作线程的单次查询接口。" + +#: ../../source/getting_started/cli.rst:76 +msgid "``server``" +msgstr "``server``" + +#: ../../source/getting_started/cli.rst:77 +msgid "Launch the LMCache server (ZMQ + HTTP). Requires full ``lmcache`` install." +msgstr "启动 LMCache 服务器（ZMQ + HTTP）。需要完整的 ``lmcache`` 安装。" + +#: ../../source/getting_started/cli.rst:78 +msgid "``ping``" +msgstr "``ping``" + +#: ../../source/getting_started/cli.rst:79 +msgid "Liveness check for LMCache or vLLM servers." +msgstr "LMCache 或 vLLM 服务器的存活检查。" + +#: ../../source/getting_started/cli.rst:80 +msgid "``bench``" +msgstr "``bench``" + +#: ../../source/getting_started/cli.rst:81 +msgid "Run sustained performance benchmarks against an inference engine." +msgstr "对推理引擎运行持续性能基准测试。" + +#: ../../source/getting_started/cli.rst:82 +#: ../../source/getting_started/cli.rst:194 +msgid "``kvcache``" +msgstr "``kvcache``" + +#: ../../source/getting_started/cli.rst:83 +msgid "Manage KV cache state (e.g. clear L1 cache) on a running server." +msgstr "在运行的服务器上管理 KV Cache 状态（例如，清除 L1 缓存）。" + +#: ../../source/getting_started/cli.rst:87 +msgid "``bench`` — Engine Benchmarking" +msgstr "``bench`` — 引擎基准测试" + +#: ../../source/getting_started/cli.rst:89 +msgid "" +"Run sustained benchmarks against an inference engine with multiple " +"workload types:" +msgstr "对推理引擎运行持续基准测试，使用多种工作负载类型：" + +#: ../../source/getting_started/cli.rst:121 +msgid "Five workloads are available:" +msgstr "提供五种工作负载：" + +#: ../../source/getting_started/cli.rst:123 +msgid "" +"**long-doc-qa** -- repeated Q&A over long documents (tests KV cache " +"reuse)." +msgstr "**long-doc-qa** -- 在长文档上进行重复的问答（测试 KV Cache 重用）。" + +#: ../../source/getting_started/cli.rst:124 +msgid "**multi-round-chat** -- multi-turn chat with stateful sessions." +msgstr "**多轮聊天** -- 带有状态会话的多轮聊天。" + +#: ../../source/getting_started/cli.rst:125 +msgid "" +"**long-doc-permutator** -- permutations of context documents (blended " +"cache reuse stress test)." +msgstr "**long-doc-permutator** -- 上下文文档的排列组合（混合缓存重用压力测试）。" + +#: ../../source/getting_started/cli.rst:127 +msgid "" +"**prefix-suffix-tuner** -- two-pass sequential workload demonstrating " +"tiered KV-cache reuse (L0/L1/L2)." +msgstr "**prefix-suffix-tuner** -- 两次传递的顺序工作负载，演示分层 KV Cache 重用 (L0/L1/L2)。" + +#: ../../source/getting_started/cli.rst:129 +msgid "**random-prefill** -- prefill-only requests fired simultaneously." +msgstr "**随机预填** -- 同时发起的仅预填请求。" + +#: ../../source/getting_started/cli.rst:131 +msgid "" +"See :doc:`/cli/bench` for full documentation including all workload " +"options, interactive mode details, and config file format." +msgstr "请参阅 :doc:`/cli/bench` 以获取完整文档，包括所有工作负载选项、交互模式详细信息和配置文件格式。" + +#: ../../source/getting_started/cli.rst:136 +msgid "``describe`` — Service Status Dashboard" +msgstr "``describe`` — 服务状态仪表板" + +#: ../../source/getting_started/cli.rst:138 +msgid "Inspect the state of a running LMCache KV cache server:" +msgstr "检查正在运行的 LMCache KV Cache 服务器的状态：" + +#: ../../source/getting_started/cli.rst:177 +msgid "The output shows:" +msgstr "输出显示：" + +#: ../../source/getting_started/cli.rst:179 +msgid "**Overview** — health status, engine type, chunk size." +msgstr "**概述** — 健康状态、引擎类型、块大小。" + +#: ../../source/getting_started/cli.rst:180 +msgid "**L1 storage** — capacity, usage, eviction policy, cached object count." +msgstr "**L1 存储** — 容量、使用情况、逐出策略、缓存对象数量。" + +#: ../../source/getting_started/cli.rst:181 +msgid "" +"**Registered models** — per-model KV cache layout including the GPU KV " +"tensor shape (symbolic and concrete), attention backend, and layer " +"details." +msgstr "**注册模型** — 每个模型的 KV Cache 布局，包括 GPU KV 张量形状（符号和具体）、注意力后端和层详细信息。" + +#: ../../source/getting_started/cli.rst:183 +msgid "**L2 adapters** — type, health, backend, stored objects, and utilization." +msgstr "**L2 适配器** — 类型、健康状况、后端、存储对象和利用率。" + +#: ../../source/getting_started/cli.rst:186 +msgid "Arguments" +msgstr "参数" + +#: ../../source/getting_started/cli.rst:192 +#: ../../source/getting_started/cli.rst:335 +msgid "Flag" +msgstr "标志" + +#: ../../source/getting_started/cli.rst:195 +msgid "Target to describe (currently only ``kvcache`` is supported)." +msgstr "描述的目标（目前仅支持 ``kvcache``）。" + +#: ../../source/getting_started/cli.rst:196 +#: ../../source/getting_started/cli.rst:339 +msgid "``--url``" +msgstr "``--url``" + +#: ../../source/getting_started/cli.rst:197 +msgid "LMCache HTTP server URL (default: ``http://localhost:8080``)." +msgstr "LMCache HTTP 服务器 URL（默认：``http://localhost:8080``）。" + +#: ../../source/getting_started/cli.rst:198 +#: ../../source/getting_started/cli.rst:342 +msgid "``--format``" +msgstr "``--format``" + +#: ../../source/getting_started/cli.rst:199 +#: ../../source/getting_started/cli.rst:343 +msgid "Output format: ``terminal`` (default) or ``json``." +msgstr "输出格式：``terminal``（默认）或``json``。" + +#: ../../source/getting_started/cli.rst:200 +#: ../../source/getting_started/cli.rst:344 +msgid "``--output PATH``" +msgstr "``--output PATH``" + +#: ../../source/getting_started/cli.rst:201 +#: ../../source/getting_started/cli.rst:345 +msgid "Save metrics to a file (format follows ``--format``)." +msgstr "将指标保存到文件中（格式遵循 ``--format``）。" + +#: ../../source/getting_started/cli.rst:204 +#: ../../source/getting_started/cli.rst:395 +msgid "JSON Output" +msgstr "JSON 输出" + +#: ../../source/getting_started/cli.rst:206 +msgid "" +"Use ``--format json`` for machine-readable output. Models and L2 adapters" +" are collected into lists for easy programmatic access:" +msgstr "使用 ``--format json`` 以机器可读的输出格式。模型和 L2 适配器被收集到列表中，以便于程序访问：" + +#: ../../source/getting_started/cli.rst:256 +msgid "GPU KV Shape Abbreviations" +msgstr "GPU KV 形状缩写" + +#: ../../source/getting_started/cli.rst:258 +msgid "The ``gpu_kv_shape`` field uses short names from the ``GPUKVFormat`` enum:" +msgstr "``gpu_kv_shape`` 字段使用 ``GPUKVFormat`` 枚举中的简短名称：" + +#: ../../source/getting_started/cli.rst:264 +msgid "Abbrev" +msgstr "缩写" + +#: ../../source/getting_started/cli.rst:265 +#: ../../source/getting_started/cli.rst:357 +msgid "Meaning" +msgstr "含义" + +#: ../../source/getting_started/cli.rst:266 +msgid "NB" +msgstr "注意事项" + +#: ../../source/getting_started/cli.rst:267 +msgid "num_blocks" +msgstr "num_blocks" + +#: ../../source/getting_started/cli.rst:268 +msgid "NL" +msgstr "NL" + +#: ../../source/getting_started/cli.rst:269 +msgid "num_layers" +msgstr "num_layers" + +#: ../../source/getting_started/cli.rst:270 +msgid "BS" +msgstr "批量大小" + +#: ../../source/getting_started/cli.rst:271 +msgid "block_size" +msgstr "块大小" + +#: ../../source/getting_started/cli.rst:272 +msgid "NH" +msgstr "NH" + +#: ../../source/getting_started/cli.rst:273 +msgid "num_heads" +msgstr "num_heads" + +#: ../../source/getting_started/cli.rst:274 +msgid "HS" +msgstr "HS" + +#: ../../source/getting_started/cli.rst:275 +msgid "head_size" +msgstr "head_size" + +#: ../../source/getting_started/cli.rst:276 +msgid "PBS" +msgstr "PBS" + +#: ../../source/getting_started/cli.rst:277 +msgid "page_buffer_size (NB × BS)" +msgstr "页面缓冲区大小 (NB × BS)" + +#: ../../source/getting_started/cli.rst:282 +#, python-brace-format +msgid "" +"The ``query engine`` subcommand sends one request to the engine API and " +"reports metrics. ``--prompt`` supports placeholders: ``{lmcache}`` loads " +"``lmcache/cli/documents/lmcache.txt``, and custom documents can be passed" +" with ``--documents NAME=PATH``. The prompt token count is taken directly" +" from the usage data reported by the engine (``stream_options: " +"{include_usage: true}``)." +msgstr "``query engine`` 子命令向引擎 API 发送一个请求并报告指标。 ``--prompt`` 支持占位符： ``{lmcache}`` 加载 ``lmcache/cli/documents/lmcache.txt``，自定义文档可以通过 ``--documents NAME=PATH`` 传递。提示的令牌计数直接来自引擎报告的使用数据 (``stream_options: {include_usage: true}``)。" + +#: ../../source/getting_started/cli.rst:309 +msgid "``ping`` — Liveness Check" +msgstr "``ping`` — 存活检查" + +#: ../../source/getting_started/cli.rst:311 +msgid "" +"Check whether an LMCache KV cache server or a vLLM serving engine is " +"reachable:" +msgstr "检查 LMCache KV Cache 服务器或 vLLM 服务引擎是否可达：" + +#: ../../source/getting_started/cli.rst:329 +msgid "Options" +msgstr "选项" + +#: ../../source/getting_started/cli.rst:337 +msgid "``kvcache`` | ``engine``" +msgstr "``kvcache`` | ``engine``" + +#: ../../source/getting_started/cli.rst:338 +msgid "Target to ping (positional, required)." +msgstr "要 ping 的目标（位置参数，必填）。" + +#: ../../source/getting_started/cli.rst:340 +msgid "" +"Server URL. Defaults to ``http://localhost:8080`` for ``kvcache``, " +"``http://localhost:8000`` for ``engine``." +msgstr "服务器 URL。默认值为 ``http://localhost:8080`` 用于 ``kvcache``，``http://localhost:8000`` 用于 ``engine``。" + +#: ../../source/getting_started/cli.rst:346 +msgid "``-q`` / ``--quiet``" +msgstr "``-q`` / ``--quiet``" + +#: ../../source/getting_started/cli.rst:347 +msgid "Suppress stdout output. Exit code only." +msgstr "抑制标准输出。仅返回退出代码。" + +#: ../../source/getting_started/cli.rst:350 +msgid "Exit Codes" +msgstr "退出代码" + +#: ../../source/getting_started/cli.rst:356 +msgid "Code" +msgstr "代码" + +#: ../../source/getting_started/cli.rst:358 +msgid "``0``" +msgstr "``0``" + +#: ../../source/getting_started/cli.rst:359 +msgid "Server is reachable (HTTP 200)." +msgstr "服务器可达（HTTP 200）。" + +#: ../../source/getting_started/cli.rst:360 +msgid "``1``" +msgstr "``1``" + +#: ../../source/getting_started/cli.rst:361 +msgid "Connection failure or non-200 response." +msgstr "连接失败或非 200 响应。" + +#: ../../source/getting_started/cli.rst:364 +msgid "``kvcache`` — KV Cache Management" +msgstr "``kvcache`` — KV Cache 管理" + +#: ../../source/getting_started/cli.rst:366 +msgid "" +"Manage KV cache state on a running LMCache server. See " +":doc:`/cli/kvcache` for full documentation including examples, options, " +"and common patterns." +msgstr "在运行中的 LMCache 服务器上管理 KV Cache 状态。有关完整文档，包括示例、选项和常见模式，请参见 :doc:`/cli/kvcache`。" + +#: ../../source/getting_started/cli.rst:369 +msgid "Quick example:" +msgstr "快速示例：" + +#: ../../source/getting_started/cli.rst:378 +msgid "Metrics Output" +msgstr "指标输出" + +#: ../../source/getting_started/cli.rst:380 +msgid "All commands that produce metrics support two output formats:" +msgstr "所有生成指标的命令支持两种输出格式：" + +#: ../../source/getting_started/cli.rst:383 +msgid "Terminal Output" +msgstr "终端输出" + +#: ../../source/getting_started/cli.rst:385 +msgid "Human-readable ASCII table:" +msgstr "可读的 ASCII 表：" + +#: ../../source/getting_started/cli.rst:397 +msgid "" +"Machine-readable output with structured keys, available via ``--format " +"json`` (stdout) or ``--output`` (file):" +msgstr "机器可读的输出，带有结构化的键，可以通过 ``--format json``（标准输出）或 ``--output``（文件）获得：" + +#: ../../source/getting_started/cli.rst:414 +msgid "" +"The terminal output uses human-readable labels (e.g., ``\"Round trip time" +" (ms)\"``), while the JSON output uses machine-readable keys (e.g., " +"``\"round_trip_time_ms\"``)." +msgstr "终端输出使用人类可读的标签（例如，``\"往返时间（毫秒）\"``），而 JSON 输出使用机器可读的键（例如，``\"round_trip_time_ms\"``）。" + +#: ../../source/getting_started/cli.rst:419 +msgid "Adding New Commands" +msgstr "添加新命令" + +#: ../../source/getting_started/cli.rst:421 +msgid "" +"New CLI subcommands can be added by creating a ``BaseCommand`` subclass " +"and registering it. See :doc:`/developer_guide/cli` for details." +msgstr "可以通过创建 ``BaseCommand`` 子类并注册它来添加新的 CLI 子命令。有关详细信息，请参见 :doc:`/developer_guide/cli`。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/faq.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/faq.po new file mode 100644 index 00000000000..ff1ed6046f9 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/faq.po @@ -0,0 +1,142 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/faq.rst:2 +msgid "FAQ" +msgstr "常见问题解答" + +#: ../../source/getting_started/faq.rst:5 +msgid "" +"What are the KV cache sizes for popular models? And why is LMCache " +"important?" +msgstr "流行模型的 KV Cache 大小是多少？LMCache 为什么重要？" + +#: ../../source/getting_started/faq.rst:7 +msgid "" +"You can calculate KV cache sizes using our :doc:`KV cache calculator " +"`. We also provide a reference table below with KV " +"cache information for some popular models." +msgstr "您可以使用我们的 :doc:`KV 缓存计算器 ` 来计算 KV 缓存大小。我们还提供了下面的参考表，其中包含一些流行模型的 KV 缓存信息。" + +#: ../../source/getting_started/faq.rst:9 +msgid "" +"As shown in the table, after loading Qwen/Qwen3-32B for example, there is" +" only enough space in the spare GPU RAM to hold 275,760 tokens for KV " +"caches. This supports only 6.73 concurrent users if each prompt is 40,960" +" tokens long. Once this capacity is exceeded, the KV cache must be " +"evicted, and when the same user returns, their request needs to be re-" +"prefilled, which takes significantly longer." +msgstr "如表所示，以 Qwen/Qwen3-32B 为例，备用显存中仅有足够的空间容纳 275,760 个 token 用于 KV 缓存。如果每个提示为 40,960 个 token，这仅支持 6.73 个并发用户。一旦超过这个容量，KV 缓存必须被逐出，当同一用户返回时，他们的请求需要重新 Prefill，这将花费更长的时间。" + +#: ../../source/getting_started/faq.rst:11 +msgid "" +"**LMCache is designed to extend this virtual memory capacity**, enabling " +"you to store more KV caches and avoid costly re-prefilling operations." +msgstr "**LMCache 旨在扩展这种虚拟内存容量**，使您能够存储更多的 KV Cache，并避免昂贵的重新预填充操作。" + +#: ../../source/getting_started/faq.rst:13 +msgid "**KV Cache Sizes for Popular Models**" +msgstr "**流行模型的 KV Cache 大小**" + +#: ../../source/getting_started/faq.rst:19 +msgid "Model" +msgstr "模型" + +#: ../../source/getting_started/faq.rst:20 +msgid "KV Cache Size per 1000 tokens" +msgstr "每1000个令牌的KV缓存大小" + +#: ../../source/getting_started/faq.rst:21 +msgid "Spare GPU RAM for KV cache" +msgstr "为 KV Cache 预留的显存" + +#: ../../source/getting_started/faq.rst:22 +msgid "Context length" +msgstr "上下文长度" + +#: ../../source/getting_started/faq.rst:23 +msgid "Number of full-length prompts that can be stored in GPU" +msgstr "可以存储在显存中的完整提示数量" + +#: ../../source/getting_started/faq.rst:24 +msgid "Qwen/Qwen3-8B" +msgstr "Qwen/Qwen3-8B" + +#: ../../source/getting_started/faq.rst:25 +msgid "0.1373 GB" +msgstr "0.1373 GB" + +#: ../../source/getting_started/faq.rst:26 +msgid "50.32 GB (or 366,400 tokens)" +msgstr "50.32 GB（或 366,400 个 token）" + +#: ../../source/getting_started/faq.rst:27 +#: ../../source/getting_started/faq.rst:32 +msgid "40,960 tokens" +msgstr "40,960 个令牌" + +#: ../../source/getting_started/faq.rst:28 +msgid "8.95x" +msgstr "8.95倍" + +#: ../../source/getting_started/faq.rst:29 +msgid "Qwen/Qwen3-32B (tp=2 on H100)" +msgstr "Qwen/Qwen3-32B (tp=2 on H100)" + +#: ../../source/getting_started/faq.rst:30 +msgid "0.2441 GB" +msgstr "0.2441 GB" + +#: ../../source/getting_started/faq.rst:31 +msgid "33.66 GB × 2 (or 275,760 tokens)" +msgstr "33.66 GB × 2（或 275,760 个 token）" + +#: ../../source/getting_started/faq.rst:33 +msgid "6.73x" +msgstr "6.73倍" + +#: ../../source/getting_started/faq.rst:34 +msgid "meta-llama/Llama-3.1-70B (tp=4 on H100)" +msgstr "meta-llama/Llama-3.1-70B (tp=4 on H100)" + +#: ../../source/getting_started/faq.rst:35 +msgid "0.3052 GB" +msgstr "0.3052 GB" + +#: ../../source/getting_started/faq.rst:36 +msgid "32.06 GB × 4 (or 420,208 tokens)" +msgstr "32.06 GB × 4（或 420,208 个 token）" + +#: ../../source/getting_started/faq.rst:37 +msgid "131,072 tokens" +msgstr "131,072 个 token" + +#: ../../source/getting_started/faq.rst:38 +msgid "3.21x" +msgstr "3.21倍" + +#: ../../source/getting_started/faq.rst:41 +msgid "" +"You may also find this `VRAM Calculator `_ useful for calculating the estimated spare GPU RAM for " +"different models and configurations." +msgstr "您还可以使用这个 `VRAM Calculator `_ 来计算不同模型和配置的估计剩余显存。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/installation.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/installation.po new file mode 100644 index 00000000000..2c039e9f696 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/installation.po @@ -0,0 +1,238 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/installation.rst:4 +msgid "Installation" +msgstr "安装" + +#: ../../source/getting_started/installation.rst:6 +msgid "" +"**Prerequisites:** Linux · Python 3.9–3.13 · NVIDIA GPU (compute 7.0+) · " +"CUDA 12.1+ · `uv `_" +msgstr "**前提条件：** Linux · Python 3.9–3.13 · NVIDIA GPU (计算 7.0+) · CUDA 12.1+ · `uv `_" + +#: ../../source/getting_started/installation.rst:9 +msgid "Install LMCache" +msgstr "安装 LMCache" + +#: ../../source/getting_started/installation.rst +msgid "Python (pip / uv)" +msgstr "Python (pip / uv)" + +#: ../../source/getting_started/installation.rst +msgid "Stable" +msgstr "稳定" + +#: ../../source/getting_started/installation.rst +msgid "CUDA 13.0" +msgstr "CUDA 13.0" + +#: ../../source/getting_started/installation.rst:31 +msgid "" +"You're all set! You can now start using LMCache. For hands-on guides and " +"more usage examples, see the :ref:`quickstart_examples` section." +msgstr "一切准备就绪！您现在可以开始使用 LMCache。有关实践指南和更多使用示例，请参阅 :ref:`quickstart_examples` 部分。" + +#: ../../source/getting_started/installation.rst +msgid "CUDA 12.9" +msgstr "CUDA 12.9" + +#: ../../source/getting_started/installation.rst:36 +msgid "" +"The CUDA 12.9 wheel is published to a dedicated `GitHub Release " +"`__ rather than PyPI." +msgstr "CUDA 12.9 的 wheel 发布在专用的 `GitHub Release `__ 上，而不是 PyPI。" + +#: ../../source/getting_started/installation.rst:51 +msgid "" +"``--extra-index-url https://download.pytorch.org/whl/cu129`` ensures the " +"CUDA 12.9 build of PyTorch is resolved. Without it, pip may select a " +"mismatched CUDA variant." +msgstr "``--extra-index-url https://download.pytorch.org/whl/cu129`` 确保解析 PyTorch 的 CUDA 12.9 构建。没有它，pip 可能会选择不匹配的 CUDA 变体。" + +#: ../../source/getting_started/installation.rst +msgid "Nightly" +msgstr "夜间版" + +#: ../../source/getting_started/installation.rst:56 +msgid "" +"Nightly wheels are built from the latest ``dev`` branch each day at 07:30" +" UTC and published to GitHub Releases. No version pinning required — " +"``--pre`` picks the latest nightly automatically." +msgstr "夜间构建的轮子每天在 UTC 时间 07:30 从最新的 ``dev`` 分支构建，并发布到 GitHub Releases。无需版本固定 — ``--pre`` 会自动选择最新的夜间版本。" + +#: ../../source/getting_started/installation.rst +msgid "From Source" +msgstr "从源代码安装" + +#: ../../source/getting_started/installation.rst:86 +msgid "" +"``--no-build-isolation`` ensures the kernels are compiled against the " +"same torch already installed in your environment, preventing undefined " +"symbol errors at runtime." +msgstr "``--no-build-isolation`` 确保内核与您环境中已安装的相同 torch 进行编译，从而防止运行时出现未定义符号错误。" + +#: ../../source/getting_started/installation.rst +msgid "ROCm" +msgstr "ROCm" + +#: ../../source/getting_started/installation.rst +msgid "Intel XPU" +msgstr "英特尔 XPU" + +#: ../../source/getting_started/installation.rst +msgid "Docker" +msgstr "Docker" + +#: ../../source/getting_started/installation.rst:214 +msgid "See :ref:`docker_deployment` for running the container and ROCm images." +msgstr "请参阅 :ref:`docker_deployment` 以获取运行容器和 ROCm 镜像的信息。" + +#: ../../source/getting_started/installation.rst +msgid "CLI Only" +msgstr "仅限 CLI" + +#: ../../source/getting_started/installation.rst:218 +msgid "" +"Lightweight CLI-only package for querying or benchmarking a remote " +"LMCache server. No CUDA required, works on any OS." +msgstr "轻量级的仅限 CLI 的软件包，用于查询或基准测试远程 LMCache 服务器。无需 CUDA，适用于任何操作系统。" + +#: ../../source/getting_started/installation.rst:227 +msgid "" +"``lmcache-cli`` and ``lmcache`` ship the same ``lmcache`` CLI command. Do" +" not install both in the same environment." +msgstr "``lmcache-cli`` 和 ``lmcache`` 提供相同的 ``lmcache`` CLI 命令。请勿在同一环境中同时安装两者。" + +#: ../../source/getting_started/installation.rst:231 +msgid "Verify Installation" +msgstr "验证安装" + +#: ../../source/getting_started/installation.rst:238 +msgid "Compatibility Matrix" +msgstr "兼容性矩阵" + +#: ../../source/getting_started/installation.rst:240 +msgid "" +"✅ compatible · ❌ API incompatible · 🕯 torch mismatch (use ``--no-build-" +"isolation``)" +msgstr "✅ 兼容 · ❌ API 不兼容 · 🕯 torch 不匹配 (使用 ``--no-build-isolation``)" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "LMCache 0.4.2" +msgstr "LMCache 0.4.2" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "LMCache 0.4.1" +msgstr "LMCache 0.4.1" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "LMCache 0.3.15" +msgstr "LMCache 0.3.15" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "LMCache 0.3.14" +msgstr "LMCache 0.3.14" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "LMCache 0.3.13" +msgstr "LMCache 0.3.13" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "LMCache 0.3.12" +msgstr "LMCache 0.3.12" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "LMCache 0.3.11" +msgstr "LMCache 0.3.11" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "LMCache 0.3.10" +msgstr "LMCache 0.3.10" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.18.0.x" +msgstr "vLLM 0.18.0.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "✅" +msgstr "✅" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "🕯️" +msgstr "🕯️" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.17.1.x" +msgstr "vLLM 0.17.1.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.17.0.x" +msgstr "vLLM 0.17.0.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.16.0.x" +msgstr "vLLM 0.16.0.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.15.1.x" +msgstr "vLLM 0.15.1.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.15.0.x" +msgstr "vLLM 0.15.0.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.14.1.x" +msgstr "vLLM 0.14.1.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.14.0.x" +msgstr "vLLM 0.14.0.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.13.0.x" +msgstr "vLLM 0.13.0.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.12.0.x" +msgstr "vLLM 0.12.0.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.11.2.x" +msgstr "vLLM 0.11.2.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.11.1.x" +msgstr "vLLM 0.11.1.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.11.0.x" +msgstr "vLLM 0.11.0.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.10.2.x" +msgstr "vLLM 0.10.2.x" + +#: ../../source/getting_started/Installation_compatibility_matrix.csv:1 +msgid "vLLM 0.10.1.x" +msgstr "vLLM 0.10.1.x" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/kv_cache_calculator.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/kv_cache_calculator.po new file mode 100644 index 00000000000..9ae9b48bc36 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/kv_cache_calculator.po @@ -0,0 +1,45 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/kv_cache_calculator.rst:2 +msgid "KV Cache Size Calculator" +msgstr "KV Cache 大小计算器" + +#: ../../source/getting_started/kv_cache_calculator.rst:4 +msgid "Use this interactive calculator to:" +msgstr "使用此交互式计算器来：" + +#: ../../source/getting_started/kv_cache_calculator.rst:6 +msgid "" +"**Calculate KV Cache Size**: Estimate the memory required for caching a " +"specific number of tokens" +msgstr "**计算 KV Cache 大小**：估算缓存特定数量的令牌所需的内存" + +#: ../../source/getting_started/kv_cache_calculator.rst:7 +msgid "" +"**Calculate Maximum Tokens**: Find out how many tokens you can cache " +"given your available GPU RAM" +msgstr "**计算最大令牌数**：找出在可用的 GPU RAM 下可以缓存多少令牌" + +#: ../../source/getting_started/kv_cache_calculator.rst:9 +msgid "This helps you plan memory requirements for your LMCache deployment." +msgstr "这可以帮助您规划 LMCache 部署的内存需求。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart.po new file mode 100644 index 00000000000..2dac320fc07 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart.po @@ -0,0 +1,381 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/quickstart.rst:4 +msgid "Quickstart" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:6 +msgid "" +"This guide helps you get LMCache running end-to-end in a couple of " +"minutes. Use the tabs below to switch the engine. Steps are the same; " +"only the libraries and launch commands change." +msgstr "" + +#: ../../source/getting_started/quickstart.rst +msgid "vLLM" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:13 +msgid "**Install LMCache**" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:21 +msgid "LMCache supports two deployment modes with vLLM:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:23 +msgid "" +"**Multiprocess (MP) mode** -- **recommended.** LMCache runs as a " +"standalone service and vLLM attaches via ``LMCacheMPConnector``. Scales " +"better, exposes management/observability endpoints, and supports sharing " +"one cache across multiple engine instances." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:27 +msgid "" +"**In-process mode** -- LMCache runs inside the vLLM process via " +"``LMCacheConnectorV1``. Single command, convenient for quick single-node " +"experiments." +msgstr "" + +#: ../../source/getting_started/quickstart.rst +msgid "MP mode (recommended)" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:37 +#: ../../source/getting_started/quickstart.rst:390 +msgid "Start the LMCache server:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:47 +msgid "" +"The ZMQ port (default **5555**) accepts connections from vLLM; the HTTP " +"frontend (default **8080**) serves the management and metrics endpoints. " +"See :doc:`../mp/quickstart` and :doc:`../mp/configuration` for the full " +"list of ``lmcache server`` and connector options." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:53 +msgid "Start vLLM with the MP connector in a separate terminal:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:62 +msgid "" +"**Where does** ``LMCacheMPConnector`` **resolve to?** This depends on " +"your vLLM version:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:64 +msgid "" +"**vLLM < 0.20.0** -- ``\"kv_connector\":\"LMCacheMPConnector\"`` always " +"resolves to vLLM's built-in " +"``vllm.distributed.kv_transfer.kv_connector.v1.LMCacheMPConnector``; " +"there is no way to redirect it to the LMCache-shipped implementation." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:69 +msgid "" +"**vLLM >= 0.20.0** -- ``\"kv_connector\":\"LMCacheMPConnector\"`` still " +"defaults to vLLM's built-in connector, but you can opt in to the LMCache-" +"shipped implementation " +"(:mod:`lmcache.integration.vllm.lmcache_mp_connector`) by adding " +"``kv_connector_module_path``:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:81 +msgid "" +"The LMCache-shipped connector tracks the latest LMCache server protocol " +"and ships fixes/features ahead of the version vendored into vLLM, so " +"prefer it whenever you are on vLLM 0.20.0 or newer." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:85 +#: ../../source/getting_started/quickstart.rst:171 +#: ../../source/getting_started/quickstart.rst:256 +msgid "" +"**Test** -- open a new terminal and send two requests whose prompts share" +" a prefix:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:88 +#: ../../source/getting_started/quickstart.rst:174 +#: ../../source/getting_started/quickstart.rst:259 +msgid "**First request**" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:101 +#: ../../source/getting_started/quickstart.rst:187 +#: ../../source/getting_started/quickstart.rst:272 +msgid "**Second request**" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:114 +msgid "" +"**You should see LMCache logs like this** -- in MP mode the " +"store/retrieve logs come from the standalone ``lmcache server`` process, " +"one entry per chunk." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:118 +msgid "**First request** -- cache is empty, so every aligned chunk is offloaded:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:128 +msgid "" +"**Second request** -- the shared prefix is retrieved from CPU RAM; only " +"the new tail is stored:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:138 +msgid "" +"For request-level statistics (hit ratio, bytes transferred) see " +":doc:`../mp/observability`." +msgstr "" + +#: ../../source/getting_started/quickstart.rst +msgid "In-process mode" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:144 +msgid "Start vLLM with LMCache embedded in the engine process:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:155 +msgid "" +"To customize further, create a config file. See " +":doc:`../api_reference/configurations` for all options." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:158 +msgid "**Alternative simpler command:**" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:167 +msgid "" +"The ``--disable-hybrid-kv-cache-manager`` flag is mandatory. All " +"configuration options from the :doc:`../api_reference/configurations` " +"page still apply." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:200 +msgid "" +"**You should see LMCache logs like this** -- in-process mode emits the " +"logs inline with the vLLM engine core." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:203 +msgid "**First request** -- prompt is offloaded to LMCache:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:209 +msgid "**Second request** -- hits the cache and stores the new tail:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:218 +msgid "**Total tokens 32**: The new prompt has 32 tokens after tokenization." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:219 +msgid "" +"**LMCache hit tokens: 24**: 24 tokens (full 8-token chunks) were found in" +" the cache from the first request that stored 31 tokens." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:220 +msgid "" +"**Need to load: 8**: vLLM auto prefix caching uses block size 16; 16 " +"tokens already sit in GPU RAM, so LMCache only loads 24-16=8." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:221 +msgid "" +"**Why 24 hit tokens instead of 31?** LMCache hashes every 8 tokens (8, " +"16, 24, 31). It matches page-aligned chunks, so it uses the 24-token " +"hash." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:222 +msgid "" +"**Stored another 8 tokens**: The new 8 tokens form a full chunk and are " +"stored for future reuse." +msgstr "" + +#: ../../source/getting_started/quickstart.rst +msgid "SGLang" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:226 +msgid "**Install SGLang**" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:234 +msgid "**Start SGLang with LMCache**" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:254 +msgid "" +"Configure LMCache via the config file. See " +":doc:`../api_reference/configurations` for the full list." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:285 +msgid "**You should see LMCache logs like this:**" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:287 +msgid "**First request** -- prompt plus generated tokens are stored:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:296 +msgid "" +"**Second request** -- Radix Cache and LMCache share the prefix; only the " +"new portion is stored:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:306 +msgid "" +"**Total tokens 140**: SGLang stores KV cache for both prefill and decode " +"tokens together, so total = 40 prompt + 100 generated = 140 tokens." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:307 +msgid "" +"**Cached tokens: 30**: SGLang's Radix Attention Cache reused 30 tokens " +"from the first request." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:308 +msgid "" +"**LMCache hit tokens: 24**: LMCache detected 24 tokens (3 full 8-token " +"chunks) stored from the first request. Since Radix Cache already provides" +" 30 tokens in GPU memory, these 24 tokens don't need to be loaded from " +"LMCache or stored again." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:309 +msgid "" +"**New tokens: 10**: Only 10 prompt tokens need prefill computation (40 " +"prompt - 30 cached = 10)." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:310 +msgid "" +"**Stored 112 out of 140**: 24 tokens (3 full chunks) are already in " +"LMCache and skipped. Of the remaining 116 tokens, 112 (14 full 8-token " +"chunks) are stored." +msgstr "" + +#: ../../source/getting_started/quickstart.rst +msgid "TensorRT-LLM" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:315 +msgid "" +"This integration depends on the connector preset registry from `NVIDIA" +"/TensorRT-LLM PR #12626 `_ and the matching LMCache adapter, neither of which has " +"shipped in a stable release yet. Until they do, install both from source:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:332 +msgid "Once both ship in a stable release, the install command will be:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:339 +msgid "" +"LMCache integrates with TensorRT-LLM via TRT-LLM's **KV Cache Connector**" +" API and supports two deployment modes:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:342 +msgid "" +"**In-process mode** (``connector: lmcache``) -- LMCache runs as a " +"singleton inside the TRT-LLM process. Simplest setup; no extra service to" +" manage." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:345 +msgid "" +"**MP mode** (``connector: lmcache-mp``) -- LMCache runs as a standalone " +"server. Multiple TRT-LLM workers on the same node can share the cache, " +"and the cache survives a TRT-LLM crash." +msgstr "" + +#: ../../source/getting_started/quickstart.rst:355 +msgid "Configure LMCache via env vars:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:364 +msgid "Build the TRT-LLM ``LLM`` with ``connector: lmcache``:" +msgstr "" + +#: ../../source/getting_started/quickstart.rst +msgid "MP mode" +msgstr "" + +#: ../../source/getting_started/quickstart.rst:386 +msgid "" +"``PYTHONHASHSEED=0`` must be set in **both** terminals -- chunk hashing " +"depends on a stable ``hash()``, and the server and client must agree on " +"the seed." +msgstr "``PYTHONHASHSEED=0`` 必须在 **两个** 终端中设置 -- 块哈希依赖于稳定的 ``hash()``, 服务器和客户端必须就种子达成一致。" + +#: ../../source/getting_started/quickstart.rst:398 +msgid "In a separate terminal, point TRT-LLM at the server via ``server_url``:" +msgstr "在另一个终端中，通过 ``server_url`` 将 TRT-LLM 指向服务器：" + +#: ../../source/getting_started/quickstart.rst:406 +msgid "where ``run_trtllm.py`` contains:" +msgstr "其中 ``run_trtllm.py`` 包含：" + +#: ../../source/getting_started/quickstart.rst:429 +msgid "" +"The TRT-LLM adapter reads :class:`LMCacheEngineConfig` the same way the " +"vLLM adapter does: ``LMCACHE_CONFIG_FILE`` for a YAML file, otherwise " +"individual ``LMCACHE_*`` environment variables. See " +":doc:`../api_reference/configurations` for all options." +msgstr "TRT-LLM 适配器以与 vLLM 适配器相同的方式读取 :class:`LMCacheEngineConfig`：对于 YAML 文件使用 ``LMCACHE_CONFIG_FILE``，否则使用单独的 ``LMCACHE_*`` 环境变量。有关所有选项，请参见 :doc:`../api_reference/configurations`。" + +#: ../../source/getting_started/quickstart.rst:435 +msgid "" +"🎉 **You now have LMCache caching and reusing KV caches across all three " +"engines.**" +msgstr "🎉 **您现在可以在所有三个引擎中使用 LMCache 缓存和重用 KV 缓存。**" + +#: ../../source/getting_started/quickstart.rst:438 +msgid "Next Steps" +msgstr "下一步" + +#: ../../source/getting_started/quickstart.rst:440 +msgid "" +"**Performance Testing**: Try the :doc:`benchmarking` section to " +"experience LMCache's performance benefits with more comprehensive " +"examples" +msgstr "**性能测试**：尝试 :doc:`benchmarking` 部分，以体验 LMCache 的性能优势和更全面的示例。" + +#: ../../source/getting_started/quickstart.rst:441 +msgid "" +"**More Examples**: Explore the :doc:`quickstart/index` section for " +"detailed examples including KV cache sharing across instances and " +"disaggregated prefill" +msgstr "**更多示例**：查看 :doc:`quickstart/index` 部分以获取详细示例，包括跨实例的 KV Cache 共享和分离式 Prefill。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/disaggregated_prefill.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/disaggregated_prefill.po new file mode 100644 index 00000000000..16c54ece840 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/disaggregated_prefill.po @@ -0,0 +1,268 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:4 +msgid "Example: Disaggregated prefill" +msgstr "示例：分离式 Prefill" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:6 +msgid "" +"With LMCache as a KV cache transfer library, we can run disaggregated " +"prefill with vLLM. Right now, LMCache uses NIXL as a transport layer to " +"enable fast KV cache transfer via NVLink, RDMA, or TCP." +msgstr "使用 LMCache 作为 KV Cache 传输库，我们可以使用 vLLM 运行分离式 Prefill。目前，LMCache 使用 NIXL 作为传输层，通过 NVLink、RDMA 或 TCP 实现快速的 KV Cache 传输。" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:9 +msgid "" +"This guide demonstrates how to run LMCache with disaggregated prefill " +"using a single prefiller and decoder setup (1P1D) on a single machine. " +"The architecture splits the LLM inference into two stages: prefill and " +"decode, running on separate GPUs for better resource utilization." +msgstr "本指南演示如何在单台机器上使用单个预填充器和解码器设置（1P1D）运行带有分离式 Prefill 的 LMCache。该架构将 LLM 推理分为两个阶段：预填充和解码，分别在不同的 GPU 上运行，以更好地利用资源。" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:13 +msgid "Prerequisites" +msgstr "先决条件" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:15 +msgid "Before you begin, ensure you have:" +msgstr "在开始之前，请确保您具备：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:17 +msgid "At least 2 GPUs" +msgstr "至少 2 个 GPU" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:18 +msgid "Python packages installed:" +msgstr "已安装的 Python 包：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:19 +msgid "``lmcache`` (0.2.1 or above)" +msgstr "``lmcache`` (0.2.1 或更高版本)" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:20 +msgid "" +"``nixl`` (Install instructions `here `_)" +msgstr "``nixl`` (安装说明 `这里 `_)" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:21 +msgid "``vllm`` (latest main branch)" +msgstr "``vllm`` (最新主分支)" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:22 +msgid "``httpx``, ``fastapi``, and ``uvicorn``" +msgstr "``httpx``, ``fastapi``, 和 ``uvicorn``" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:23 +msgid "" +"A valid Hugging Face token (``HF_TOKEN``) with access to Llama 3.1 8B " +"models" +msgstr "一个有效的 Hugging Face 令牌 (``HF_TOKEN``)，具有访问 Llama 3.1 8B 模型的权限" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:25 +msgid "(Recommended) A machine with NVLink or RDMA enabled GPUs" +msgstr "（推荐）一台启用 NVLink 或 RDMA 的 GPU 机器" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:29 +msgid "" +"You can use ``ucx_perftest`` to check the GPU-GPU memory transfer and " +"verify the NVLink or RDMA connection. Please refer to this link: `UCX " +"Performance Test `_." +msgstr "您可以使用 ``ucx_perftest`` 来检查 GPU-GPU 内存传输并验证 NVLink 或 RDMA 连接。请参阅此链接： `UCX Performance Test `_。" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:33 +msgid "Architecture Overview" +msgstr "架构概述" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:35 +msgid "The disaggregated prefill setup consists of three main components:" +msgstr "分离式 Prefill 设置由三个主要组件组成：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:37 +msgid "" +"**Prefiller Server (Port 8100)**: Handles the prefill phase of LLM " +"inference" +msgstr "**Prefill 服务器 (端口 8100)**: 处理 LLM 推理的 Prefill 阶段" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:38 +msgid "**Decoder Server (Port 8200)**: Manages the decoding/generation phase" +msgstr "**解码器服务器 (端口 8200)**: 管理解码/生成阶段" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:39 +msgid "**Proxy Server (Port 9000)**: Coordinates between prefiller and decoder" +msgstr "**代理服务器 (端口 9000)**: 协调预填充器和解码器之间的关系" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:42 +msgid "Configuration" +msgstr "配置" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:44 +msgid "**Prefiller Server Configuration** (``lmcache-prefiller-config.yaml``):" +msgstr "**Prefiller Server Configuration** (``lmcache-prefiller-config.yaml``):" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:59 +msgid "**Decoder Server Configuration** (``lmcache-decoder-config.yaml``):" +msgstr "**解码器服务器配置** (``lmcache-decoder-config.yaml``):" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:76 +msgid "Step-by-Step Setup" +msgstr "逐步设置" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:78 +msgid "**Environment Setup**" +msgstr "**环境设置**" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:80 +msgid "Set your Hugging Face token before running the vLLM servers." +msgstr "在运行 vLLM 服务器之前，请设置您的 Hugging Face 令牌。" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:86 +msgid "**Launch the vLLM + LMCache Inference Servers**" +msgstr "**启动 vLLM + LMCache 推理服务器**" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:88 +msgid "You can launch the components individually:" +msgstr "您可以单独启动各个组件：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:90 +msgid "Launch Decoder (on GPU 1):" +msgstr "启动解码器（在 GPU 1 上）：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:103 +msgid "Launch Prefiller (on GPU 0):" +msgstr "启动 Prefill（在 GPU 0 上）：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:116 +msgid "Launch a proxy server to coordinate between prefiller and decoder:" +msgstr "启动一个代理服务器，以协调 Prefiller 和解码器：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:118 +msgid "" +"The code for the proxy server is available `in vLLM repo " +"`_." +msgstr "代理服务器的代码可在 `vLLM 仓库 `_ 中找到。" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:138 +msgid "" +"The ``UCX_TLS`` environment variable is used to specify the transport " +"layer for UCX (the example uses NVLink) The ``CUDA_VISIBLE_DEVICES`` " +"environment variable is used to specify the GPUs to use for the servers." +msgstr "``UCX_TLS`` 环境变量用于指定 UCX 的传输层（示例使用 NVLink）。``CUDA_VISIBLE_DEVICES`` 环境变量用于指定服务器使用的 GPU。" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:142 +msgid "**Verify Setup**" +msgstr "**验证设置**" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:144 +msgid "The servers are ready when you can access:" +msgstr "当您可以访问以下内容时，服务器已准备就绪：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:146 +msgid "Prefiller: ``http://localhost:7100/v1/completions``" +msgstr "Prefill: ``http://localhost:7100/v1/completions``" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:147 +msgid "Decoder: ``http://localhost:7200/v1/completions``" +msgstr "解码器: ``http://localhost:7200/v1/completions``" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:148 +msgid "Proxy: ``http://localhost:9100/v1/completions``" +msgstr "代理: ``http://localhost:9100/v1/completions``" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:151 +msgid "Usage" +msgstr "用法" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:153 +msgid "" +"Send requests to the proxy server (port 9000) using either the " +"completions or chat completions endpoint:" +msgstr "通过 completions 或 chat completions 端点向代理服务器（端口 9000）发送请求：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:165 +msgid "" +"You can also test the setup with the following command, which runs vLLM's" +" serving benchmark:" +msgstr "您还可以使用以下命令测试设置，该命令运行 vLLM 的服务基准测试：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:177 +msgid "Monitoring" +msgstr "监控" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:179 +msgid "The prefiller instance will log the throughput of KV cache transfer:" +msgstr "预填充实例将记录 KV Cache 传输的吞吐量：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:181 +msgid "" +"LMCache INFO: Store 5271 tokens takes: 6.5000 ms, throughput: 98.9889 " +"GB/s; offload_time: 2.6594 ms, put_time: 3.4539 ms " +"(cache_engine.py:190:lmcache.v1.cache_engine)" +msgstr "LMCache 信息：存储 5271 个 token 耗时：6.5000 毫秒，吞吐量：98.9889 GB/s；卸载时间：2.6594 毫秒，放置时间：3.4539 毫秒 (cache_engine.py:190:lmcache.v1.cache_engine)" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:183 +msgid "" +"The decoder instance will log how many tokens are fetched from the " +"LMCache:" +msgstr "解码器实例将记录从 LMCache 中获取了多少个 token：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:185 +msgid "" +"LMCache INFO: Reqid: cmpl-b8bf01cbe47e4d108732ceeb4158d310-0, Total " +"tokens 5170, LMCache hit tokens: 5169, need to load: 5169 " +"(vllm_v1_adapter.py:543:lmcache.integration.vllm.vllm_v1_adapter)" +msgstr "LMCache 信息：请求 ID：cmpl-b8bf01cbe47e4d108732ceeb4158d310-0，总令牌数 5170，LMCache 命中令牌：5169，需要加载：5169 (vllm_v1_adapter.py:543:lmcache.integration.vllm.vllm_v1_adapter)" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:187 +msgid "The proxy server will log the TTFT of the prefiller node:" +msgstr "代理服务器将记录预填充节点的 TTFT：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:201 +msgid "Troubleshooting" +msgstr "故障排除" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:203 +msgid "Common issues and solutions:" +msgstr "常见问题及解决方案：" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:205 +msgid "**GPU Requirements**: Ensure you have at least 2 GPUs available" +msgstr "**GPU 要求**：确保您至少有 2 个可用的 GPU" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:206 +msgid "**Port Conflicts**: Check if ports used above are available" +msgstr "**端口冲突**：检查上述使用的端口是否可用" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:207 +msgid "" +"**HF Token**: Verify your token starts with ``hf_`` and has necessary " +"model access" +msgstr "**HF Token**: 验证您的令牌是否以 ``hf_`` 开头，并具有必要的模型访问权限" + +#: ../../source/getting_started/quickstart/disaggregated_prefill.rst:208 +msgid "" +"**CUDA Errors**: Ensure CUDA_VISIBLE_DEVICES is set correctly for each " +"server" +msgstr "**CUDA 错误**：确保每个服务器的 CUDA_VISIBLE_DEVICES 设置正确" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/index.po new file mode 100644 index 00000000000..65b33ef256a --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/index.po @@ -0,0 +1,161 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/quickstart/index.rst:4 +msgid "More Examples" +msgstr "更多示例" + +#: ../../source/getting_started/quickstart/index.rst:6 +msgid "" +"This section provides quick examples to help you get started with " +"LMCache's key features." +msgstr "本节提供快速示例，以帮助您开始使用 LMCache 的关键功能。" + +#: ../../source/getting_started/quickstart/index.rst:9 +msgid "KV Cache Offloading" +msgstr "KV Cache 卸载" + +#: ../../source/getting_started/quickstart/index.rst:11 +msgid "" +"KV cache offloading allows you to move KV caches from GPU memory to CPU " +"memory or other storage devices. This feature is particularly useful " +"when:" +msgstr "KV Cache 卸载允许您将 KV 缓存从显存移动到 CPU 内存或其他存储设备。此功能在以下情况下特别有用：" + +#: ../../source/getting_started/quickstart/index.rst:13 +msgid "" +"There are requests shares the same prefix (e.g., long system prompt, " +"reusing chat history in chat applications, or caching offline-processed " +"data)." +msgstr "存在共享相同前缀的请求（例如，长系统提示、在聊天应用中重用聊天历史或缓存离线处理的数据）。" + +#: ../../source/getting_started/quickstart/index.rst:14 +msgid "The GPU memory is limited to save all the KV caches." +msgstr "显存有限，无法保存所有的 KV Cache。" + +#: ../../source/getting_started/quickstart/index.rst:16 +msgid "" +"By offloading KV caches, LMCache can reduce both time-to-first-token " +"(TTFT) and GPU cycles." +msgstr "通过卸载 KV 缓存，LMCache 可以减少首次令牌时间 (TTFT) 和显存周期。" + +#: ../../source/getting_started/quickstart/index.rst:18 +msgid "See :ref:`offload_kv_cache` for more details." +msgstr "有关更多详细信息，请参见 :ref:`offload_kv_cache`。" + +#: ../../source/getting_started/quickstart/index.rst:21 +msgid "KV Cache Sharing" +msgstr "KV Cache 共享" + +#: ../../source/getting_started/quickstart/index.rst:23 +msgid "" +"KV cache sharing enables sharing the KV cache across different LLM " +"instances. This feature is beneficial when:" +msgstr "KV Cache 共享允许在不同的 LLM 实例之间共享 KV Cache。当满足以下条件时，此功能非常有用：" + +#: ../../source/getting_started/quickstart/index.rst:25 +msgid "There are multiple LLM instances running in the same system." +msgstr "在同一系统中运行多个 LLM 实例。" + +#: ../../source/getting_started/quickstart/index.rst:26 +msgid "The requests that share the same prefix may go to different LLM instances." +msgstr "具有相同前缀的请求可能会发送到不同的 LLM 实例。" + +#: ../../source/getting_started/quickstart/index.rst:28 +msgid "" +"Sharing KV caches also reduces TTFT and GPU computation by eliminating " +"redundant calculations across different LLM instances." +msgstr "共享 KV Cache 还通过消除不同 LLM 实例之间的冗余计算来减少 TTFT 和 GPU 计算。" + +#: ../../source/getting_started/quickstart/index.rst:30 +msgid "See :ref:`share_kv_cache` for more details." +msgstr "有关更多详细信息，请参阅 :ref:`share_kv_cache`。" + +#: ../../source/getting_started/quickstart/index.rst:33 +msgid "Disaggregated Prefill" +msgstr "分离式 Prefill" + +#: ../../source/getting_started/quickstart/index.rst:35 +msgid "" +"Disaggregated prefill separates the prefill and decode phases across " +"different compute resources. This approach:" +msgstr "分离式 Prefill 将预填充和解码阶段分开，使用不同的计算资源。此方法：" + +#: ../../source/getting_started/quickstart/index.rst:37 +msgid "Allows specialized hardware allocation for each phase of inference" +msgstr "允许为推理的每个阶段分配专用硬件" + +#: ../../source/getting_started/quickstart/index.rst:38 +msgid "Enables more efficient resource utilization in distributed settings" +msgstr "在分布式环境中实现更高效的资源利用。" + +#: ../../source/getting_started/quickstart/index.rst:39 +msgid "" +"Improves overall system throughput by optimizing for the different " +"computational patterns of prefill vs. decode" +msgstr "通过优化 Prefill 和解码的不同计算模式，提高整体系统吞吐量。" + +#: ../../source/getting_started/quickstart/index.rst:41 +msgid "" +"This architecture is particularly valuable in large-scale deployment " +"scenarios where maximizing resource efficiency and keeping a stable " +"generation speed are both important." +msgstr "这种架构在大规模部署场景中尤其有价值，在这些场景中，最大化资源效率和保持稳定的生成速度都很重要。" + +#: ../../source/getting_started/quickstart/index.rst:43 +msgid "See :ref:`disaggregated_prefill` for more details." +msgstr "有关更多详细信息，请参见 :ref:`disaggregated_prefill`。" + +#: ../../source/getting_started/quickstart/index.rst:46 +msgid "Standalone Starter" +msgstr "独立启动器" + +#: ../../source/getting_started/quickstart/index.rst:48 +msgid "" +"The LMCache Standalone Starter allows you to run LMCacheEngine as a " +"standalone service without vLLM or GPU dependencies. This is particularly" +" useful for:" +msgstr "LMCache 独立启动器允许您将 LMCacheEngine 作为独立服务运行，而无需 vLLM 或 GPU 依赖。这在以下情况下特别有用：" + +#: ../../source/getting_started/quickstart/index.rst:50 +msgid "Testing and development environments" +msgstr "测试和开发环境" + +#: ../../source/getting_started/quickstart/index.rst:51 +msgid "CPU-only deployments" +msgstr "仅 CPU 部署" + +#: ../../source/getting_started/quickstart/index.rst:52 +msgid "Distributed cache scenarios" +msgstr "分布式缓存场景" + +#: ../../source/getting_started/quickstart/index.rst:53 +msgid "Integration with custom applications" +msgstr "与自定义应用程序集成" + +#: ../../source/getting_started/quickstart/index.rst:55 +msgid "See :ref:`standalone_starter` for more details." +msgstr "有关更多详细信息，请参阅 :ref:`standalone_starter`。" + +#: ../../source/getting_started/quickstart/index.rst:58 +msgid "Detailed Examples" +msgstr "详细示例" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/multimodality.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/multimodality.po new file mode 100644 index 00000000000..f088a764fbd --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/multimodality.po @@ -0,0 +1,73 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/quickstart/multimodality.rst:2 +msgid "Example: Multimodal KV Cache Support" +msgstr "示例：多模态 KV Cache 支持" + +#: ../../source/getting_started/quickstart/multimodality.rst:5 +msgid "Quick Start Example (Audio Model):" +msgstr "快速入门示例（音频模型）：" + +#: ../../source/getting_started/quickstart/multimodality.rst:7 +msgid "" +"We going to be running audio inference with " +"``ultravox-v0_5-llama-3_2-1b`` and using LMCache to speed up the TTFT " +"after the first request." +msgstr "我们将使用 ``ultravox-v0_5-llama-3_2-1b`` 进行音频推理，并使用 LMCache 加速第一次请求后的 TTFT。" + +#: ../../source/getting_started/quickstart/multimodality.rst:9 +msgid "**Install and Serve:**" +msgstr "**安装和服务：**" + +#: ../../source/getting_started/quickstart/multimodality.rst:11 +msgid "``pip install lmcache vllm[audio] openai``" +msgstr "``pip install lmcache vllm[audio] openai``" + +#: ../../source/getting_started/quickstart/multimodality.rst:20 +msgid "**Benchmark:**" +msgstr "**基准测试：**" + +#: ../../source/getting_started/quickstart/multimodality.rst:22 +msgid "Save as ``audio_query.py``" +msgstr "保存为 ``audio_query.py``" + +#: ../../source/getting_started/quickstart/multimodality.rst:169 +msgid "**Run and see TTFT speedup:**" +msgstr "**运行并查看 TTFT 加速：**" + +#: ../../source/getting_started/quickstart/multimodality.rst:180 +msgid "**Retrieval and speed up in logs:**" +msgstr "**检索和日志中的加速：**" + +#: ../../source/getting_started/quickstart/multimodality.rst:182 +msgid "After First Request:" +msgstr "首次请求后：" + +#: ../../source/getting_started/quickstart/multimodality.rst:191 +#: ../../source/getting_started/quickstart/multimodality.rst:213 +msgid "*Example Output:*" +msgstr "*示例输出:*" + +#: ../../source/getting_started/quickstart/multimodality.rst:204 +msgid "After Second Request:" +msgstr "第二次请求后：" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/offload_kv_cache.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/offload_kv_cache.po new file mode 100644 index 00000000000..0f62ab3ba1f --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/offload_kv_cache.po @@ -0,0 +1,301 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:4 +msgid "Example: Offload KV cache to CPU" +msgstr "示例：将 KV Cache 卸载到 CPU" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:6 +msgid "In this example, we will show you how to offload KV cache to CPU memory." +msgstr "在这个示例中，我们将向您展示如何将 KV Cache 卸载到 CPU 内存。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:9 +msgid "" +"Besides CPU memory, LMCache also supports offloading KV cache to many " +"different destinations. See " +":ref:`getting_started/quickstart/offload_kv_cache:Supported offloading " +"destinations` for more details." +msgstr "除了 CPU 内存，LMCache 还支持将 KV Cache 卸载到许多不同的目标。有关更多详细信息，请参见 :ref:`getting_started/quickstart/offload_kv_cache:Supported offloading destinations`。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:13 +msgid "Prerequisites" +msgstr "前提条件" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:15 +msgid "Before you begin, make sure you have:" +msgstr "在开始之前，请确保您具备：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:17 +msgid "vLLM v1 with LMCache installed (see :doc:`Installation <../installation>`)" +msgstr "安装了 LMCache 的 vLLM v1（请参见 :doc:`安装 <../installation>`）" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:18 +msgid "A GPU that can run a LLM" +msgstr "可以运行 LLM 的 GPU" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:22 +msgid "Use CPU offloading in offline inference" +msgstr "在离线推理中使用 CPU 卸载" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:24 +msgid "" +"This section demonstrates how to use CPU memory offloading in offline " +"inference scenarios using LMCache with vLLM. The example script we use " +"here is available in `vLLM examples `_." +" See the `examples README `_ " +"to understand how to run the script for vLLM v1." +msgstr "本节演示如何在离线推理场景中使用 LMCache 和 vLLM 进行 CPU 内存卸载。我们在这里使用的示例脚本可以在 `vLLM examples `_ 中找到。请参阅 `examples README `_ 以了解如何运行 vLLM v1 的脚本。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:28 +msgid "First, set up the necessary environment variables for LMCache:" +msgstr "首先，设置 LMCache 所需的环境变量：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:41 +msgid "Next, configure vLLM with LMCache integration:" +msgstr "接下来，配置 vLLM 与 LMCache 集成：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:61 +msgid "Now you can run inference with automatic KV cache offloading:" +msgstr "现在您可以通过自动卸载 KV Cache 来进行推理：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:80 +msgid "When the inference is complete, clean up the LMCache backend:" +msgstr "推理完成后，清理 LMCache 后端：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:89 +msgid "" +"During inference, LMCache will automatically handle storing and managing " +"KV cache in CPU memory. You can monitor this through the logs, which will" +" show messages like::" +msgstr "在推理过程中，LMCache 将自动处理在 CPU 内存中存储和管理 KV Cache。您可以通过日志监控这一过程，日志中会显示类似于以下消息：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:93 +msgid "" +"This indicates that the KV cache has been successfully offloaded to CPU " +"memory." +msgstr "这表明 KV Cache 已成功卸载到 CPU 内存。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:96 +msgid "Adjust ``gpu_memory_utilization`` based on your GPU's available memory" +msgstr "根据您的 GPU 可用内存调整 ``gpu_memory_utilization``" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:97 +msgid "" +"The CPU offloading buffer size can be adjusted through " +"``LMCACHE_MAX_LOCAL_CPU_SIZE``" +msgstr "可以通过 ``LMCACHE_MAX_LOCAL_CPU_SIZE`` 调整 CPU 卸载缓冲区大小。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:100 +msgid "Use CPU offloading in online inference" +msgstr "在在线推理中使用 CPU 卸载" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:102 +msgid "" +"This section demonstrates how to use CPU memory offloading in online " +"serving scenarios." +msgstr "本节演示如何在在线服务场景中使用 CPU 内存卸载。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:104 +msgid "" +"First, create a configuration file named ``lmcache_config.yaml`` with the" +" following content:" +msgstr "首先，创建一个名为 ``lmcache_config.yaml`` 的配置文件，内容如下：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:113 +msgid "" +"LMCache supports extensive configuration through a " +"``lmcache_config.yaml`` file where you can customize chunk sizes, memory " +"limits, storage backends, and more. We'll cover advanced configuration " +"options in later examples. For now, let's run a minimal example with " +"default configuration." +msgstr "LMCache 支持通过 ``lmcache_config.yaml`` 文件进行广泛的配置，您可以在其中自定义块大小、内存限制、存储后端等更多内容。我们将在后面的示例中介绍高级配置选项。现在，让我们运行一个使用默认配置的最小示例。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:115 +msgid "" +"Launch the vLLM server with LMCache integration using environment " +"variables. Here's an example command:" +msgstr "使用环境变量启动集成了 LMCache 的 vLLM 服务器。以下是一个示例命令：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:127 +msgid "Key parameters explained:" +msgstr "关键参数说明：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:129 +msgid "``LMCACHE_CONFIG_FILE``: Path to the LMCache configuration file." +msgstr "``LMCACHE_CONFIG_FILE``: LMCache 配置文件的路径。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:130 +msgid "``--kv-transfer-config``: Configures LMCache integration" +msgstr "``--kv-transfer-config``: 配置 LMCache 集成" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:131 +msgid "``kv_connector``: Specifies the LMCache connector" +msgstr "``kv_connector``: 指定 LMCache 连接器" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:132 +msgid "``kv_role``: Set to \"kv_both\" for both storing and loading KV cache" +msgstr "``kv_role``: 设置为 \\\"kv_both\\\" 以同时存储和加载 KV Cache" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:134 +msgid "" +"Once the server is running, you can send requests to it using curl. " +"Here's an example of how to send a request to the vLLM server with " +"LMCache integration:" +msgstr "一旦服务器运行起来，您可以使用 curl 向其发送请求。以下是如何向集成了 LMCache 的 vLLM 服务器发送请求的示例：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:147 +msgid "You should see the following logs:" +msgstr "您应该看到以下日志：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:154 +msgid "" +"Once you send the same curl request again, you should see the following " +"logs:" +msgstr "一旦您再次发送相同的 curl 请求，您应该会看到以下日志：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:163 +msgid "Example: CPU offloading benefits" +msgstr "示例：CPU 卸载的好处" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:165 +msgid "" +"This section demonstrates the performance benefits of using CPU " +"offloading with LMCache. We'll use a script that generates multiple " +"prompts and compare the performance with and without LMCache." +msgstr "本节演示了使用 CPU 卸载与 LMCache 结合的性能优势。我们将使用一个生成多个提示的脚本，并比较使用和不使用 LMCache 的性能。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:168 +msgid "Prerequisites (Setup)" +msgstr "前提条件（设置）" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:170 +msgid "At least 24GB GPU memory" +msgstr "至少 24GB 显存" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:171 +msgid "Sufficient CPU memory (LMCache will use 15 GB by default in this example)." +msgstr "足够的 CPU 内存（在此示例中，LMCache 默认将使用 15 GB）。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:174 +msgid "Example script" +msgstr "示例脚本" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:176 +msgid "Save the following script as ``cpu-offloading.py``:" +msgstr "将以下脚本保存为 ``cpu-offloading.py``：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:343 +msgid "Running the Example" +msgstr "运行示例" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:345 +msgid "First, run the script without LMCache:" +msgstr "首先，运行不带 LMCache 的脚本：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:351 +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:366 +msgid "You'll see output like:" +msgstr "您将看到类似的输出：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:357 +msgid "" +"Without LMCache, there's no speedup between runs even if vLLM has prefix " +"caching enabled. This is because the KV cache exceeds GPU memory and " +"can't be reused." +msgstr "没有 LMCache，即使 vLLM 启用了前缀缓存，运行之间也没有加速。这是因为 KV Cache 超出了显存，无法被重用。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:360 +msgid "Now, run with LMCache enabled:" +msgstr "现在，启用 LMCache 运行：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:372 +msgid "" +"The significant speedup in the second case demonstrates how LMCache " +"effectively manages KV cache offloading to CPU memory. When the total " +"size of KV cache exceeds GPU memory, LMCache allows you to store and " +"reuse the cache from CPU memory, resulting in much faster subsequent " +"generations for prompts with shared prefixes." +msgstr "第二种情况显著的加速展示了 LMCache 如何有效管理 KV Cache 卸载到 CPU 内存。当 KV Cache 的总大小超过显存时，LMCache 允许您从 CPU 内存中存储和重用缓存，从而为具有共享前缀的提示生成更快的后续结果。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:378 +msgid "Supported offloading destinations" +msgstr "支持的卸载目标" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:380 +msgid "LMCache now supports offloading KV cache to the following destinations:" +msgstr "LMCache 现在支持将 KV Cache 卸载到以下目标：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:382 +msgid ":doc:`CPU memory <../../kv_cache/storage_backends/cpu_ram>`" +msgstr ":doc:`CPU 内存 <../../kv_cache/storage_backends/cpu_ram>`" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:383 +msgid ":doc:`Local file system <../../kv_cache/storage_backends/local_storage>`" +msgstr ":doc:`本地文件系统 <../../kv_cache/storage_backends/local_storage>`" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:384 +msgid ":doc:`Mooncake Storage <../../kv_cache/storage_backends/mooncake>`" +msgstr ":doc:`Mooncake 存储 <../../kv_cache/storage_backends/mooncake>`" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:385 +msgid ":doc:`InfiniStore <../../kv_cache/storage_backends/infinistore>`" +msgstr ":doc:`InfiniStore <../../kv_cache/storage_backends/infinistore>`" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:386 +msgid ":doc:`Redis <../../kv_cache/storage_backends/redis>`" +msgstr "：doc:`Redis <../../kv_cache/storage_backends/redis>`" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:387 +msgid ":doc:`ValKey <../../kv_cache/storage_backends/valkey>`" +msgstr ":doc:`ValKey <../../kv_cache/storage_backends/valkey>`" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:390 +msgid "Troubleshooting" +msgstr "故障排除" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:392 +msgid "If you encounter the following error:" +msgstr "如果您遇到以下错误：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:399 +msgid "You can resolve this issue using one of the following methods:" +msgstr "您可以通过以下方法解决此问题：" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:401 +msgid "Set ``VLLM_WORKER_MULTIPROC_METHOD=spawn`` in the environment variables." +msgstr "在环境变量中设置 ``VLLM_WORKER_MULTIPROC_METHOD=spawn``。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:402 +msgid "" +"Or update the Python code to guard usage of vllm behind a if ``__name__ " +"== '__main__':`` block." +msgstr "或者更新 Python 代码，将 vllm 的使用放在 ``if __name__ == '__main__':`` 块中。" + +#: ../../source/getting_started/quickstart/offload_kv_cache.rst:413 +msgid "" +"For details, please refer to the `vLLM Troubleshooting Guide: Python " +"multiprocessing " +"`_." +msgstr "有关详细信息，请参阅 `vLLM 故障排除指南：Python 多进程 `_。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/share_kv_cache.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/share_kv_cache.po new file mode 100644 index 00000000000..9b65e35d34e --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/share_kv_cache.po @@ -0,0 +1,246 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:4 +msgid "Example: Share KV cache across multiple LLMs" +msgstr "示例：在多个 LLM 之间共享 KV Cache" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:6 +msgid "" +"LMCache should be able to reduce the generation time of the second and " +"following calls." +msgstr "LMCache 应该能够减少第二次及后续调用的生成时间。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:8 +msgid "" +"We have examples for the following types of across-instance KV cache " +"sharing:" +msgstr "我们有以下类型的跨实例 KV Cache 共享的示例：" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:10 +msgid "" +"KV cache sharing through a centralized cache server: " +"``centralized_sharing``" +msgstr "通过集中式缓存服务器的 KV Cache 共享：``centralized_sharing``" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:11 +msgid "KV cache sharing through p2p cache transfer: ``p2p_sharing``" +msgstr "通过点对点缓存传输共享 KV 缓存: ``p2p_sharing``" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:14 +msgid "Prerequisites" +msgstr "前提条件" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:16 +msgid "Your server should have at least 2 GPUs." +msgstr "您的服务器应至少配备 2 个 GPU。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:18 +msgid "" +"For Centralized sharing, this will use the port 8000 and 8001 (for vLLM) " +"and port 65432 (for LMCache)." +msgstr "对于集中式共享，这将使用端口 8000 和 8001（用于 vLLM）以及端口 65432（用于 LMCache）。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:20 +msgid "For P2P sharing:" +msgstr "对于 P2P 共享：" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:22 +msgid "`NIXL `_ installed on the host." +msgstr "`NIXL `_ 安装在主机上。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:23 +msgid "Port 8010 and 8011 for 2 vllms servers." +msgstr "用于 2 个 vLLM 服务器的 8010 和 8011 端口。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:24 +msgid "Port 8200 and 8202 for 2 p2p initialization connections." +msgstr "端口 8200 和 8202 用于 2 个 p2p 初始化连接。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:25 +msgid "Port 8201 and 8203 for 2 p2p lookup connections." +msgstr "端口 8201 和 8203 用于 2 个 p2p 查找连接。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:26 +msgid "Port 8300 for controller pull requests." +msgstr "端口 8300 用于控制器拉取请求。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:27 +msgid "Port 8400 for controller reply requests." +msgstr "控制器回复请求的端口为 8400。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:28 +msgid "Port 8500 and 8501 for 2 LMCache workers." +msgstr "8500 和 8501 端口用于 2 个 LMCache 工作线程。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:29 +msgid "" +"Port 9000 for controller main port (arbitrary and can be changed) to " +"start the controller." +msgstr "端口 9000 用于控制器主端口（任意且可以更改），以启动控制器。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:32 +msgid "Centralized KV cache sharing" +msgstr "集中式 KV Cache 共享" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:34 +msgid "" +"This section demonstrates how to share KV cache across multiple vLLM " +"instances using a centralized LMCache server." +msgstr "本节演示如何通过集中式 LMCache 服务器在多个 vLLM 实例之间共享 KV Cache。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:36 +msgid "" +"**Important**: For centralized cache sharing (which is cross-process " +"cases), ensure all processes use the same `PYTHONHASHSEED` to keep the " +"hash of the KV cache consistent across processes: ``export " +"PYTHONHASHSEED=0``." +msgstr "**重要**：对于集中式缓存共享（跨进程情况），确保所有进程使用相同的 `PYTHONHASHSEED` 以保持 KV 缓存在进程之间的一致性：``export PYTHONHASHSEED=0``。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:39 +msgid "Setup centralized sharing" +msgstr "设置集中式共享" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:41 +msgid "" +"First, create a configuration file named ``lmcache_config.yaml`` with the" +" following content:" +msgstr "首先，创建一个名为 ``lmcache_config.yaml`` 的配置文件，内容如下：" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:51 +msgid "Run centralized sharing example" +msgstr "运行集中式共享示例" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:53 +msgid "Start the LMCache centralized server," +msgstr "启动 LMCache 中央服务器，" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:59 +msgid "In a different terminal," +msgstr "在另一个终端中，" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:71 +msgid "In another terminal," +msgstr "在另一个终端中，" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:84 +msgid "Wait until both engines are ready." +msgstr "等待直到两个引擎都准备好。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:86 +msgid "Send one request to the engine at port 8000," +msgstr "向端口 8000 的引擎发送一个请求，" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:98 +msgid "Send the same request to the engine at port 8001," +msgstr "将相同的请求发送到端口 8001 的引擎。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:110 +msgid "" +"The second request will automatically retrieve and reuse the KV cache " +"from the first instance, significantly reducing generation time." +msgstr "第二个请求将自动从第一个实例中检索并重用 KV Cache，显著减少生成时间。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:113 +msgid "P2P KV cache sharing" +msgstr "P2P KV Cache 共享" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:115 +msgid "" +"This section demonstrates how to share KV cache across multiple vLLM " +"instances using peer-to-peer transfer." +msgstr "本节演示如何通过点对点传输在多个 vLLM 实例之间共享 KV Cache。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:118 +msgid "Configure LMCache instances" +msgstr "配置 LMCache 实例" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:120 +msgid "" +"Create two configuration files for the P2P sharing setup. The values that" +" differ between the files are the ``lmcache_instance_id`` and the " +"P2P/controller port assignments." +msgstr "为 P2P 共享设置创建两个配置文件。两个文件之间不同的值是 ``lmcache_instance_id`` 和 P2P/控制器端口分配。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:122 +msgid "Instance 1 configuration (``p2p_example1.yaml``):" +msgstr "实例 1 配置 (``p2p_example1.yaml``):" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:148 +msgid "Instance 2 configuration (``p2p_example2.yaml``):" +msgstr "实例 2 配置 (``p2p_example2.yaml``):" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:174 +msgid "" +"Save both files in the directory that you will mount into the container " +"(referenced later as ``$YAML_FILES``)." +msgstr "将两个文件保存在您将挂载到容器中的目录中（稍后称为 ``$YAML_FILES``）。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:177 +msgid "Run the P2P sharing workflow" +msgstr "运行 P2P 共享工作流" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:179 +msgid "" +"Configure the environment on the host and open a shell inside the " +"container:" +msgstr "在主机上配置环境并在容器内打开一个 shell：" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:199 +msgid "Start the LMCache controller and monitoring endpoints:" +msgstr "启动 LMCache 控制器和监控端点：" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:205 +msgid "Launch two vLLM engines, each with its own LMCache worker configuration." +msgstr "启动两个 vLLM 引擎，每个引擎都有自己的 LMCache 工作配置。" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:207 +msgid "Start vLLM engine 1 on GPU 0:" +msgstr "在 GPU 0 上启动 vLLM 引擎 1：" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:217 +msgid "Start vLLM engine 2 on GPU 1:" +msgstr "在 GPU 1 上启动 vLLM 引擎 2：" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:227 +msgid "Populate the KV cache by sending a request to the first engine:" +msgstr "通过向第一个引擎发送请求来填充 KV Cache：" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:239 +msgid "Send the same request to the second engine to demonstrate cache retrieval:" +msgstr "将相同的请求发送到第二个引擎以演示缓存检索：" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:252 +msgid "Expected output" +msgstr "预期输出" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:254 +msgid "" +"When the second request successfully retrieves the cache from the first " +"instance, the logs should include entries similar to:" +msgstr "当第二个请求成功从第一个实例检索到缓存时，日志应包含类似于以下内容的条目：" + +#: ../../source/getting_started/quickstart/share_kv_cache.rst:265 +msgid "" +"These logs indicate that the peer connection was established and the " +"cache was transferred successfully." +msgstr "这些日志表明对等连接已建立，并且缓存已成功传输。" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/standalone_starter.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/standalone_starter.po new file mode 100644 index 00000000000..20696a66bb9 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/standalone_starter.po @@ -0,0 +1,372 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:4 +msgid "Standalone Starter" +msgstr "独立启动器" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:6 +msgid "" +"The LMCache Standalone Starter allows you to run LMCacheEngine as a " +"standalone service without vLLM or GPU dependencies. This is particularly" +" useful for:" +msgstr "LMCache 独立启动器允许您将 LMCacheEngine 作为独立服务运行，而无需 vLLM 或 GPU 依赖。这对于以下情况特别有用：" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:8 +msgid "Testing and development environments" +msgstr "测试和开发环境" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:9 +msgid "CPU-only or P2P backend deployments" +msgstr "仅 CPU 或 P2P 后端部署" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:12 +msgid "Quick Start" +msgstr "快速开始" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:15 +msgid "Basic Usage" +msgstr "基本用法" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:30 +msgid "CPU-Only Mode" +msgstr "CPU-Only 模式" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:41 +msgid "Remote P2P Mode" +msgstr "远程 P2P 模式" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:42 +msgid "TO be added" +msgstr "待添加" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:45 +msgid "Configuration Section" +msgstr "配置部分" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:47 +msgid "" +"The standalone starter supports multiple configuration sources with the " +"following priority order:" +msgstr "独立启动器支持多个配置源，优先级顺序如下：" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:49 +msgid "**Command-line arguments** (highest priority)" +msgstr "**命令行参数**（最高优先级）" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:50 +msgid "" +"**Configuration file** (specified by ``--config`` or " +"``LMCACHE_CONFIG_FILE``)" +msgstr "**配置文件**（由 ``--config`` 或 ``LMCACHE_CONFIG_FILE`` 指定）" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:51 +msgid "**Environment variables** (e.g., ``LMCACHE_CHUNK_SIZE=512``)" +msgstr "**环境变量**（例如，``LMCACHE_CHUNK_SIZE=512``）" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:52 +msgid "**Default values** (lowest priority)" +msgstr "**默认值**（最低优先级）" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:55 +msgid "Parameter Details" +msgstr "参数详细信息" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:57 +msgid "**KV Cache Shape Specification**" +msgstr "**KV Cache 形状规范**" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:59 +msgid "" +"The ``--kvcache_shape_spec`` parameter supports multi-layer group " +"configurations:" +msgstr "``--kvcache_shape_spec`` 参数支持多层组配置：" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:61 +msgid "Format: ``(shape_string):dtype:layer_count;[...]``" +msgstr "格式: ``(shape_string):dtype:layer_count;[...]``" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:62 +msgid "shape_string: comma-separated shape (e.g., '2,2,256,4,16')" +msgstr "shape_string: 以逗号分隔的形状（例如，'2,2,256,4,16'）" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:63 +msgid "" +"Examples: - Single group: ``(2,2,256,4,16):float16:2`` - Multiple groups:" +" ``(2,2,256,4,16):float16:2;(3,2,256,4,4):float32:3``" +msgstr "示例： - 单个组：``(2,2,256,4,16):float16:2`` - 多个组：``(2,2,256,4,16):float16:2;(3,2,256,4,4):float32:3``" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:67 +msgid "**Device Support**" +msgstr "**设备支持**" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:69 +msgid "``--device=cpu``: CPU-only mode (default)" +msgstr "``--device=cpu``: 仅 CPU 模式（默认）" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:70 +msgid "``--device=cuda``: CUDA GPU acceleration" +msgstr "``--device=cuda``: CUDA GPU 加速" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:71 +msgid "``--device=xpu``: XPU GPU acceleration" +msgstr "``--device=xpu``: XPU GPU 加速" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:73 +msgid "**MLA (Multi-Level Attention)**" +msgstr "**多级注意力 (MLA)**" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:75 +msgid "``--use_mla``: Enable MLA for improved attention performance" +msgstr "``--use_mla``: 启用 MLA 以提高注意力性能" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:76 +msgid "Requires compatible model and configuration" +msgstr "需要兼容的模型和配置" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:78 +msgid "**Cache Formats**" +msgstr "**缓存格式**" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:80 +msgid "``--fmt=vllm``: vLLM-compatible format (default)" +msgstr "``--fmt=vllm``: vLLM 兼容格式（默认）" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:81 +msgid "Supports other formats for different inference engines" +msgstr "支持其他格式以适应不同的推理引擎" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:84 +msgid "Command-Line Parameters" +msgstr "命令行参数" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:87 +msgid "Basic Parameters" +msgstr "基本参数" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:104 +msgid "Usage Examples" +msgstr "使用示例" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:107 +msgid "Custom Configuration" +msgstr "自定义配置" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:118 +msgid "Multi-Layer Group Configuration" +msgstr "多层组配置" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:130 +msgid "GPU device" +msgstr "GPU 设备" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:144 +msgid "MLA Configuration" +msgstr "MLA 配置" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:157 +msgid "Internal API Server" +msgstr "内部 API 服务器" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:159 +msgid "" +"The standalone starter includes an internal API server for monitoring and" +" management:" +msgstr "独立启动器包括一个用于监控和管理的内部 API 服务器：" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:172 +msgid "Troubleshooting" +msgstr "故障排除" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:175 +msgid "Common Issues" +msgstr "常见问题" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:177 +msgid "" +"**Issue**: \"No config file specified\" **Solution**: Set " +"``LMCACHE_CONFIG_FILE`` or use ``--config`` parameter" +msgstr "**问题**: “未指定配置文件” **解决方案**: 设置 ``LMCACHE_CONFIG_FILE`` 或使用 ``--config`` 参数" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:180 +msgid "" +"**Issue**: \"Failed to connect to controller\" **Solution**: Start " +"controller first: ``python -m lmcache.v1.api_server``" +msgstr "**问题**: “无法连接到控制器” **解决方案**: 首先启动控制器: ``python -m lmcache.v1.api_server``" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:183 +msgid "" +"**Issue**: \"Invalid KV shape specification\" **Solution**: Check format:" +" ``(shape):dtype:layer_count``, e.g., ``(2,2,256,4,16):float16:2``" +msgstr "**问题**: \\\"无效的 KV 形状规范\\\" **解决方案**: 检查格式: ``(shape):dtype:layer_count``，例如，``(2,2,256,4,16):float16:2``" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:186 +msgid "" +"**Issue**: \"Device not available\" **Solution**: Verify device support: " +"use ``--device=cpu`` if GPU not available" +msgstr "**问题**: \"设备不可用\" **解决方案**: 验证设备支持: 如果 GPU 不可用，请使用 ``--device=cpu``" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:189 +msgid "" +"**Issue**: \"MLA configuration error\" **Solution**: Ensure compatible " +"model and check ``--use_mla`` parameter" +msgstr "**问题**: \"MLA 配置错误\" **解决方案**: 确保模型兼容并检查 ``--use_mla`` 参数" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:193 +msgid "Debug Mode" +msgstr "调试模式" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:195 +msgid "Enable debug logging for troubleshooting:" +msgstr "启用调试日志以进行故障排除：" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:203 +msgid "Advanced Debugging" +msgstr "高级调试" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:205 +msgid "For detailed layer group information:" +msgstr "有关详细的层组信息：" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:215 +msgid "Performance Tuning" +msgstr "性能调优" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:218 +msgid "Memory Configuration" +msgstr "内存配置" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:229 +msgid "Layer Group Optimization" +msgstr "层组优化" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:240 +msgid "GPU Acceleration" +msgstr "GPU 加速" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:251 +msgid "MLA Performance" +msgstr "MLA 性能" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:263 +msgid "Best Practices" +msgstr "最佳实践" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:265 +msgid "**Use configuration files** for production deployments" +msgstr "**在生产环境中使用配置文件**" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:266 +msgid "**Set appropriate cache sizes** based on available memory" +msgstr "**根据可用内存设置适当的缓存大小**" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:267 +msgid "**Enable internal API** for monitoring and management" +msgstr "**启用内部 API** 以进行监控和管理" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:268 +msgid "**Monitor logs** for performance and error tracking" +msgstr "**监控日志**以进行性能和错误跟踪" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:269 +msgid "**Use multi-layer group configurations** for complex model architectures" +msgstr "**使用多层组配置** 以支持复杂的模型架构" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:270 +msgid "**Enable MLA** for improved attention performance on supported hardware" +msgstr "**启用 MLA** 以在支持的硬件上提高注意力性能" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:271 +msgid "**Choose appropriate device** based on available resources (CPU/GPU/XPU)" +msgstr "**根据可用资源（CPU/GPU/XPU）选择合适的设备**" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:272 +msgid "**Validate KV shape specifications** before deployment" +msgstr "**在部署之前** 验证 KV 形状规范" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:273 +msgid "**Test with debug logging** when configuring new layer groups" +msgstr "**在配置新层组时**进行调试日志测试" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:274 +msgid "**Optimize chunk sizes** for specific hardware configurations" +msgstr "**为特定硬件配置优化块大小**" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:277 +msgid "Multi-Layer Group Best Practices" +msgstr "多层组最佳实践" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:279 +msgid "Use consistent chunk sizes across layer groups for optimal performance" +msgstr "在层组之间使用一致的块大小以获得最佳性能" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:280 +msgid "Group layers with similar precision requirements together" +msgstr "将具有相似精度要求的层分组在一起" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:281 +msgid "Validate shape specifications in development environment first" +msgstr "首先在开发环境中验证形状规范" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:282 +msgid "Monitor memory usage when using multiple layer groups" +msgstr "在使用多个层组时监控内存使用情况" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:285 +msgid "MLA Configuration Tips" +msgstr "MLA 配置提示" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:287 +msgid "Enable MLA only on supported hardware configurations" +msgstr "仅在支持的硬件配置上启用 MLA" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:288 +msgid "Use bfloat16 or float16 precision for best MLA performance" +msgstr "使用 bfloat16 或 float16 精度以获得最佳 MLA 性能" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:289 +msgid "Test MLA performance impact before production deployment" +msgstr "在生产部署之前测试 MLA 性能影响" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:290 +msgid "Monitor attention performance metrics with MLA enabled" +msgstr "启用 MLA 后监控注意力性能指标" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:293 +msgid "Related Documentation" +msgstr "相关文档" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:295 +msgid ":doc:`../quickstart`" +msgstr ":doc:`../quickstart`" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:296 +msgid ":doc:`../../api_reference/configurations`" +msgstr ":doc:`../../api_reference/configurations`" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:297 +msgid ":doc:`../../kv_cache/storage_backends/index`" +msgstr ":doc:`../../kv_cache/storage_backends/index`" + +#: ../../source/getting_started/quickstart/standalone_starter.rst:298 +msgid ":doc:`../../kv_cache_management/index`" +msgstr ":doc:`../../kv_cache_management/index`" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/troubleshoot.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/troubleshoot.po new file mode 100644 index 00000000000..253541a58db --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/troubleshoot.po @@ -0,0 +1,29 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/getting_started/troubleshoot.rst:2 +msgid "TroubleShoot" +msgstr "" + +#: ../../source/getting_started/troubleshoot.rst:4 +msgid "Coming soon..." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/common_apis.po b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/common_apis.po new file mode 100644 index 00000000000..530a34daf4c --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/common_apis.po @@ -0,0 +1,403 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/internal_api_server/common_apis.rst:4 +msgid "Common APIs" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:6 +msgid "" +"Common APIs are available across all components (scheduler, worker, " +"controller)." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:11 +msgid "Endpoints" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:14 +msgid "``GET /env`` — Environment Variables" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:16 +msgid "Get all environment variables of the running process." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:18 +#: ../../source/internal_api_server/common_apis.rst:47 +#: ../../source/internal_api_server/common_apis.rst:103 +#: ../../source/internal_api_server/common_apis.rst:133 +#: ../../source/internal_api_server/common_apis.rst:163 +#: ../../source/internal_api_server/common_apis.rst:228 +#: ../../source/internal_api_server/common_apis.rst:259 +msgid "**Method**: ``GET``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:19 +msgid "**Path**: ``/env``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:20 +#: ../../source/internal_api_server/common_apis.rst:105 +#: ../../source/internal_api_server/common_apis.rst:120 +#: ../../source/internal_api_server/common_apis.rst:261 +msgid "**Parameters**: None" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:21 +msgid "" +"**Response**: ``application/json`` — JSON object of all environment " +"variables (sorted by key)." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:27 +#: ../../source/internal_api_server/common_apis.rst:188 +#: ../../source/internal_api_server/common_apis.rst:334 +msgid "**Example Response**:" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:39 +msgid "``GET /loglevel`` — Log Level Management" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:41 +msgid "" +"Get or set the log level for Python loggers. Behavior depends on query " +"parameters:" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:43 +msgid "**No parameters**: List all loggers and their levels." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:44 +msgid "**``logger_name`` only**: Get the level of the specified logger." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:45 +msgid "" +"**``logger_name`` and ``level``**: Set the level of the specified logger " +"(including all its handlers)." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:48 +msgid "**Path**: ``/loglevel``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:49 +#: ../../source/internal_api_server/common_apis.rst:135 +#: ../../source/internal_api_server/common_apis.rst:165 +#: ../../source/internal_api_server/common_apis.rst:307 +msgid "**Parameters**:" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:52 +#: ../../source/internal_api_server/common_apis.rst:138 +#: ../../source/internal_api_server/common_apis.rst:168 +#: ../../source/internal_api_server/common_apis.rst:233 +#: ../../source/internal_api_server/common_apis.rst:310 +msgid "Name" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:52 +#: ../../source/internal_api_server/common_apis.rst:138 +#: ../../source/internal_api_server/common_apis.rst:168 +#: ../../source/internal_api_server/common_apis.rst:233 +#: ../../source/internal_api_server/common_apis.rst:310 +msgid "Type" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:52 +#: ../../source/internal_api_server/common_apis.rst:138 +#: ../../source/internal_api_server/common_apis.rst:168 +#: ../../source/internal_api_server/common_apis.rst:233 +#: ../../source/internal_api_server/common_apis.rst:310 +msgid "Description" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:54 +msgid "``logger_name``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:54 +#: ../../source/internal_api_server/common_apis.rst:55 +#: ../../source/internal_api_server/common_apis.rst:140 +#: ../../source/internal_api_server/common_apis.rst:170 +#: ../../source/internal_api_server/common_apis.rst:235 +msgid "str" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:54 +msgid "(Optional) Logger name to query or set" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:55 +#: ../../source/internal_api_server/common_apis.rst:170 +msgid "``level``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:55 +msgid "" +"(Optional) Log level to set (e.g. ``DEBUG``, ``INFO``, ``WARNING``, " +"``ERROR``)" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:58 +msgid "**Response**: ``text/plain``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:71 +msgid "**Example Response** (list all):" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:79 +msgid "**Example Response** (get):" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:85 +msgid "**Example Response** (set):" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:91 +#: ../../source/internal_api_server/common_apis.rst:214 +msgid "**Error Response** (invalid level, HTTP 400):" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:99 +msgid "``GET /metrics`` — Prometheus Metrics" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:101 +msgid "Get Prometheus metrics data in the standard exposition format." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:104 +msgid "**Path**: ``/metrics``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:106 +msgid "**Response**: ``text/plain`` — Prometheus text-based exposition format." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:114 +msgid "``POST /metrics/reset`` — Reset Prometheus Metrics" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:116 +msgid "Reset all Prometheus metrics to their initial state." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:118 +#: ../../source/internal_api_server/common_apis.rst:304 +msgid "**Method**: ``POST``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:119 +msgid "**Path**: ``/metrics/reset``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:121 +msgid "**Response**: ``text/plain`` — ``\"ok\"`` on success." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:129 +msgid "``GET /threads`` — Thread Information" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:131 +msgid "Get information about active threads with optional filtering." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:134 +msgid "**Path**: ``/threads``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:140 +msgid "``name``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:140 +msgid "(Optional) Filter by thread name (fuzzy match, case-insensitive)" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:141 +msgid "``thread_id``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:141 +msgid "int" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:141 +msgid "(Optional) Filter by thread ID" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:144 +msgid "**Response**: ``text/plain`` — Thread info with stack traces and summary." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:159 +msgid "``GET /periodic-threads`` — Periodic Thread Status" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:161 +msgid "Get information about registered periodic threads." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:164 +msgid "**Path**: ``/periodic-threads``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:170 +msgid "" +"(Optional) Filter by thread level: ``critical``, ``high``, ``medium``, " +"``low``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:171 +msgid "``running_only``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:171 +#: ../../source/internal_api_server/common_apis.rst:172 +msgid "bool" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:171 +msgid "(Optional) Only show running threads (default: ``false``)" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:172 +msgid "``active_only``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:172 +msgid "(Optional) Only show active threads (default: ``false``)" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:175 +#: ../../source/internal_api_server/common_apis.rst:238 +#: ../../source/internal_api_server/common_apis.rst:262 +msgid "**Response**: ``application/json``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:224 +#, python-brace-format +msgid "``GET /periodic-threads/{thread_name}`` — Single Periodic Thread" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:226 +msgid "Get detailed information about a specific periodic thread by name." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:229 +#, python-brace-format +msgid "**Path**: ``/periodic-threads/{thread_name}``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:230 +msgid "**Path Parameters**:" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:235 +msgid "``thread_name``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:235 +msgid "Name of the periodic thread" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:244 +msgid "**Error Response** (not found, HTTP 404):" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:254 +msgid "``GET /periodic-threads-health`` — Periodic Thread Health Check" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:256 +msgid "" +"Quick health check for periodic threads. Returns healthy status if all " +"``critical`` and ``high`` level threads are active." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:260 +msgid "**Path**: ``/periodic-threads-health``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:268 +msgid "**Example Response** (healthy):" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:278 +msgid "**Example Response** (unhealthy):" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:297 +msgid "``POST /run_script`` — Run Script" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:299 +msgid "" +"Upload and execute a Python script in a restricted sandbox environment. " +"The script has access to ``app`` (the FastAPI application instance) and a" +" limited set of builtins. Import is restricted to modules configured in " +"``script_allowed_imports``." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:305 +msgid "**Path**: ``/run_script``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:306 +msgid "**Content-Type**: ``multipart/form-data``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:312 +msgid "``script``" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:312 +msgid "file" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:312 +msgid "Python script file to execute" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:315 +msgid "" +"**Response**: ``text/plain`` — The ``result`` variable from the script, " +"or ``\"Script executed successfully\"`` if no ``result`` is set." +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:323 +msgid "**Example Script** (``scratch.py``):" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:340 +msgid "**Error Response** (no script, HTTP 400):" +msgstr "" + +#: ../../source/internal_api_server/common_apis.rst:346 +msgid "**Error Response** (execution error, HTTP 500):" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/controller_apis.po b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/controller_apis.po new file mode 100644 index 00000000000..21803212726 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/controller_apis.po @@ -0,0 +1,309 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/internal_api_server/controller_apis.rst:4 +msgid "Controller APIs" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:6 +msgid "" +"These APIs are specific to the LMCache Controller component. They provide" +" visibility into registered instances, workers, and key statistics." +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:12 +msgid "Endpoints" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:15 +msgid "``GET /controller/key-stats`` — Key Statistics" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:17 +msgid "Get key statistics across all instances and workers." +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:19 +#: ../../source/internal_api_server/controller_apis.rst:90 +msgid "**Method**: ``GET``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:20 +msgid "**Path**: ``/controller/key-stats``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:21 +msgid "**Parameters**: None" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:22 +#: ../../source/internal_api_server/controller_apis.rst:101 +msgid "**Response**: ``application/json``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:28 +msgid "**Example Response** (HTTP 200):" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:50 +#: ../../source/internal_api_server/controller_apis.rst:175 +msgid "**Error Response** (controller not available, HTTP 503):" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:58 +msgid "**Response Schema**:" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:61 +#: ../../source/internal_api_server/controller_apis.rst:72 +#: ../../source/internal_api_server/controller_apis.rst:186 +msgid "Field" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:61 +#: ../../source/internal_api_server/controller_apis.rst:72 +#: ../../source/internal_api_server/controller_apis.rst:95 +#: ../../source/internal_api_server/controller_apis.rst:186 +msgid "Type" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:61 +#: ../../source/internal_api_server/controller_apis.rst:72 +#: ../../source/internal_api_server/controller_apis.rst:95 +#: ../../source/internal_api_server/controller_apis.rst:186 +msgid "Description" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:63 +msgid "``total_key_count``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:63 +#: ../../source/internal_api_server/controller_apis.rst:64 +#: ../../source/internal_api_server/controller_apis.rst:65 +#: ../../source/internal_api_server/controller_apis.rst:75 +#: ../../source/internal_api_server/controller_apis.rst:76 +#: ../../source/internal_api_server/controller_apis.rst:98 +#: ../../source/internal_api_server/controller_apis.rst:189 +#: ../../source/internal_api_server/controller_apis.rst:191 +#: ../../source/internal_api_server/controller_apis.rst:195 +msgid "int" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:63 +msgid "Total number of KV keys across all instances" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:64 +msgid "``total_instance_count``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:64 +msgid "Total number of registered instances" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:65 +msgid "``total_worker_count``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:65 +msgid "Total number of workers across all instances" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:66 +msgid "``instances``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:66 +msgid "list" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:66 +msgid "Per-instance breakdown (see below)" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:69 +msgid "**Instance Schema**:" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:74 +#: ../../source/internal_api_server/controller_apis.rst:97 +#: ../../source/internal_api_server/controller_apis.rst:188 +msgid "``instance_id``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:74 +#: ../../source/internal_api_server/controller_apis.rst:97 +#: ../../source/internal_api_server/controller_apis.rst:188 +#: ../../source/internal_api_server/controller_apis.rst:190 +#: ../../source/internal_api_server/controller_apis.rst:192 +msgid "str" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:74 +msgid "Unique identifier of the instance" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:75 +#: ../../source/internal_api_server/controller_apis.rst:195 +msgid "``key_count``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:75 +msgid "Number of KV keys held by this instance" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:76 +msgid "``worker_count``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:76 +msgid "Number of workers in this instance" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:81 +msgid "``GET /controller/workers`` — Worker Information" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:83 +msgid "" +"Get worker information with flexible query parameters. Behavior depends " +"on the combination of parameters:" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:86 +msgid "**No parameters**: List all registered workers across all instances." +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:87 +msgid "**``instance_id`` only**: List all workers for a specific instance." +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:88 +msgid "" +"**``instance_id`` and ``worker_id``**: Get detailed info about a specific" +" worker." +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:91 +msgid "**Path**: ``/controller/workers``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:92 +msgid "**Parameters**:" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:95 +msgid "Name" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:97 +msgid "(Optional) Instance ID to filter workers" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:98 +#: ../../source/internal_api_server/controller_apis.rst:189 +msgid "``worker_id``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:98 +msgid "" +"(Optional) Worker ID for specific worker details (requires " +"``instance_id``)" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:114 +msgid "**Example Response** (list workers):" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:144 +msgid "**Example Response** (single worker):" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:159 +msgid "**Error Response** (worker not found, HTTP 404):" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:167 +msgid "**Error Response** (instance not found, HTTP 404):" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:183 +msgid "**Worker Response Schema**:" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:188 +msgid "Instance this worker belongs to" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:189 +msgid "Worker index within the instance" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:190 +msgid "``ip``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:190 +msgid "Worker IP address" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:191 +msgid "``port``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:191 +msgid "Worker port number" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:192 +msgid "``peer_init_url``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:192 +msgid "(Optional) Peer initialization URL" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:193 +msgid "``registration_time``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:193 +#: ../../source/internal_api_server/controller_apis.rst:194 +msgid "float" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:193 +msgid "Unix timestamp of worker registration" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:194 +msgid "``last_heartbeat_time``" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:194 +msgid "Unix timestamp of last heartbeat" +msgstr "" + +#: ../../source/internal_api_server/controller_apis.rst:195 +msgid "Number of KV keys held by this worker" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/dynamic_backend_management.po b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/dynamic_backend_management.po new file mode 100644 index 00000000000..e91faa85192 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/dynamic_backend_management.po @@ -0,0 +1,159 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:4 +msgid "Dynamic Backend Management" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:6 +msgid "" +"LMCache provides a set of internal API endpoints that allow you to " +"**list**, **close**, and **create** storage backends at runtime without " +"restarting the serving engine. This is useful when you need to switch " +"between different storage configurations on the fly — for example, " +"migrating from a ``LocalDiskBackend`` to a ``GdsBackend``, or changing " +"the remote connector from a filesystem connector to Redis." +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:14 +msgid "Overview" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:16 +msgid "The workflow for dynamically switching a storage backend is:" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:18 +msgid "**Close** the backend you want to replace." +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:19 +msgid "**Update** the relevant configuration via the ``POST /conf`` API." +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:20 +msgid "" +"**Create** new backends — only backends that are not already present will" +" be created." +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:23 +msgid "" +"Any backend that was **not** closed will be skipped during creation, so " +"the operation is safe and idempotent." +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:27 +msgid "API Endpoints" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:30 +msgid "``GET /backends``" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:32 +msgid "List all active storage backends." +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:38 +#: ../../source/internal_api_server/dynamic_backend_management.rst:58 +#: ../../source/internal_api_server/dynamic_backend_management.rst:80 +msgid "Response:" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:48 +#, python-brace-format +msgid "``DELETE /backends/{backend_name}``" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:50 +msgid "" +"Close and remove a specific storage backend. After this call the backend" +" is fully shut down and removed from the internal dictionary, ensuring no" +" stale references remain." +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:71 +msgid "``POST /backends``" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:73 +msgid "" +"Create new storage backends based on the current ``LMCacheEngineConfig``." +" Existing backends are skipped." +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:96 +msgid "Use-Case Examples" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:99 +msgid "Switching from ``LocalDiskBackend`` to ``GdsBackend``" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:101 +msgid "" +"If you originally configured a local-disk backend and want to migrate to " +"NVIDIA GPUDirect Storage (GDS) at runtime:" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:124 +msgid "Switching ``RemoteBackend`` connector (FS → Redis)" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:126 +msgid "To change the remote connector type without restarting:" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:147 +msgid "Replacing only one backend while keeping others" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:149 +msgid "" +"If only the ``RemoteBackend`` needs to be updated but the " +"``LocalCPUBackend`` should stay untouched:" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:166 +msgid "Notes" +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:168 +msgid "" +"Closing the ``LocalCPUBackend`` is possible but should be done with " +"caution since many other backends rely on it as an intermediate buffer." +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:170 +msgid "" +"The ``POST /backends`` endpoint calls the same ``CreateStorageBackends`` " +"factory that is used during engine initialization, so all backend types " +"(local CPU, local disk, GDS, remote, P2P, plugins, etc.) are supported." +msgstr "" + +#: ../../source/internal_api_server/dynamic_backend_management.rst:174 +msgid "" +"After creating backends, the ``StorageManager`` automatically refreshes " +"its internal references (``non_allocator_backends``, " +"``local_cpu_backend``, etc.)." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/internal_api_server.po b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/internal_api_server.po new file mode 100644 index 00000000000..58f651f24b0 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/internal_api_server.po @@ -0,0 +1,117 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/internal_api_server/internal_api_server.rst:4 +msgid "Internal API Server" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:6 +msgid "" +"The ``internal_api_server`` provides HTTP APIs for managing and " +"inspecting the LMCache engine at runtime. APIs are organized into three " +"categories:" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:9 +msgid "" +"**Common APIs** — Available across all components (scheduler, worker, " +"controller)." +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:10 +msgid "**vLLM / Inference APIs** — Specific to vLLM inference workers." +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:11 +msgid "**Controller APIs** — Specific to the LMCache Controller." +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:22 +msgid "Configuration" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:24 +msgid "The following parameters can be configured in the YAML file:" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:46 +msgid "Port Assignment" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:48 +msgid "The port for each component is computed as:" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:54 +msgid "Where ``port_offset`` is:" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:56 +msgid "``0`` for the Scheduler" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:57 +msgid "" +"``1 + worker_id`` for Workers (e.g., Worker 0 → offset 1, Worker 1 → " +"offset 2)" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:61 +msgid "API Category & Route Discovery" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:63 +#, python-brace-format +msgid "" +"The server uses ``APIRegistry`` to automatically discover and register " +"API endpoint modules. Any file named ``*_api.py`` under " +"``lmcache/v1/internal_api_server/{common,vllm,controller}/`` that exports" +" a ``router = APIRouter()`` will be automatically included." +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:70 +msgid "Extending the Server" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:72 +msgid "To add a new API endpoint:" +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:74 +msgid "" +"Create a new file in the appropriate category directory (``common/``, " +"``vllm/``, or ``controller/``)." +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:76 +msgid "Name the file with ``_api.py`` suffix (e.g., ``my_feature_api.py``)." +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:77 +msgid "Define ``router = APIRouter()`` and add your endpoints." +msgstr "" + +#: ../../source/internal_api_server/internal_api_server.rst:79 +msgid "" +"The endpoint will be automatically discovered and registered on the next " +"server startup." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/vllm_apis.po b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/vllm_apis.po new file mode 100644 index 00000000000..43ac811b251 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/vllm_apis.po @@ -0,0 +1,1020 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/internal_api_server/vllm_apis.rst:4 +msgid "vLLM / Inference APIs" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:6 +msgid "" +"These APIs are specific to vLLM inference workers and provide cache " +"management, configuration, freeze control, chunk statistics, and version " +"information." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:12 +msgid "Endpoints" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:15 +msgid "Version & Info" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:18 +msgid "``GET /lmc_version`` — LMCache Version" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:20 +msgid "Get the LMCache library version string." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:22 +#: ../../source/internal_api_server/vllm_apis.rst:37 +#: ../../source/internal_api_server/vllm_apis.rst:52 +#: ../../source/internal_api_server/vllm_apis.rst:67 +#: ../../source/internal_api_server/vllm_apis.rst:98 +#: ../../source/internal_api_server/vllm_apis.rst:124 +#: ../../source/internal_api_server/vllm_apis.rst:205 +#: ../../source/internal_api_server/vllm_apis.rst:342 +#: ../../source/internal_api_server/vllm_apis.rst:438 +#: ../../source/internal_api_server/vllm_apis.rst:578 +#: ../../source/internal_api_server/vllm_apis.rst:661 +#: ../../source/internal_api_server/vllm_apis.rst:715 +#: ../../source/internal_api_server/vllm_apis.rst:963 +#: ../../source/internal_api_server/vllm_apis.rst:1003 +msgid "**Method**: ``GET``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:23 +msgid "**Path**: ``/lmc_version``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:24 +#: ../../source/internal_api_server/vllm_apis.rst:39 +#: ../../source/internal_api_server/vllm_apis.rst:54 +#: ../../source/internal_api_server/vllm_apis.rst:100 +#: ../../source/internal_api_server/vllm_apis.rst:440 +#: ../../source/internal_api_server/vllm_apis.rst:530 +#: ../../source/internal_api_server/vllm_apis.rst:555 +#: ../../source/internal_api_server/vllm_apis.rst:580 +#: ../../source/internal_api_server/vllm_apis.rst:612 +#: ../../source/internal_api_server/vllm_apis.rst:638 +#: ../../source/internal_api_server/vllm_apis.rst:663 +#: ../../source/internal_api_server/vllm_apis.rst:717 +#: ../../source/internal_api_server/vllm_apis.rst:752 +#: ../../source/internal_api_server/vllm_apis.rst:825 +#: ../../source/internal_api_server/vllm_apis.rst:884 +#: ../../source/internal_api_server/vllm_apis.rst:917 +#: ../../source/internal_api_server/vllm_apis.rst:941 +#: ../../source/internal_api_server/vllm_apis.rst:965 +#: ../../source/internal_api_server/vllm_apis.rst:1005 +msgid "**Parameters**: None" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:25 +msgid "**Response**: Plain version string." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:33 +msgid "``GET /commit_id`` — LMCache Commit ID" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:35 +msgid "Get the LMCache git commit ID." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:38 +msgid "**Path**: ``/commit_id``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:40 +msgid "**Response**: Plain commit ID string." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:48 +msgid "``GET /version`` — Full Version Info" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:50 +msgid "Get full version info (version + commit ID)." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:53 +msgid "**Path**: ``/version``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:55 +msgid "**Response**: Combined version string." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:63 +msgid "``GET /inference_info`` — Inference Information" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:65 +msgid "Get inference information including vLLM config and LMCache details." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:68 +msgid "**Path**: ``/inference_info``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:69 +#: ../../source/internal_api_server/vllm_apis.rst:126 +#: ../../source/internal_api_server/vllm_apis.rst:207 +#: ../../source/internal_api_server/vllm_apis.rst:236 +#: ../../source/internal_api_server/vllm_apis.rst:271 +#: ../../source/internal_api_server/vllm_apis.rst:311 +#: ../../source/internal_api_server/vllm_apis.rst:344 +#: ../../source/internal_api_server/vllm_apis.rst:401 +#: ../../source/internal_api_server/vllm_apis.rst:776 +#: ../../source/internal_api_server/vllm_apis.rst:1031 +#: ../../source/internal_api_server/vllm_apis.rst:1074 +msgid "**Parameters**:" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:72 +#: ../../source/internal_api_server/vllm_apis.rst:129 +#: ../../source/internal_api_server/vllm_apis.rst:210 +#: ../../source/internal_api_server/vllm_apis.rst:239 +#: ../../source/internal_api_server/vllm_apis.rst:274 +#: ../../source/internal_api_server/vllm_apis.rst:314 +#: ../../source/internal_api_server/vllm_apis.rst:347 +#: ../../source/internal_api_server/vllm_apis.rst:404 +#: ../../source/internal_api_server/vllm_apis.rst:779 +#: ../../source/internal_api_server/vllm_apis.rst:1034 +#: ../../source/internal_api_server/vllm_apis.rst:1077 +msgid "Name" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:72 +#: ../../source/internal_api_server/vllm_apis.rst:129 +#: ../../source/internal_api_server/vllm_apis.rst:210 +#: ../../source/internal_api_server/vllm_apis.rst:239 +#: ../../source/internal_api_server/vllm_apis.rst:274 +#: ../../source/internal_api_server/vllm_apis.rst:314 +#: ../../source/internal_api_server/vllm_apis.rst:347 +#: ../../source/internal_api_server/vllm_apis.rst:404 +#: ../../source/internal_api_server/vllm_apis.rst:475 +#: ../../source/internal_api_server/vllm_apis.rst:779 +#: ../../source/internal_api_server/vllm_apis.rst:1034 +#: ../../source/internal_api_server/vllm_apis.rst:1077 +msgid "Type" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:72 +#: ../../source/internal_api_server/vllm_apis.rst:129 +#: ../../source/internal_api_server/vllm_apis.rst:210 +#: ../../source/internal_api_server/vllm_apis.rst:239 +#: ../../source/internal_api_server/vllm_apis.rst:274 +#: ../../source/internal_api_server/vllm_apis.rst:314 +#: ../../source/internal_api_server/vllm_apis.rst:347 +#: ../../source/internal_api_server/vllm_apis.rst:404 +#: ../../source/internal_api_server/vllm_apis.rst:475 +#: ../../source/internal_api_server/vllm_apis.rst:779 +#: ../../source/internal_api_server/vllm_apis.rst:1034 +#: ../../source/internal_api_server/vllm_apis.rst:1077 +msgid "Description" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:74 +msgid "``format``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:74 +#: ../../source/internal_api_server/vllm_apis.rst:131 +#: ../../source/internal_api_server/vllm_apis.rst:212 +#: ../../source/internal_api_server/vllm_apis.rst:276 +#: ../../source/internal_api_server/vllm_apis.rst:316 +#: ../../source/internal_api_server/vllm_apis.rst:349 +#: ../../source/internal_api_server/vllm_apis.rst:406 +#: ../../source/internal_api_server/vllm_apis.rst:477 +#: ../../source/internal_api_server/vllm_apis.rst:1036 +#: ../../source/internal_api_server/vllm_apis.rst:1079 +msgid "str" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:74 +msgid "(Optional) Reserved for future use" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:77 +#: ../../source/internal_api_server/vllm_apis.rst:101 +#: ../../source/internal_api_server/vllm_apis.rst:165 +#: ../../source/internal_api_server/vllm_apis.rst:244 +#: ../../source/internal_api_server/vllm_apis.rst:279 +#: ../../source/internal_api_server/vllm_apis.rst:319 +#: ../../source/internal_api_server/vllm_apis.rst:354 +#: ../../source/internal_api_server/vllm_apis.rst:409 +#: ../../source/internal_api_server/vllm_apis.rst:441 +#: ../../source/internal_api_server/vllm_apis.rst:482 +#: ../../source/internal_api_server/vllm_apis.rst:531 +#: ../../source/internal_api_server/vllm_apis.rst:556 +#: ../../source/internal_api_server/vllm_apis.rst:581 +#: ../../source/internal_api_server/vllm_apis.rst:613 +#: ../../source/internal_api_server/vllm_apis.rst:639 +#: ../../source/internal_api_server/vllm_apis.rst:664 +#: ../../source/internal_api_server/vllm_apis.rst:718 +#: ../../source/internal_api_server/vllm_apis.rst:753 +#: ../../source/internal_api_server/vllm_apis.rst:784 +#: ../../source/internal_api_server/vllm_apis.rst:826 +#: ../../source/internal_api_server/vllm_apis.rst:885 +#: ../../source/internal_api_server/vllm_apis.rst:918 +#: ../../source/internal_api_server/vllm_apis.rst:942 +#: ../../source/internal_api_server/vllm_apis.rst:966 +#: ../../source/internal_api_server/vllm_apis.rst:1006 +#: ../../source/internal_api_server/vllm_apis.rst:1039 +#: ../../source/internal_api_server/vllm_apis.rst:1082 +msgid "**Response**: ``application/json``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:83 +msgid "**Error Response** (HTTP 500):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:94 +msgid "``GET /inference_version`` — vLLM Version" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:96 +msgid "Get the vLLM version information." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:99 +msgid "**Path**: ``/inference_version``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:107 +#: ../../source/internal_api_server/vllm_apis.rst:254 +#: ../../source/internal_api_server/vllm_apis.rst:285 +#: ../../source/internal_api_server/vllm_apis.rst:325 +#: ../../source/internal_api_server/vllm_apis.rst:422 +#: ../../source/internal_api_server/vllm_apis.rst:447 +#: ../../source/internal_api_server/vllm_apis.rst:537 +#: ../../source/internal_api_server/vllm_apis.rst:562 +#: ../../source/internal_api_server/vllm_apis.rst:587 +#: ../../source/internal_api_server/vllm_apis.rst:619 +#: ../../source/internal_api_server/vllm_apis.rst:645 +#: ../../source/internal_api_server/vllm_apis.rst:670 +#: ../../source/internal_api_server/vllm_apis.rst:759 +#: ../../source/internal_api_server/vllm_apis.rst:891 +#: ../../source/internal_api_server/vllm_apis.rst:924 +#: ../../source/internal_api_server/vllm_apis.rst:948 +#: ../../source/internal_api_server/vllm_apis.rst:972 +#: ../../source/internal_api_server/vllm_apis.rst:1012 +#: ../../source/internal_api_server/vllm_apis.rst:1045 +#: ../../source/internal_api_server/vllm_apis.rst:1088 +msgid "**Example Response**:" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:117 +msgid "Configuration & Metadata" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:120 +msgid "``GET /conf`` — Get Configuration" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:122 +msgid "Get current LMCache engine configuration values." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:125 +#: ../../source/internal_api_server/vllm_apis.rst:162 +msgid "**Path**: ``/conf``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:131 +#: ../../source/internal_api_server/vllm_apis.rst:212 +msgid "``names``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:131 +msgid "(Optional) Comma-separated list of config names to filter" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:134 +msgid "" +"**Response**: ``application/json`` — JSON object of configuration key-" +"value pairs." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:146 +msgid "``POST /conf`` — Update Configuration (Experimental)" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:148 +msgid "Update one or more configuration values at runtime." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:152 +msgid "" +"This feature is currently **experimental**. All configuration keys are " +"mutable at runtime by default unless explicitly marked as ``\"mutable\": " +"False`` in ``_CONFIG_DEFINITIONS``. The default will be changed to " +"**immutable** once the feature is stabilized." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:157 +msgid "" +"Updating a configuration only modifies the value in the " +"``LMCacheEngineConfig`` object. If a component has already cached the " +"value elsewhere, the change will **not** take effect for that component." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:161 +#: ../../source/internal_api_server/vllm_apis.rst:269 +#: ../../source/internal_api_server/vllm_apis.rst:309 +#: ../../source/internal_api_server/vllm_apis.rst:399 +#: ../../source/internal_api_server/vllm_apis.rst:469 +#: ../../source/internal_api_server/vllm_apis.rst:750 +#: ../../source/internal_api_server/vllm_apis.rst:774 +#: ../../source/internal_api_server/vllm_apis.rst:823 +#: ../../source/internal_api_server/vllm_apis.rst:882 +#: ../../source/internal_api_server/vllm_apis.rst:915 +#: ../../source/internal_api_server/vllm_apis.rst:939 +msgid "**Method**: ``POST``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:163 +#: ../../source/internal_api_server/vllm_apis.rst:471 +msgid "**Content-Type**: ``application/json``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:164 +msgid "**Request Body**: JSON object with config name-value pairs." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:173 +#: ../../source/internal_api_server/vllm_apis.rst:491 +msgid "**Example Response** (HTTP 200):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:184 +msgid "**Example Response** (partial failure, HTTP 400):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:193 +msgid "**Error Cases**:" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:195 +msgid "Unknown config key → ``\"Unknown config\"``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:196 +msgid "Immutable config key → ``\"Config is not mutable at runtime\"``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:197 +msgid "Invalid JSON body → HTTP 400" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:201 +msgid "``GET /meta`` — Engine Metadata" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:203 +msgid "" +"Get metadata of the LMCache engine (e.g., worker_id, model_name, " +"kv_shape)." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:206 +msgid "**Path**: ``/meta``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:212 +msgid "(Optional) Comma-separated list of attribute names to filter" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:215 +msgid "**Response**: ``application/json`` — JSON object of metadata attributes." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:227 +msgid "Cache Operations" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:230 +msgid "``DELETE /cache/clear`` — Clear Cache" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:232 +msgid "Clear cached KV data from the LMCache engine." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:234 +msgid "**Method**: ``DELETE``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:235 +msgid "**Path**: ``/cache/clear``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:241 +msgid "``locations``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:241 +msgid "list[str]" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:241 +msgid "" +"(Optional) Storage backends to clear (e.g. ``LocalCPUBackend``, " +"``LocalDiskBackend``). If not specified, clears all." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:265 +msgid "``POST /cache/store`` — Store KV Cache" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:267 +msgid "Store KV cache data into the LMCache engine using mock tokens." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:270 +msgid "**Path**: ``/cache/store``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:276 +#: ../../source/internal_api_server/vllm_apis.rst:316 +msgid "``tokens_mock``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:276 +msgid "" +"Two comma-separated numbers: ``\"start,end\"`` (e.g. ``\"0,100\"`` " +"generates tokens [0..99])" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:294 +msgid "**Error Response** (missing params, HTTP 400):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:305 +msgid "``POST /cache/retrieve`` — Retrieve KV Cache" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:307 +msgid "Retrieve KV cache data from the LMCache engine using mock tokens." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:310 +msgid "**Path**: ``/cache/retrieve``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:316 +msgid "Two comma-separated numbers: ``\"start,end\"`` (e.g. ``\"0,100\"``)" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:337 +msgid "``GET /cache/kvcache/check`` — KVCache Checksum" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:339 +msgid "" +"Compute MD5 checksums for kvcaches at specified slot_mapping positions. " +"Used for verifying that stored and retrieved kvcaches are identical." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:343 +msgid "**Path**: ``/cache/kvcache/check``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:349 +msgid "``slot_mapping``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:349 +msgid "" +"Slot indices, comma-separated. Supports ranges: ``\"0,1,2,3\"`` or " +"``\"1,2,3,[9,12],17,19\"``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:350 +msgid "``chunk_size``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:350 +#: ../../source/internal_api_server/vllm_apis.rst:478 +#: ../../source/internal_api_server/vllm_apis.rst:479 +msgid "int" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:350 +msgid "Chunk size for computing per-chunk checksums (required)" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:351 +msgid "``layerwise``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:351 +#: ../../source/internal_api_server/vllm_apis.rst:781 +msgid "bool" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:351 +msgid "If ``true``, output per-layer checksums per chunk (default: ``false``)" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:364 +msgid "**Example Response** (``layerwise=false``):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:377 +msgid "**Example Response** (``layerwise=true``):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:395 +msgid "``POST /cache/kvcache/record_slot`` — Toggle Slot Logging" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:397 +msgid "" +"Enable or disable KVCache slot_mapping logging during store/retrieve " +"operations." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:400 +msgid "**Path**: ``/cache/kvcache/record_slot``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:406 +msgid "``enabled``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:406 +msgid "" +"``\"true\"`` to enable, ``\"false\"`` to disable. Omit to query current " +"status." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:433 +msgid "``GET /cache/kvcache/info`` — KVCache Information" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:435 +msgid "" +"Get information about the current kvcaches structure including layer " +"names, shapes, and device info." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:439 +msgid "**Path**: ``/cache/kvcache/info``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:465 +msgid "``POST /cache/load-fs-chunks`` — Load FS Chunks" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:467 +msgid "" +"Load chunk files from FSConnector storage into LocalCPUBackend's hot " +"cache." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:470 +msgid "**Path**: ``/cache/load-fs-chunks``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:472 +msgid "**Request Body**:" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:475 +msgid "Field" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:477 +msgid "``config_path``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:477 +msgid "Path to LMCache engine configuration YAML file (required)" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:478 +msgid "``max_chunks``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:478 +msgid "(Optional) Maximum number of chunks to load" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:479 +msgid "``max_failed_keys``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:479 +msgid "Maximum failed keys to report (default: 10)" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:483 +msgid "**Tags**: ``cache-management``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:503 +msgid "**Error Response** (invalid config, HTTP 400):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:515 +msgid "Freeze Mode" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:518 +msgid "``PUT /freeze/enable`` — Enable Freeze Mode" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:520 +msgid "Enable freeze mode for the LMCache engine. When enabled:" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:522 +msgid "All store operations will be skipped (no new data stored)" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:523 +msgid "Only ``local_cpu`` backend will be used for retrieval" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:524 +msgid "No admit/evict messages will be generated" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:526 +msgid "This protects the local_cpu hot cache from changes." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:528 +#: ../../source/internal_api_server/vllm_apis.rst:553 +#: ../../source/internal_api_server/vllm_apis.rst:610 +#: ../../source/internal_api_server/vllm_apis.rst:636 +#: ../../source/internal_api_server/vllm_apis.rst:1029 +#: ../../source/internal_api_server/vllm_apis.rst:1072 +msgid "**Method**: ``PUT``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:529 +msgid "**Path**: ``/freeze/enable``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:549 +msgid "``PUT /freeze/disable`` — Disable Freeze Mode" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:551 +msgid "Disable freeze mode. Store operations will proceed normally." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:554 +msgid "**Path**: ``/freeze/disable``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:574 +msgid "``GET /freeze/status`` — Freeze Status" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:576 +msgid "Get the current freeze mode status." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:579 +msgid "**Path**: ``/freeze/status``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:599 +msgid "Hot Cache" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:601 +msgid "" +"These endpoints control the hot cache feature of LocalCPUBackend. When " +"hot cache is enabled, frequently accessed KV cache data will be kept in " +"CPU memory for faster retrieval." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:606 +msgid "``PUT /hot_cache/enable`` — Enable Hot Cache" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:608 +msgid "Enable hot cache for the LocalCPUBackend." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:611 +msgid "**Path**: ``/hot_cache/enable``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:631 +msgid "``PUT /hot_cache/disable`` — Disable Hot Cache" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:633 +msgid "" +"Disable hot cache for the LocalCPUBackend. Existing hot cache entries " +"will be cleared and no new data will be written." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:637 +msgid "**Path**: ``/hot_cache/disable``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:657 +msgid "``GET /hot_cache/status`` — Hot Cache Status" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:659 +msgid "Get the current hot cache status of LocalCPUBackend." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:662 +msgid "**Path**: ``/hot_cache/status``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:682 +msgid "Chunk Statistics" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:684 +msgid "" +"These endpoints manage chunk-level statistics collection via " +"``ChunkStatisticsLookupClient``. They are only available when the lookup " +"client supports statistics." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:690 +msgid "Lookup Client/Server Management" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:692 +msgid "" +"These endpoints allow runtime management of the lookup client and server." +" They are useful for dynamically reconfiguring the lookup mechanism " +"without restarting the service." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:698 +msgid "**Configuration Update Required First**" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:700 +msgid "" +"Before calling ``/lookup/create`` or ``/lookup/recreate``, you **MUST** " +"update the configuration via the ``/conf`` API first. The new lookup " +"client/server will be created using ``LookupClientFactory``." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:704 +msgid "" +"For some configurations (e.g., switching " +"``enable_scheduler_bypass_lookup``), you only need to update the " +"**scheduler's** configuration and recreate its lookup client. The workers" +" don't need changes in this case." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:710 +msgid "``GET /lookup/info`` — Lookup Client/Server Information" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:712 +msgid "" +"Get information about the current lookup client and server status. Shows " +"wrapper chain if applicable (e.g., " +"``HitLimitLookupClient(LMCacheLookupClient)``)." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:716 +msgid "**Path**: ``/lookup/info``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:724 +#: ../../source/internal_api_server/vllm_apis.rst:848 +msgid "**Example Response** (scheduler):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:734 +#: ../../source/internal_api_server/vllm_apis.rst:858 +msgid "**Example Response** (worker):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:746 +msgid "``POST /lookup/close`` — Close Lookup Client/Server" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:748 +msgid "Close the current lookup client (scheduler) or server (worker)." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:751 +msgid "**Path**: ``/lookup/close``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:770 +msgid "``POST /lookup/create`` — Create Lookup Client/Server" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:772 +msgid "" +"Create a new lookup client (scheduler) or server (worker) using current " +"config." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:775 +msgid "**Path**: ``/lookup/create``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:781 +msgid "``dryrun``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:781 +msgid "If ``true``, only show what would be created" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:794 +msgid "**Example Response** (dryrun):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:804 +msgid "**Example Response** (actual create):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:815 +msgid "``POST /lookup/recreate`` — Recreate Lookup Client/Server" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:817 +msgid "" +"Recreate the lookup client or server (equivalent to close + create). The " +"endpoint automatically determines which component based on role:" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:820 +msgid "**scheduler** role: recreates lookup client" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:821 +msgid "**worker** role: recreates lookup server" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:824 +msgid "**Path**: ``/lookup/recreate``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:828 +msgid "**Usage Flow**:" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:870 +msgid "**Client-only Changes**" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:872 +msgid "" +"For some configuration changes (e.g., switching " +"``enable_scheduler_bypass_lookup``), you only need to update the " +"scheduler's configuration and recreate its lookup client. Worker-side " +"lookup servers don't need to be recreated in this case." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:878 +msgid "``POST /chunk_statistics/start`` — Start Statistics" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:880 +msgid "Start collecting chunk statistics." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:883 +msgid "**Path**: ``/chunk_statistics/start``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:900 +msgid "**Error Response** (not supported, HTTP 400):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:911 +msgid "``POST /chunk_statistics/stop`` — Stop Statistics" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:913 +msgid "Stop collecting chunk statistics." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:916 +msgid "**Path**: ``/chunk_statistics/stop``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:935 +msgid "``POST /chunk_statistics/reset`` — Reset Statistics" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:937 +msgid "Reset all collected chunk statistics." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:940 +msgid "**Path**: ``/chunk_statistics/reset``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:959 +msgid "``GET /chunk_statistics/status`` — Statistics Status" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:961 +msgid "Get current chunk statistics and auto-exit configuration." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:964 +msgid "**Path**: ``/chunk_statistics/status``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:990 +msgid "Bypass Mode" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:992 +msgid "" +"Bypass mode allows dynamically skipping specific storage backends at " +"runtime. Bypassed backends are excluded from ``contains``/``put``/``get``" +" operations. This is useful for fault injection testing, isolating a " +"problematic backend, or debugging without restarting the engine." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:999 +msgid "``GET /bypass/list`` — List Bypassed Backends" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1001 +msgid "List all currently bypassed backends and all available backend names." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1004 +msgid "**Path**: ``/bypass/list``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1024 +msgid "``PUT /bypass/add`` — Add a Backend to Bypass List" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1026 +msgid "" +"Add a backend to the bypass list. The bypassed backend will be excluded " +"from ``contains``/``put``/``get`` operations." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1030 +msgid "**Path**: ``/bypass/add``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1036 +#: ../../source/internal_api_server/vllm_apis.rst:1079 +msgid "``backend_name``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1036 +msgid "Name of the backend to bypass (required)" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1057 +#: ../../source/internal_api_server/vllm_apis.rst:1100 +msgid "**Error Response** (unknown backend, HTTP 400):" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1068 +msgid "``PUT /bypass/remove`` — Remove a Backend from Bypass List" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1070 +msgid "Remove a backend from the bypass list, restoring it to normal operation." +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1073 +msgid "**Path**: ``/bypass/remove``" +msgstr "" + +#: ../../source/internal_api_server/vllm_apis.rst:1079 +msgid "Name of the backend to restore (required)" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/async_loading.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/async_loading.po new file mode 100644 index 00000000000..939ccc345ec --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/async_loading.po @@ -0,0 +1,214 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/async_loading.rst:2 +msgid "Async Loading" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:6 +msgid "Table of Contents" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:9 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:11 +msgid "" +"This document explains the principle, benefits, differences from vLLM PR " +"`19330 `_, and " +"limitations of the LMCache ``async_loading`` feature. It focuses on " +"LMCache v1 integration with vLLM and the internal storage pipeline." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:14 +msgid "Key change of components in this feature include:" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:16 +msgid "LMCache async lookup client/server (ZMQ-based)" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:17 +msgid "Storage manager orchestrating backends and concurrency" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:18 +msgid "Cache engine async API entrypoints" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:19 +msgid "vLLM adapter integration points" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:22 +msgid "Principle and Theory" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:24 +msgid "" +"At a high level, ``async_loading`` decouples scheduler-side lookup from " +"worker-side prefetch/retrieval, allowing overlap between I/O and " +"computation while preserving prefix-based correctness." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:26 +msgid "The scheduler sends lookup requests with token chunk hashes and offsets." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:27 +msgid "" +"Worker-side servers perform tiered ``batched_async_contains`` over " +"available backends and eagerly launch non-blocking batched get operations" +" for hit prefixes." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:28 +msgid "" +"Completion is tracked via an ``EventManager`` to safely deliver loaded " +"memory objects back to the requesting path." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:29 +msgid "" +"A weighted semaphore with an ``AsyncSerializer`` prevents allocator " +"deadlocks by shaping concurrency according to chunk budget." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:31 +msgid "The following Mermaid sequence diagram illustrates the end-to-end flow:" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:63 +msgid "Architecture (Worker Side)" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:96 +msgid "Benefits" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:98 +msgid "Performance overlap" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:99 +msgid "" +"**I/O–Compute Overlap**: Decoupling lookup/prefetch from loading enables " +"fetching KV chunks while vLLM continues scheduling/computation." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:100 +msgid "Robustness and error handling" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:101 +msgid "" +"**Event-driven Synchronization**: ``EventManager`` ensures safe hand-off " +"of futures and avoids race conditions between threads and the async loop." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:102 +msgid "" +"**Backpressure & Deadlock Avoidance**: ``AsyncSerializer`` with a " +"weighted semaphore caps concurrent chunk retrievals based on allocator " +"budget, preventing starvation or allocator lockups." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:103 +msgid "" +"**Graceful Miss Path**: Immediate response with ``None`` hit tokens when " +"nothing is retrievable; worker returns quickly without stalling the " +"scheduler." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:106 +msgid "Comparison with vLLM Load Failure Recovery feature" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:108 +msgid "" +"The `VLLM_PR_19330 `_ " +"introduces a fault recovery mechanism for vLLM's KV connector " +"infrastructure that enables graceful handling of KV cache load failures " +"by automatically detecting failed block loads and rescheduling only " +"affected requests for recomputation from a valid prefix. By contrast, " +"LMCache’s ``async_loading`` is an externalized caching layer with its own" +" client/server, storage backends, and concurrency control." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:112 +msgid "Limitations" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:114 +msgid "" +"Only works with vllm merged `VLLM_PR_23620 `_" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:115 +msgid "" +"Backend support constraint: This feature currently requires backends that" +" implement ``batched_async_contains``; limited to a few backends, e.g.:" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:116 +msgid "``LocalCpuBackend``" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:117 +msgid "``LocalDiskBackend``" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:118 +msgid "``S3Connector``" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:119 +msgid "``FSConnector``" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:120 +msgid "``RedisConnector/RedisClusterConnector``" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:123 +msgid "Future Work" +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:125 +msgid "" +"Introduce a default ``batched_async_contains`` implementation, so all " +"backends can support ``async_loading``." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:126 +msgid "" +"Add metrics and observability to track the number of asynchronous lookup " +"requests and the number of occupied ``MemoryObj`` instances." +msgstr "" + +#: ../../source/kv_cache/async_loading.rst:127 +msgid "" +"Improve the lookup framework by passing vLLM prefix cache hit tokens so " +"that async lookup can skip loading parts already hit in vLLM." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/caching_policies.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/caching_policies.po new file mode 100644 index 00000000000..b8247b30615 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/caching_policies.po @@ -0,0 +1,43 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/caching_policies.rst:2 +msgid "Using Different Caching Policies" +msgstr "" + +#: ../../source/kv_cache/caching_policies.rst:4 +msgid "LMCache supports multiple caching policies." +msgstr "" + +#: ../../source/kv_cache/caching_policies.rst:6 +msgid "" +"For example, to use LRU, you can set the environment variable " +"``LMCACHE_CACHE_POLICY=LRU`` or set it in the configuration file with " +"``cache_policy=\"LRU\"``." +msgstr "" + +#: ../../source/kv_cache/caching_policies.rst:8 +msgid "" +"Currently, LMCache supports \"LRU\" (Least Recently Used), \"MRU\" (Most " +"Recently Used), \"LFU\" (Least Frequently Used) and \"FIFO\" (First-In-" +"First-Out) caching policies." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/multiprocess_mode.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/multiprocess_mode.po new file mode 100644 index 00000000000..c2d8b5b118a --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/multiprocess_mode.po @@ -0,0 +1,38 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/multiprocess_mode.rst:4 +msgid "Multiprocess Mode (Redirect)" +msgstr "" + +#: ../../source/kv_cache/multiprocess_mode.rst:7 +msgid "" +"This page has moved. The multiprocess mode documentation is now at " +":doc:`/mp/index`." +msgstr "" + +#: ../../source/kv_cache/multiprocess_mode.rst:10 +msgid "" +"Please see the new :doc:`/mp/index` section for complete documentation " +"including quick start, configuration reference, L2 storage, deployment, " +"observability, and architecture guides." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po new file mode 100644 index 00000000000..c569d13ca0f --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po @@ -0,0 +1,193 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/p2p_sharing.rst:4 +msgid "P2P KV Cache Sharing" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:6 +msgid "" +"P2P (Peer-to-Peer) KV cache sharing enables direct cache transfer between" +" multiple serving engine instances without requiring a centralized cache " +"server. This approach provides high-performance cache sharing with " +"reduced latency and improved scalability, especially beneficial in " +"distributed inference scenarios." +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:8 +msgid "" +"LMCache supports P2P sharing through a controller-based architecture " +"using NIXL (NVIDIA Inference Xfer Library) for optimized data transfer " +"between instances." +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:11 +msgid "Prerequisites" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:13 +msgid "**Multi-GPU Setup**: Your server should have at least 2 GPUs" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:14 +msgid "**NIC**: RDMA is recommended for more performance." +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:15 +msgid "**NIXL**: Install from `NIXL `_" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:16 +msgid "" +"**vLLM**: v1 version is required, refer to :ref:`installation_guide` for " +"details." +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:17 +msgid "**LMCache**: Install from :ref:`installation_guide`" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:20 +msgid "Configuration" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:22 +msgid "Create two configuration files for the P2P sharing setup." +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:25 +msgid "**Instance 1 Configuration (example1.yaml)**:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:51 +msgid "**Instance 2 Configuration (example2.yaml)**:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:78 +msgid "Setup and Usage" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:80 +msgid "**Step 1: Start the LMCache Controller**" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:86 +msgid "" +"Make sure that the 8300 and 8400 ports are set up in " +"**controller_pull_url** and **controller_reply_url** in the configuration" +" files. Port 9000 is the controller main port, which is arbitrary and can" +" be changed." +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:89 +msgid "After starting the controller, access the WebUI at:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:91 +msgid "http://localhost:9000/" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:93 +msgid "**Step 2: Start vLLM Engines with LMCache Workers**" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:95 +msgid "If the NIC supports RDMA:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:101 +msgid "If the NIC does not support RDMA:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:107 +msgid "Start vLLM engine 1 at port 8010:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:117 +msgid "Start vLLM engine 2 at port 8011:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:127 +msgid "**Step 3: Test P2P Cache Sharing**" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:129 +msgid "Send a request to vLLM engine 1 to populate the cache:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:141 +msgid "" +"Send the same request to vLLM engine 2 to demonstrate cache retrieval " +"from **engine 1**:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:154 +msgid "Expected Output" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:156 +msgid "" +"When the second request successfully retrieves cache from the first " +"instance, you should see logs similar to:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:163 +msgid "" +"These logs indicate successful P2P connection establishment and high-" +"throughput cache retrieval." +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:167 +msgid "**Step 4: Benchmarking P2P Cache Sharing**" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:169 +msgid "Send a request workload to instance 1 to populate the cache:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:183 +msgid "" +"Send the same request workload to instance 2 to demonstrate cache " +"retrieval from **instance 1**:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:199 +msgid "Benchmark Results" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:201 +msgid "First instance metrics:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:216 +msgid "Second instance metrics:" +msgstr "" + +#: ../../source/kv_cache/p2p_sharing.rst:231 +#, python-format +msgid "" +"In this example, the warm-up round metric in long_doc_qa is used because " +"no existing KV cache is reused within an instance to benefit solely from " +"P2P sharing. With LMCache P2P sharing enabled, the time to first token " +"(TTFT) is reduced by 54.7%, from 2.286 s to 1.036 s, with a 63.6% " +"reduction in total inference time (37.957 s → 13.814 s)." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/3fs.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/3fs.po new file mode 100644 index 00000000000..07c303866bf --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/3fs.po @@ -0,0 +1,159 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/3fs.rst:2 +msgid "3FS" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:9 +msgid "" +"3FS (Fire-Flyer File System) is an AI native distributed file-system " +"which provides high performance and low latency. It is a supported option" +" for KV Cache offloading in LMCache. Even though the FSConnector backend " +"can work with 3FS storage cluster, but it access data in 3FS storage " +"cluster by FUSE interfaces. It can't leverage 3FS high performance. This " +"particular backend uses 3FS native USRBIO(User Space Ring Based IO) " +"interfaces to access 3FS storage cluster which get high performance." +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:16 +msgid "Configure LMCache 3FS Offloading" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:18 +msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:20 +msgid "Example ``config.yaml``:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:66 +msgid "There are 2 methods to config hf3fs remote backend:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:67 +msgid "Plugin name mode, uses the parameter remote_storage_plugins(Recommend)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:68 +msgid "" +"URL mode, uses the parameter remote_url(Deprecated, will be removed in a " +"future)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:70 +msgid "" +"For URL mode, the base_path is contained in the url. For plugin name " +"mode, there are 2 methods to set base_path:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:71 +#, python-brace-format +msgid "" +"remote_storage_plugin.{plugin name}.base_path, it sets the base_path for " +"a plugin instance." +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:72 +msgid "e.g.:remote_storage_plugin.hf3fs.primary.base_path" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:73 +msgid "hf3fs_base_path, it set the base_path for all plugin instances" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:76 +msgid "Installation" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:80 +msgid "**Prerequisites:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:82 +msgid "" +"A Machine with at least one GPU. You can adjust the max model length of " +"your vllm instance depending on your GPU memory." +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:84 +msgid "vllm and lmcache installed" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:86 +msgid "**Step 1. Install 3FS hf3fs_py_usrbio package**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:88 +msgid "" +"The inference server need install 3FS hf3fs_py_usrbio package, recommend " +"to build the package from source:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:99 +msgid "" +"`3FS Build `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:102 +msgid "**Step 2. Setup 3FS storage cluster**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:104 +msgid "" +"`3FS Setup `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:107 +msgid "**Step 3. Deploy 3FS FUSE client in inference server**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:109 +msgid "" +"The inference server must deploy 3FS FUSE client(a fuse daemon process " +"provided by 3FS), otherwise, it can't access 3FS storage cluster" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:112 +msgid "" +"`Setup 3FS FUSE Client `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:115 +msgid "**Step 4. Start a vLLM server with 3FS offloading enabled**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:117 +msgid "Create a lmcache configuration file called: ``3fs-offload.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/3fs.rst:155 +msgid "Start vllm:" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/cpu_ram.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/cpu_ram.po new file mode 100644 index 00000000000..4f7531246e8 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/cpu_ram.po @@ -0,0 +1,235 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:2 +msgid "CPU RAM" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:9 +msgid "" +"CPU RAM and Local Storage are the two ways of offloading KV cache onto " +"non-GPU memory of the same machine that is running inference." +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:13 +msgid "Two ways to configure LMCache CPU Offloading:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:15 +msgid "**1. Environment Variables:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:26 +msgid "**2. Configuration File**:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:28 +msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:30 +msgid "Example ``config.yaml``:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:42 +msgid "CPU RAM Explanation:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:44 +msgid "" +"The ``LMCACHE_MAX_LOCAL_CPU_SIZE`` is the amount of page-locked (for fast" +" GPU transfer) CPU memory that LMCache will reserve and must be set to a " +"number greater than 0 since local and remote backends also use CPU RAM as" +" an intermediate buffer when transferring KV caches with the GPU. This " +"means it is possible to set ``LMCACHE_LOCAL_CPU=False`` even though " +"``LMCACHE_MAX_LOCAL_CPU_SIZE`` is set to a non-zero number." +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:51 +msgid "" +"However, it is recommended to *always* set ``LMCACHE_LOCAL_CPU=True`` " +"(the default is ``True`` so if you don't specify, CPU offloading will " +"automatically be enabled) since this allows all currently unused pinned " +"CPU RAM that LMCache has reserved to hold KV caches. When the pinned CPU " +"RAM is required for any disk or remote transfers, the CPU KV caches will " +"be LRU evicted to make space so there is no danger of running out of " +"pinned CPU RAM." +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:56 +msgid "" +"When ``LMCACHE_LOCAL_CPU=True`` is used in conjunction with the disk " +"backend or a remote backend (:doc:`Redis <./redis>`, :doc:`Mooncake " +"<./mooncake>`, :doc:`Valkey <./valkey>`, or :doc:`Infinistore " +"<./infinistore>`), we can think of the CPU RAM as a \"hot cache\" that " +"will contain the \"hottest\" (most recently accessed)subset of KV caches " +"from Disk and Remote storage." +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:61 +msgid "" +"Thus, the cache engine also has a **prefetch** mechanism to preload the " +"KV caches for specified tokens into the pinned CPU RAM from the disk or " +"remote storage (*if* the KV caches for these tokens are already stored " +"there). This can preemptively avoid the latency of the disk and remote KV" +" transfer if we predict these tokens will be requested soon (e.g. " +"structured or agentic workflows)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:69 +msgid "Online Inference Example" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:71 +msgid "Let's feel the TTFT (time to first token) differential!" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:75 +msgid "**Prerequisites:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:77 +msgid "" +"A Machine with at least one GPU. Adjust the max model length of your vllm" +" instance depending on your GPU memory and the long context you want to " +"use." +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:79 +msgid "" +"vllm and lmcache installed (:doc:`Installation Guide " +"<../../getting_started/installation>`)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:81 +msgid "Hugging Face access to ``meta-llama/Meta-Llama-3.1-8B-Instruct``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:87 +msgid "A few packages:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:93 +msgid "**Step 0. Set up a directory for this example:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:100 +msgid "**Step 1. Prepare a long context!**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:102 +msgid "" +"We want a context long enough that vllm's prefix caching will not be able" +" to hold the KV caches in GPU memory and LMCache is necessary to keep KV " +"caches in non-GPU memory:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:110 +msgid "**Step 2. Start a vLLM server with CPU offloading enabled:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:112 +msgid "Create a an lmcache configuration file called: ``cpu-offload.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:120 +msgid "" +"If you don't want to use a config file, uncomment the first three " +"environment variables and then comment out the ``LMCACHE_CONFIG_FILE`` " +"below:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:135 +msgid "" +"``--kv-transfer-config``: This is the parameter that actually tells vLLM " +"to use LMCache for KV cache offloading." +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:136 +msgid "``kv_connector``: Specifies the LMCache connector for vLLM V1" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:137 +msgid "" +"``kv_role``: Set to \"kv_both\" for both storing and loading KV cache " +"(important because we will run two queries and the first will " +"produce/store a KV cache while the second will consume/load that KV " +"cache)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:139 +msgid "**Step 3. Query TTFT improvements with LMCache:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:141 +msgid "" +"Once the Open AI compatible server is running on default vllm port 8000, " +"let's query it twice with the same long context!" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:143 +msgid "Create a script called ``query-twice.py`` and paste the following code:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:209 +msgid "Then run:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:215 +msgid "" +"Since we're in streaming mode, you'll be able to feel the TTFT " +"differential in real time!" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:218 +msgid "**Example Output:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:243 +msgid "" +"If you look at the logs of your vLLM server, you should see (the logs are" +" truncated for cleanliness):" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:271 +msgid "Tips:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:273 +msgid "" +"If you want to run the ``query-twice.py`` script multiple times, you'll " +"need to either restart the vLLM LMCache server or change the prefix of " +"the context you pass in since you've already warmed LMCache." +msgstr "" + +#: ../../source/kv_cache/storage_backends/cpu_ram.rst:275 +msgid "" +"The max model length here was decided by running an L4 with only 23GB of " +"GPU memory. If you have more memory, you can increase the max model " +"length and modify ``query-twice.py`` to use more of the long context. " +"LMCache TTFT improvement becomes more pronounced as the context length " +"increases!" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/custom_backend.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/custom_backend.po new file mode 100644 index 00000000000..aa3363b9f9d --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/custom_backend.po @@ -0,0 +1,39 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/custom_backend.rst:2 +msgid "Custom Storage Backends" +msgstr "" + +#: ../../source/kv_cache/storage_backends/custom_backend.rst:4 +msgid "" +"LMCache supports integrating custom storage backends through dynamic " +"loading or plug and play capability. This allows extending cache storage " +"capabilities without modifying core code." +msgstr "" + +#: ../../source/kv_cache/storage_backends/custom_backend.rst:6 +msgid "" +"See :doc:`Storage Plugins " +"<../../developer_guide/extending_lmcache/storage_plugins>` for more " +"details." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/dax.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/dax.po new file mode 100644 index 00000000000..e875210caf1 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/dax.po @@ -0,0 +1,274 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/dax.rst:1 +msgid "# SPDX-License-Identifier: Apache-2.0" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:4 +msgid "Device-DAX (/dev/dax)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:9 +msgid "" +"The DAX storage plugin maps a ``/dev/dax`` device using " +"``mmap(MAP_SHARED)`` and uses the mapped region as a fixed-size arena for" +" KV cache chunks. Typical ``/dev/dax`` devices include persistent memory," +" CXL-attached memory, and other byte-addressable memory devices." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:14 +msgid "" +"Data stored on the DAX device may survive process restarts, but is not " +"guaranteed to be durable." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:17 +msgid "" +"KV cache data is stored in the DAX region as part of the backend's " +"storage flow. Reads copy data back into CPU-backed memory objects." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:22 +msgid "Configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:42 +msgid "Multiprocess Mode" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:44 +msgid "" +"In LMCache multiprocess mode, Device-DAX is configured as a built-in L2 " +"adapter named ``dax``. The MP adapter uses the normal L2 adapter " +"``submit -> event fd -> query`` contract; no vLLM connector protocol " +"changes are required." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:64 +msgid "The ``--l2-adapter`` JSON accepts these fields:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:66 +msgid "``device_path``: required path to a readable and writable DAX device." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:67 +msgid "" +"``max_dax_size_gb``: required mapped size in GiB. The value must fit " +"within the device capacity when capacity can be determined with " +"``fstat``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:69 +msgid "" +"``slot_bytes``: required fixed slot size in bytes. It must be large " +"enough for one full LMCache chunk." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:71 +msgid "``num_store_workers``: optional store worker count, default ``1``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:72 +msgid "``num_lookup_workers``: optional lookup worker count, default ``1``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:73 +msgid "" +"``num_load_workers``: optional load worker count, default ``min(4, " +"os.cpu_count())``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:75 +msgid "" +"``persist_enabled``: accepted by common MP L2 parsing but ignored by " +"``dax`` in this release." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:78 +msgid "" +"MP DAX stores opaque ``ObjectKey`` values in memory and is volatile-only " +"in this release. Closing and reopening the server on the same DAX path " +"starts with an empty index, so previously written bytes are not " +"discoverable after restart." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:83 +msgid "" +"MP DAX uses one mapped device path per LMCache server. It does not add " +"per-TP DAX partitions, multi-device striping, on-device metadata, or " +"restart recovery. Capacity accounting and eviction are slot-based: a " +"stored object occupies one slot even if its payload is smaller than " +"``slot_bytes``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:90 +msgid "Using The Batched Restore Path" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:92 +msgid "" +"The current DAX optimization is a staged batched restore path for " +"retrieval. It is enabled automatically whenever the DAX backend is " +"configured. No extra feature flag is required." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:96 +msgid "The retrieve flow is:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:98 +msgid "Reserve a batched set of readable DAX chunks." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:99 +msgid "Allocate CPU restore buffers from ``LocalCPUBackend``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:100 +msgid "" +"Copy DAX data into a backend-owned pinned staging slab in coalesced " +"regions." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:101 +msgid "Copy from the staging slab into the final CPU ``MemoryObj`` outputs." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:102 +msgid "Upload those CPU outputs through the normal GPU connector path." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:104 +msgid "" +"The store flow is unchanged: KV data is still staged through CPU memory " +"before being written into the DAX arena." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:107 +msgid "The new DAX tuning knobs control the batched restore path:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:109 +msgid "" +"``dax.restore_workers``: number of persistent worker threads used to " +"execute restore regions in parallel." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:111 +msgid "" +"``dax.restore_max_regions``: maximum number of restore regions in one " +"wave. Larger values increase parallelism but also increase slab space " +"requirements." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:113 +msgid "" +"``dax.retrieve_staging_slab_bytes``: total size in bytes of the reusable " +"pinned retrieve slab. This must be large enough to hold one full chunk " +"per configured restore region." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:117 +msgid "For a first pass, start with:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:119 +msgid "" +"``dax.restore_workers`` equal to the number of CPU workers you want " +"devoted to DAX restores" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:121 +msgid "``dax.restore_max_regions`` equal to ``dax.restore_workers``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:122 +msgid "" +"``dax.retrieve_staging_slab_bytes`` at least ``dax.restore_max_regions * " +"full_chunk_size``, then scale upward if larger batched restores are " +"common" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:126 +msgid "" +"If retrieve throughput is low, increase the slab size first, then " +"increase worker and region counts together. If CPU pressure is high, " +"reduce ``dax.restore_workers`` and ``dax.restore_max_regions``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:132 +msgid "Runtime Requirements" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:134 +msgid "" +"``extra_config['dax.device_path']`` is required and must point to a " +"readable and writable DAX device." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:136 +msgid "" +"The process must have read-write access to the DAX device (e.g., via " +"appropriate permissions or group membership)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:138 +msgid "" +"``LocalCPUBackend`` must be enabled because DAX reads return CPU-backed " +"memory objects." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:143 +msgid "Validation and Current Limits" +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:145 +msgid "" +"Tensor parallelism is currently limited to TP=1 (``metadata.world_size ==" +" 1``)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:147 +msgid "" +"Only single-tensor chunk layouts are supported. Multi-tensor put requests" +" are rejected." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:149 +msgid "" +"Batched restore uses a backend-owned retrieve staging slab and persistent" +" restore executors. The slab and region count can be tuned with " +"``dax.restore_workers``, ``dax.restore_max_regions``, and " +"``dax.retrieve_staging_slab_bytes``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/dax.rst:153 +msgid "" +"Blocking batched restore preserves positional output semantics, while " +"asynchronous batched restore returns only the consecutive hit prefix." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/eic.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/eic.po new file mode 100644 index 00000000000..0944d58f303 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/eic.po @@ -0,0 +1,72 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/eic.rst:2 +msgid "EIC" +msgstr "" + +#: ../../source/kv_cache/storage_backends/eic.rst:4 +msgid "" +"EIC(Elastic Instant Cache) is a distributed database designed for LLM KV " +"Cache. It supports RDMA, GDR and has the capabilities of distributed " +"disaster tolerance and expansion. You can understand the principles and " +"architecture of EIC through these articles:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/eic.rst:7 +msgid "https://mp.weixin.qq.com/s/tasDqXf0Gxr3o_WCJ2IJUQ" +msgstr "" + +#: ../../source/kv_cache/storage_backends/eic.rst:8 +msgid "https://mp.weixin.qq.com/s/b_4YhTa96Zeklh23lv8qBw" +msgstr "" + +#: ../../source/kv_cache/storage_backends/eic.rst:11 +msgid "Deploy EIC" +msgstr "" + +#: ../../source/kv_cache/storage_backends/eic.rst:13 +msgid "" +"You can visit the official link https://console.volcengine.com/eic and " +"deploy EIC KVCache on your compute cluster with web UI. In addition, we " +"provide particular image in volcano engine, which integrates various " +"optimizations based on the official image. You may use " +"tests/v1/storage_backend/test_eic.py to detect the connectivity of EIC." +msgstr "" + +#: ../../source/kv_cache/storage_backends/eic.rst:17 +msgid "Deploy Model With EIC" +msgstr "" + +#: ../../source/kv_cache/storage_backends/eic.rst:19 +msgid "You can enable EIC KVCache offload with the official interface, such as" +msgstr "" + +#: ../../source/kv_cache/storage_backends/eic.rst:30 +msgid "Example ``config.yaml``:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/eic.rst:40 +msgid "" +"For more details, you can see " +"https://www.volcengine.com/docs/85848/1749188." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/gds.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/gds.po new file mode 100644 index 00000000000..f0761822f0e --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/gds.po @@ -0,0 +1,324 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/gds.rst:2 +msgid "GDS Backend" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:9 +msgid "" +"This backend will work with any file system, whether local, remote, and " +"remote with GDS-based optimizations. Remote file systems allow for " +"multiple LMCache instances to share data seamlessly. The GDS (GPU-Direct " +"Storage) optimizations are used for zero-copy I/O from GPU memory to " +"storage systems. Supports both NVIDIA cuFile and AMD hipFile for GPU-" +"direct storage." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:17 +msgid "Ways to configure LMCache GDS Backend" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:19 +msgid "**1. Environment Variables:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:33 +msgid "**2. Configuration File**:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:35 +msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:37 +msgid "Example ``config.yaml``:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:52 +msgid "Multi-Path (Multi-Device) Support" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:54 +msgid "" +"When a system has multiple NVMe drives, you can distribute GDS I/O across" +" them by specifying a comma-separated list of paths in ``gds_path``. The " +"``gds_path_sharding`` option controls how each GPU worker selects its " +"path. Currently only ``\"by_gpu\"`` is supported (the default), which " +"selects a path based on the device index (``device_id % num_paths``), so " +"traffic is spread evenly across the drives without any manual pinning." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:61 +msgid "" +"**Why this helps:** a single PCIe Gen 4 x4 NVMe tops out at ~7 GB/s. With" +" four drives the aggregate bandwidth can reach ~28 GB/s, matching what " +"multi-GPU systems need for KV cache eviction and prefetch." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:65 +msgid "**Environment variables:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:72 +msgid "**YAML config:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:79 +msgid "With the above configuration on a 4-GPU node:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:81 +msgid "``cuda:0`` writes to ``/mnt/nvme0/cache``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:82 +msgid "``cuda:1`` writes to ``/mnt/nvme1/cache``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:83 +msgid "``cuda:2`` writes to ``/mnt/nvme2/cache``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:84 +msgid "``cuda:3`` writes to ``/mnt/nvme3/cache``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:86 +msgid "" +"If there are more GPUs than paths, the assignment wraps around (e.g. " +"``cuda:4`` maps back to ``/mnt/nvme0/cache``). A single path (no commas) " +"works exactly as before." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:90 +msgid "" +"All directories are created automatically at startup. Every path in the " +"list must reside on a filesystem that the rest of the GDS configuration " +"expects (e.g., all paths on GDS-capable mounts when using cuFile)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:94 +msgid "" +"**Read behavior:** on startup the backend scans **all** configured paths " +"for previously-stored KV cache entries, regardless of GPU affinity. This" +" means a ``cuda:0`` worker whose write affinity is ``/mnt/nvme0/cache`` " +"will still discover entries that were written to ``/mnt/nvme1/cache`` by " +"``cuda:1`` in a prior run. Writes, however, always go to the single " +"affinity-selected path." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:110 +msgid "GDS Buffer Size Explanation" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:112 +msgid "" +"The backend currently pre-registers buffer space to speed up GDS " +"operations. This buffer space is registered in VRAM so options like " +"``--gpu-memory-utilization`` from ``vllm`` should be considered when " +"setting it. For example, a good rule of thumb for H100 which generally " +"has 80GiBs of VRAM would be to start with 8GiB and set ``--gpu-memory-" +"utilization 0.85`` and depending on your workflow fine-tune it from " +"there." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:120 +msgid "Using AMD hipFile" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:124 +msgid "" +"hipFile is alpha software and has been tested on limited hardware. For " +"full installation details, see the `hipFile install guide " +"`__." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:128 +#: ../../source/kv_cache/storage_backends/gds.rst:179 +msgid "**Prerequisites:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:130 +msgid "" +"**ROCm >= 7.2** with ``amdgpu-dkms >= 30.20.1`` (see the `ROCm quick " +"start installation guide `__)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:132 +msgid "**Supported storage:** local NVMe drives only" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:133 +msgid "**Supported filesystems:** ext4 (mounted with ``data=ordered``) and xfs" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:134 +msgid "**Kernel:** ``CONFIG_PCI_P2PDMA`` must be enabled" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:136 +msgid "**Quick install (Ubuntu 24.04):**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:147 +msgid "" +"You can verify that the HIP libraries and kernel support AIS (AMD " +"Infinity Storage) by running:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:153 +msgid "" +"Successful output will show ``True`` for ``Kernel P2PDMA support``, ``HIP" +" runtime``, and ``amdgpu``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:155 +msgid "**LMCache configuration:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:157 +msgid "To use AMD hipFile instead of NVIDIA cuFile, set the GDS backend:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:159 +msgid "**Environment Variables:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:165 +msgid "**Configuration File:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:171 +msgid "" +"Note: The ``gds_buffer_size`` configuration is used for both cuFile and " +"hipFile buffers." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:175 +msgid "Setup Example" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:181 +msgid "" +"A Machine with at least one GPU. You can adjust the max model length of " +"your vllm instance depending on your GPU memory." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:183 +msgid "A mounted file system. A file system supportings GDS will work best." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:185 +msgid "" +"vllm and lmcache installed (:doc:`Installation Guide " +"<../../getting_started/installation>`)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:187 +msgid "Hugging Face access to ``meta-llama/Llama-3.1-8B-Instruct``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:193 +msgid "**Step 1. Create cache directory under your file system mount:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:195 +msgid "" +"To find all the types of file systems supporting GDS in your system, use " +"`gdscheck` from NVIDIA:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:201 +msgid "Check with your storage vendor on how to mount the remote file system." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:203 +msgid "" +"(For example, if you want to use a GDS-enabled NFS driver, try the " +"modified [NFS stack](https://vastnfs.vastdata.com/), which is an open " +"source driver that works with any standard [NFS " +"RDMA](https://datatracker.ietf.org/doc/html/rfc5532) server. More vendor-" +"specific instructions will be added here in the future)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:209 +msgid "" +"Create a directory under the file systew mount (the name here is " +"arbitrary):" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:215 +msgid "**Step 2. Start a vLLM server with file backend enabled:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:217 +msgid "Create a an lmcache configuration file called: ``gds-backend.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:226 +msgid "" +"If you don't want to use a config file, uncomment the first three " +"environment variables and then comment out the ``LMCACHE_CONFIG_FILE`` " +"below:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:244 +msgid "POSIX fallback" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:246 +msgid "" +"In some cases, libcufile implements its own internal POSIX fallback " +"without `GdsBackend` being aware. In others, an error such as " +"`RuntimeError: cuFileHandleRegister failed (cuFile err=5030, cuda_err=0)`" +" may be throwned. Thus, backend can be configured to fallback to its own " +"POSIX implementation when the usage of the GDS APIs is not successful." +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:250 +msgid "" +"To force `GdsBackend` not use GDS APIs for any reason, you can override " +"its behavior via configuration:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:256 +msgid "Or via environment variable:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:262 +msgid "" +"The ``gds_backend`` field (default: ``cufile``) selects which GDS library" +" to use. Supported backends are ``cufile`` (NVIDIA cuFile) and " +"``hipfile`` (AMD hipFile):" +msgstr "" + +#: ../../source/kv_cache/storage_backends/gds.rst:270 +msgid "" +"Note that under this mode it would still use CUDA APIs to map and do " +"operations the pre-registered GPU memory." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/hfbucket.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/hfbucket.po new file mode 100644 index 00000000000..15dce22767c --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/hfbucket.po @@ -0,0 +1,168 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:2 +msgid "Hugging Face Buckets Backend" +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:4 +msgid "" +"The Hugging Face Buckets backend stores LMCache chunks in a Hugging Face " +"Bucket using LMCache's built-in remote storage plugin framework. This is " +"a persistent remote backend that fits warm and cold KV cache persistence " +"better than the hottest local tiers." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:10 +msgid "When to use it" +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:12 +msgid "Use the HFBucket backend when you want:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:14 +msgid "A Hub-native persistent store for KV cache data." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:15 +msgid "" +"A remote backend that can be configured through " +"``remote_storage_plugins``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:16 +msgid "Multiple named bucket instances in one LMCache deployment." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:18 +msgid "" +"Avoid using it as the primary hot path for the lowest-latency cache " +"lookups. Local CPU, local disk, and other lower-latency backends are a " +"better fit for the hottest cache tier." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:24 +msgid "Requirements and limitations" +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:26 +msgid "" +"LMCache uses ``huggingface_hub`` bucket APIs for uploads, downloads, " +"listing, and deletes." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:28 +msgid "The first built-in release is intentionally conservative:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:30 +msgid "Only full chunks are supported." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:31 +msgid "Partial chunk uploads are rejected." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:32 +msgid "" +"Downloads are rejected when the stored object size does not match the " +"expected full LMCache chunk size." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:34 +msgid "Chunk metadata is not stored in the bucket objects." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:38 +msgid "Minimal configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:57 +msgid "Multiple instances" +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:59 +msgid "" +"Use instance-qualified plugin names to configure more than one bucket-" +"backed remote store in the same LMCache config." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:73 +msgid "Configuration reference" +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:75 +msgid "" +"All configuration keys live under " +"``extra_config.remote_storage_plugin..*`` where " +"``plugin_name`` is either ``hfbucket`` or an instance-qualified name such" +" as ``hfbucket.prod``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:79 +msgid "" +"``bucket_handle`` (required): Hugging Face Bucket handle in " +"``hf://buckets//[/]`` format." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:81 +msgid "" +"``token_env`` (optional, default ``HF_TOKEN``): Environment variable used" +" to resolve the Hugging Face access token." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:83 +msgid "" +"``token`` (optional): Direct token override. ``token_env`` takes " +"precedence when both are set." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:85 +msgid "" +"``create_bucket_if_missing`` (optional, default ``false``): Lazily create" +" the bucket on the first write path." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:87 +msgid "" +"``download_tmp_dir`` (optional): Root directory for connector-local " +"download scratch space. On Linux, pointing this at a tmpfs mount such as " +"``/dev/shm/lmcache-hfbucket`` avoids the disk write on the download path." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:90 +msgid "" +"``metadata_cache_ttl_secs`` (optional, default ``30``): TTL for cached " +"exact existence and size metadata." +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:95 +msgid "Notes" +msgstr "" + +#: ../../source/kv_cache/storage_backends/hfbucket.rst:97 +msgid "" +"The backend stores objects under the configured bucket prefix using a " +"reversible encoding of LMCache keys, so ``list()`` returns LMCache key " +"strings instead of raw bucket object paths." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/index.po new file mode 100644 index 00000000000..1bcb2e16501 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/index.po @@ -0,0 +1,35 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/index.rst:2 +msgid "Using Different Storage Backends" +msgstr "" + +#: ../../source/kv_cache/storage_backends/index.rst:4 +msgid "" +"LMCache supports various storage backends to offload and share KV cache " +"data." +msgstr "" + +#: ../../source/kv_cache/storage_backends/index.rst:7 +msgid "Supported Backends" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/infinistore.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/infinistore.po new file mode 100644 index 00000000000..b5a057948a9 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/infinistore.po @@ -0,0 +1,232 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:2 +msgid "InfiniStore" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:9 +msgid "" +"`InfiniStore `_ is an open-" +"source high-performance KV store. It's designed to support LLM Inference " +"clusters, whether the cluster is in prefill-decoding disaggregation mode " +"or not. InfiniStore provides high-performance and low-latency KV cache " +"transfer and KV cache reuse among inference nodes in the cluster." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:11 +msgid "There are two major scenarios how InfiniStore supports:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:13 +msgid "" +"Prefill-Decoding disaggregation clusters: in such mode inference " +"workloads are separated into two node pools: prefill nodes and decoding " +"nodes. InfiniStore enables KV cache transfer among these two types of " +"nodes, and also KV cache reuse." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:14 +msgid "" +"Non-disaggregated clusters: in such mode prefill and decoding workloads " +"are mixed on every node. InfiniStore serves as an extra large KV cache " +"pool in addition to GPU cache and local CPU cache, and also enables " +"cross-node KV cache reuse." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:16 +msgid "InfiniStore Usage Diagram" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:19 +msgid "" +"For more details, please refer to the `InfiniStore Documentation " +"`_." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:21 +msgid "" +"InfiniStore supports both RDMA and TCP for transport. LMCache’s " +"InfiniStore connector only uses the RDMA transport." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:25 +msgid "Quick Start" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:27 +msgid "Install InfiniStore via pip:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:33 +msgid "This package includes the InfiniStore server and the Python bindings." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:35 +msgid "" +"To build InfiniStore from source, follow the instructions in the `GitHub " +"repository `_." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:38 +msgid "Setup and Deployment" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:40 +msgid "**Prerequisites:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:42 +msgid "Machine with at least one GPU for vLLM inference" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:43 +msgid "RDMA-capable network hardware and drivers" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:44 +msgid "Python 3.8+ with pip" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:45 +msgid "vLLM and LMCache installed" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:47 +msgid "**Step 1: Start InfiniStore Server**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:49 +msgid "For InfiniBand based RDMA:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:55 +msgid "For RoCE based RDMA:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:61 +msgid "" +"You can also specify the ``--hint-gid-index`` option to set the GID index" +" for the InfiniStore server. This is useful when you are in a k8s managed" +" environment." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:63 +msgid "**Step 2: Create Configuration File**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:65 +msgid "Create your ``infinistore-config.yaml``:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:75 +msgid "**Step 3: Start vLLM with InfiniStore**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:88 +msgid "**Step 4: Verify the Setup**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:90 +msgid "Test the integration with a sample request:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:103 +msgid "**Debugging Tips:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:105 +msgid "**Enable verbose logging:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:111 +msgid "**Check server status:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:120 +msgid "Query TTFT Improvement" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:122 +msgid "" +"Once the OpenAI compatible server is running, let's query it twice and " +"see the TTFT improvement." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:124 +msgid "Run vLLM's serving benchmark twice with the following parameters:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:139 +msgid "**Example Output:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:141 +msgid "For the first run, you might see:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:166 +msgid "For the second run, you should see a significant reduction in TTFT:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:191 +msgid "TTFT Improvement: 33.323 seconds (12.6x faster)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:193 +msgid "**Tips:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:195 +msgid "" +"If you want to run vLLM's serving benchmark multiple times, you'll need " +"to either restart the vLLM LMCache server and the InfiniStore server, or " +"change the ``--seed`` parameter to a different value each time, since " +"you've already warmed up LMCache." +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:196 +msgid "" +"The benchmark result here was produced by running an L40 with 48GB of GPU" +" memory with ``--gpu-memory-utilization 0.8``. You can adjust the GPU " +"memory utilization and increase the max model length to use more of the " +"long context. LMCache TTFT improvement becomes more pronounced as the " +"context length increases!" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:200 +msgid "Additional Resources" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:202 +msgid "" +"`InfiniStore Documentation " +"`_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/infinistore.rst:203 +msgid "`GitHub Repository `_" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/local_storage.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/local_storage.po new file mode 100644 index 00000000000..e12d505ee2a --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/local_storage.po @@ -0,0 +1,363 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:2 +msgid "Local storage" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:9 +msgid "" +"CPU RAM and Local Storage are the two ways of offloading KV cache onto " +"non-GPU memory of the same machine that is running inference." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:14 +msgid "Two ways to configure LMCache Disk Offloading:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:17 +msgid "**1. Environment Variables:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:35 +msgid "**2. Configuration File**:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:37 +msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:54 +msgid "Multi-Path (Multi-Device) Disk Offloading" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:56 +msgid "" +"If you have **multiple NVMe devices** (or any independent mount points), " +"you can assign each GPU its own disk path so that each device writes to a" +" dedicated drive." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:59 +msgid "" +"Specify a **comma-separated list** of paths in ``local_disk``. Each path " +"can optionally use the ``file://`` prefix. The " +"``local_disk_path_sharding`` option controls how each GPU worker selects " +"its path. Currently only ``\"by_gpu\"`` is supported (the default), " +"which selects a path based on the device index (``device_id % " +"num_paths``), so all KV cache files from a given GPU land on the same " +"NVMe. This is especially useful when GPUs and NVMe devices share a PCIe " +"switch or NUMA node." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:67 +msgid "For example, with two GPUs and two paths:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:69 +msgid "``cuda:0`` → ``/mnt/nvme0/kvcache/``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:70 +msgid "``cuda:1`` → ``/mnt/nvme1/kvcache/``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:72 +msgid "``max_local_disk_size`` is the **total budget** shared across all paths." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:74 +msgid "**Environment variable example:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:82 +msgid "**YAML example:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:92 +msgid "" +"Each GPU worker uses only its assigned path, so O_DIRECT alignment is " +"determined by that path's filesystem block size. Different devices may " +"have different block sizes without issue." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:98 +msgid "" +"If you are able to use kernel-level RAID 0 (e.g. ``mdadm --level=0``) you" +" will get true block-level striping (even a single large file can use " +"bandwidth from both devices simultaneously). The multi-path feature is " +"most useful when you cannot or do not want to reconfigure the block " +"devices — for example, when they already have other data." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:105 +msgid "Local Storage Explanation:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:107 +msgid "" +"Unlike CPU RAM offloading, disk offloading is *disabled* by default " +"(``local_disk`` is set to ``None``) and the max local disk size is set to" +" 0GB instead of 5GB like the default max local cpu size since the disk " +"space is not strictly necessary for LMCache to function." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:111 +msgid "" +"Furthermore, instead of greedily allocating the max space up front like " +"the pinned CPU RAM, the disk backend will create one file per KV cache " +"chunk as they are stored, evicting if capacity is exceeded (LRU " +"currently)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:114 +msgid "" +"The disk and remote (see :doc:`Redis <./redis>`, :doc:`Mooncake " +"<./mooncake>`, :doc:`Valkey <./valkey>`, :doc:`InfiniStore " +"<./infinistore>`) backends have asynchronous put() operations so that the" +" IO latency will not slow down inference in addition to blocking get() " +"operations. The local disk backend also has a prefetch() operation that " +"will preemptively move KV caches from the disk to CPU RAM offloading " +"storage (i.e. ``LMCACHE_LOCAL_CPU=True`` should be set, see :doc:`CPU RAM" +" <./cpu_ram>`) for specified tokens (these KV caches are also still kept " +"in the disk)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:121 +msgid "Architecture Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:123 +msgid "" +"The following diagram shows the overall architecture of the Local Disk " +"Backend:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:171 +msgid "**Key Components:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:173 +msgid "" +"**Metadata Dictionary**: Maps each ``CacheEngineKey`` to its disk " +"metadata (file path, size, shape, dtype, pin status)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:174 +msgid "" +"**Cache Policy**: Configurable eviction policy (LRU, LFU, FIFO, or MRU) " +"that tracks access patterns and decides which entries to evict when space" +" is needed" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:175 +msgid "" +"**LocalDiskWorker**: Async task executor with priority queue - prefetch " +"tasks run first (priority 0), then deletes (priority 1), then saves " +"(priority 2)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:176 +msgid "" +"**Local Disk**: Filesystem where KV cache chunks are stored as individual" +" ``.pt`` files" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:180 +msgid "Save Flow (PUT)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:199 +msgid "Load Flow (GET)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:221 +msgid "Online Inference Example" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:223 +msgid "" +"This example is almost identical to the :doc:`CPU RAM <./cpu_ram>` " +"example." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:225 +msgid "Let's feel the TTFT (time to first token) differential!" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:229 +msgid "**Prerequisites:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:231 +msgid "" +"A Machine with at least one GPU. Adjust the max model length of your vllm" +" instance depending on your GPU memory and the long context you want to " +"use." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:233 +msgid "" +"vllm and lmcache installed (:doc:`Installation Guide " +"<../../getting_started/installation>`)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:235 +msgid "Hugging Face access to ``meta-llama/Meta-Llama-3.1-8B-Instruct``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:241 +msgid "A few packages:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:249 +msgid "**Step 0. Set up a directory for this example:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:256 +msgid "**Step 1. Prepare a long context!**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:258 +msgid "" +"We want a context long enough that vllm's prefix caching will not be able" +" to hold the KV caches in GPU memory and LMCache is necessary to keep KV " +"caches in non-GPU memory:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:266 +msgid "**Step 2. Start a vLLM server with Disk offloading enabled:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:268 +msgid "" +"*Generally, it is not recommended but we will disable CPU offloading to " +"feel just the disk offloading latency.*" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:270 +msgid "Create a an lmcache configuration file called: ``disk-offload.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:272 +msgid "Example ``config.yaml``:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:282 +msgid "" +"If you don't want to use a config file, uncomment the first five " +"environment variables and then comment out the ``LMCACHE_CONFIG_FILE`` " +"below:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:299 +msgid "" +"``--kv-transfer-config``: This is the parameter that actually tells vLLM " +"to use LMCache for KV cache offloading." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:300 +msgid "``kv_connector``: Specifies the LMCache connector for vLLM V1" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:301 +msgid "" +"``kv_role``: Set to \"kv_both\" for both storing and loading KV cache " +"(important because we will run two queries and the first will " +"produce/store a KV cache while the second will consume/load that KV " +"cache)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:304 +msgid "**Step 3. Query TTFT improvements with LMCache:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:306 +msgid "" +"Once the Open AI compatible server is running on default vllm port 8000, " +"let's query it twice with the same long context!" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:308 +msgid "Create a script called ``query-twice.py`` and paste the following code:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:374 +msgid "Then run:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:380 +msgid "" +"Since we're in streaming mode, you'll be able to feel the TTFT " +"differential in real time!" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:383 +msgid "" +"Note that if we were to enable ``LMCACHE_LOCAL_CPU=True``, we would just " +"be using the same example from :doc:`CPU RAM <./cpu_ram>` since the CPU " +"RAM is checked before the disk by LMCache. In practice, the disk will be " +"capable of storing a larger quantity of KV caches so the CPU RAM " +"offloading will only be able to store a subset of the disk's KV caches." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:389 +msgid "**Example Output:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:415 +msgid "TTFT Improvement: 6.166 seconds (42.6x faster)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:417 +msgid "" +"If you look at the logs of your vLLM server, you should see (the logs are" +" truncated for cleanliness):" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:442 +msgid "Check out your KV Cache in your SSD:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:451 +msgid "Tips:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:453 +msgid "" +"If you want to run the ``query-twice.py`` script multiple times, you'll " +"need to either restart the vLLM LMCache server or change the prefix of " +"the context you pass in since you've already warmed LMCache." +msgstr "" + +#: ../../source/kv_cache/storage_backends/local_storage.rst:455 +msgid "" +"The max model length here was decided by running an L4 with only 23GB of " +"GPU memory. If you have more memory, you can increase the max model " +"length and modify ``query-twice.py`` to use more of the long context. " +"LMCache TTFT improvement becomes more pronounced as the context length " +"increases!" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/maru.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/maru.po new file mode 100644 index 00000000000..f7428e32fc2 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/maru.po @@ -0,0 +1,203 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/maru.rst:2 +msgid "Maru" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:9 +msgid "" +"`Maru `_ is a high-performance KV " +"cache storage engine built on CXL shared memory, designed for LLM " +"inference scenarios where multiple instances need to share a KV cache " +"with minimal latency." +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:12 +msgid "KV Cache Sharing: Without vs With Maru" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:15 +msgid "" +"For architecture details, see the `Maru documentation `_." +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:18 +msgid "Quick Start" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:20 +msgid "Install Maru:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:28 +msgid "" +"This installs ``maru-server``, ``maru-resourced``, and the ``maru`` " +"Python package." +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:31 +msgid "Deploy Model With Maru" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:33 +msgid "" +"**Prerequisites:** CXL device (``/dev/dax*``), Python 3.12+, vLLM and " +"LMCache installed." +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:35 +msgid "**1. Start the Maru Server**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:41 +msgid "**2. Create configuration file** (``maru-config.yaml``):" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:54 +msgid "**3. Start vLLM with Maru**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:66 +msgid "Configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:68 +msgid "**LMCache Parameters:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:74 +#: ../../source/kv_cache/storage_backends/maru.rst:90 +msgid "Parameter" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:75 +#: ../../source/kv_cache/storage_backends/maru.rst:91 +msgid "Default" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:76 +#: ../../source/kv_cache/storage_backends/maru.rst:92 +msgid "Description" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:77 +msgid "``maru_path``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:78 +msgid "Required" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:79 +msgid "Maru server URL (format: ``maru://host:port``)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:80 +msgid "``maru_pool_size``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:81 +msgid "``4.0``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:82 +msgid "CXL memory pool size per instance in GB (e.g., ``4``, ``0.5``)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:84 +msgid "**Advanced Parameters (via extra_config):**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:93 +msgid "``maru_instance_id``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:94 +msgid "auto UUID" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:95 +msgid "Unique client instance identifier" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:96 +msgid "``maru_timeout_ms``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:97 +msgid "5000" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:98 +msgid "ZMQ RPC socket timeout in milliseconds" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:99 +msgid "``maru_use_async_rpc``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:100 +#: ../../source/kv_cache/storage_backends/maru.rst:106 +msgid "true" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:101 +msgid "Async DEALER-ROUTER RPC (``false`` for synchronous REQ-REP)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:102 +msgid "``maru_max_inflight``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:103 +msgid "64" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:104 +msgid "Max concurrent async RPC requests" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:105 +msgid "``maru_eager_map``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:107 +msgid "Pre-map all shared regions on connect" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:110 +msgid "Additional Resources" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:112 +msgid "`Maru GitHub Repository `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/maru.rst:113 +msgid "`Maru Documentation `_" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mock.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mock.po new file mode 100644 index 00000000000..d17a90bb437 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mock.po @@ -0,0 +1,104 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/mock.rst:2 +msgid "Mock" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:4 +msgid "" +"LMCache provides a mock remote connector that allows you to manually set " +"the peeking latency, read throughput, and write throughput inside of the " +"remote url. It will create copies of your KV cache in unmanaged local " +"RAM." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:7 +msgid "Configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:9 +msgid "" +"Create a configuration file (e.g., ``mock.yaml``) with the following " +"content:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:18 +msgid "" +"The ``remote_url`` format is " +"``mock://SIZE/?peeking_latency=LATENCY&read_throughput=READ_GBPS&write_throughput=WRITE_GBPS``" +" where:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:20 +msgid "``SIZE``: Maximum storage size" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:21 +msgid "``peeking_latency``: Latency for peeking operations (in milliseconds)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:22 +msgid "``read_throughput``: Read throughput (in GB/s)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:23 +msgid "``write_throughput``: Write throughput (in GB/s)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:26 +msgid "Usage" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:28 +msgid "Deploy a serving engine with the mock remote backend:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:37 +msgid "" +"Check the retrieval (storing is async so the throughput there is " +"meaningless) logs on the second query to confirm that the throughput is " +"slightly lower than 2 GB/s (the CPU <-> GPU allocation/transfer also has " +"overhead)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:40 +msgid "Example Query" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:42 +msgid "Send a test request:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:55 +msgid "Expected Logs" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:57 +msgid "You should see logs similar to the following:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mock.rst:64 +msgid "" +"The logs confirm that the throughput is slightly lower than 2 GB/s due to" +" CPU <-> GPU allocation/transfer overhead." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mooncake.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mooncake.po new file mode 100644 index 00000000000..bbda73c95c3 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mooncake.po @@ -0,0 +1,525 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:2 +msgid "Mooncake" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:9 +msgid "" +"`Mooncake `_ is an open-source " +"distributed KV cache storage system designed specifically for LLM " +"inference scenarios. The system creates a distributed memory pool by " +"aggregating memory space contributed by various client nodes, enabling " +"efficient resource utilization across clusters." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:12 +msgid "" +"By pooling underutilized DRAM and SSD resources from multiple nodes, the " +"system forms a unified distributed storage service that maximizes " +"resource efficiency." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:14 +msgid "Mooncake Architecture Diagram" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:18 +msgid "Key Features" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:20 +msgid "" +"**Distributed memory pooling**: Aggregates memory contributions from " +"multiple client nodes into a unified storage pool" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:21 +msgid "" +"**High bandwidth utilization**: Supports striping and parallel I/O " +"transfer of large objects, fully utilizing multi-NIC aggregated bandwidth" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:22 +msgid "" +"**RDMA optimization**: Built on Transfer Engine with support for TCP, " +"RDMA (InfiniBand/RoCEv2/eRDMA/NVIDIA GPUDirect)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:23 +msgid "" +"**Dynamic resource scaling**: Supports dynamically adding and removing " +"nodes for elastic resource management" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:25 +msgid "" +"For detailed architecture information, see the `Mooncake Architecture " +"Guide `_." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:28 +msgid "Quick Start" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:30 +msgid "Install Mooncake via pip:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:36 +msgid "This package includes all necessary components:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:38 +msgid "" +"``mooncake_master``: Master service that manages cluster metadata and " +"coordinates distributed storage operations" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:39 +msgid "" +"``mooncake_http_metadata_server``: HTTP-based metadata server used by the" +" underlying transfer engine for connection establishment" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:40 +msgid "Mooncake Python bindings" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:42 +msgid "" +"For production deployments or custom builds, see the `Build Instructions " +"`_." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:45 +msgid "Setup and Deployment" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:47 +msgid "**Prerequisites:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:49 +msgid "Machine with at least one GPU for vLLM inference" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:50 +msgid "RDMA-capable network hardware and drivers (recommended) or TCP network" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:51 +msgid "Python 3.8+ with pip" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:52 +msgid "vLLM and LMCache installed" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:54 +msgid "**Step 1: Start Infrastructure Services**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:56 +msgid "Start the Mooncake master service (with built‑in HTTP metadata server):" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:64 +msgid "Expected output:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:72 +msgid "**Step 2: Create Configuration File**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:74 +msgid "Create your ``mooncake-config.yaml``:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:98 +msgid "**Step 3: Start vLLM with Mooncake**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:111 +msgid "**Step 4: Verify the Setup**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:113 +msgid "Test the integration with a sample request:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:126 +msgid "**Debugging Tips:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:128 +msgid "**Enable verbose logging:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:134 +msgid "**Check service status:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:142 +msgid "**Monitor metrics:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:144 +msgid "" +"Access metrics at ``http://localhost:9003`` when master service is " +"running." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:147 +msgid "Configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:149 +msgid "**LMCache Parameters:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:155 +#: ../../source/kv_cache/storage_backends/mooncake.rst:186 +msgid "Parameter" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:156 +#: ../../source/kv_cache/storage_backends/mooncake.rst:187 +msgid "Default" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:157 +#: ../../source/kv_cache/storage_backends/mooncake.rst:188 +msgid "Description" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:158 +msgid "``chunk_size``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:159 +msgid "256" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:160 +msgid "Number of tokens per KV chunk" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:161 +msgid "``remote_url``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:162 +#: ../../source/kv_cache/storage_backends/mooncake.rst:171 +#: ../../source/kv_cache/storage_backends/mooncake.rst:190 +#: ../../source/kv_cache/storage_backends/mooncake.rst:193 +#: ../../source/kv_cache/storage_backends/mooncake.rst:196 +msgid "Required" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:163 +msgid "Mooncake store connection URL (format: ``mooncakestore://host:port/``)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:164 +msgid "``remote_serde``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:165 +msgid "\"naive\"" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:166 +msgid "Serialization method for remote storage" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:167 +msgid "``local_cpu``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:168 +#: ../../source/kv_cache/storage_backends/mooncake.rst:220 +#: ../../source/kv_cache/storage_backends/mooncake.rst:223 +#: ../../source/kv_cache/storage_backends/mooncake.rst:226 +msgid "False" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:169 +msgid "" +"Enable/disable local CPU caching (set to False for pure Mooncake " +"evaluation)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:170 +msgid "``max_local_cpu_size``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:172 +msgid "Maximum local CPU cache size in GB (required even when local_cpu is False)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:173 +msgid "``numa_mode``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:174 +msgid "\"auto\"" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:175 +msgid "" +"NUMA binding mode. \"auto\" is recommended on multi‑NIC/multi‑NUMA " +"systems to reduce tail latency." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:176 +msgid "``pre_caching_hash_algorithm``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:177 +msgid "\"sha256_cbor_64bit\"" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:178 +msgid "" +"Hash used for pre-caching keying. For cross‑process consistency, fix " +"``PYTHONHASHSEED`` (e.g., export ``PYTHONHASHSEED=0``)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:180 +msgid "**Mooncake Parameters (via extra_config):**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:189 +msgid "``local_hostname``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:191 +msgid "Hostname/IP of the local node for Mooncake client identification" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:192 +msgid "``metadata_server``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:194 +msgid "" +"HTTP metadata server address. When starting master with " +"``--enable_http_metadata_server=1``, it exposes this endpoint." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:195 +msgid "``master_server_address``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:197 +msgid "Mooncake master service address (host:port format)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:198 +msgid "``protocol``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:199 +msgid "\"rdma\"" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:200 +msgid "" +"Communication protocol (\"rdma\" for high performance; \"tcp\" for " +"compatibility)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:201 +msgid "``device_name``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:202 +#: ../../source/kv_cache/storage_backends/mooncake.rst:217 +msgid "\"\"" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:203 +msgid "" +"RDMA device specification (e.g., \"erdma_0,erdma_1\" or " +"\"mlx5_0,mlx5_1\"). Leave empty for autodetection in most setups." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:204 +msgid "``global_segment_size``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:205 +msgid "21474836480" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:206 +msgid "" +"**Memory size contributed by each vLLM worker** in bytes (e.g., 20 GiB " +"recommended)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:207 +msgid "``local_buffer_size``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:208 +msgid "0" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:209 +msgid "" +"Local buffer size in bytes used by Mooncake. Behavior depends on " +"``save_chunk_meta``: - When ``save_chunk_meta: False`` (recommended), " +"LMCache uses its local CPU backend for zero‑copy RDMA, so Mooncake's " +"``local_buffer_size`` can be ``0``. - When ``save_chunk_meta: True``, " +"Mooncake uses its own local buffer; set this to a proper value (e.g., " +"several GiB). - Note: Some RDMA NICs have memory registration limits; " +"registering LMCache's large CPU buffer can fail on constrained devices. " +"In those cases, consider enabling ``save_chunk_meta: True`` and sizing " +"``local_buffer_size`` instead." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:213 +msgid "``transfer_timeout``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:214 +msgid "1" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:215 +msgid "Timeout for transfer operations in seconds" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:216 +msgid "``storage_root_dir``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:218 +msgid "The root directory for persistence (e.g., \"/mnt/mooncake\")" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:219 +msgid "``save_chunk_meta``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:221 +msgid "" +"Whether to save chunk metadata alongside data. Set to ``False`` to enable" +" the optimized zero‑copy path in LMCache." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:222 +msgid "``use_exists_sync``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:224 +msgid "" +"Use synchronous existence checks to avoid async scheduling overhead in " +"hot paths." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:225 +msgid "``mooncake_prefer_local_alloc``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:227 +msgid "Prefer allocating on the local segment when possible." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:230 +msgid "" +"**Understanding global_segment_size**: This parameter defines the amount " +"of memory each vLLM worker contributes to the distributed memory pool. " +"The total cluster memory available for KV cache storage will be: " +"``number_of_vllm_workers × global_segment_size``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:233 +msgid "" +"Adjust this value based on your available system memory and expected " +"cache requirements." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:236 +msgid "" +"If you consistently get misses (no Mooncake hits), ensure all processes " +"use the same hashing seed: ``export PYTHONHASHSEED=0``. This keeps " +"pre‑caching keys consistent across processes." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:239 +msgid "" +"RDMA device(s) usually do not need to be specified; leaving " +"``device_name`` empty works for most deployments." +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:242 +msgid "Additional Resources" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:244 +msgid "" +"`Mooncake Store Architecture `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:245 +msgid "" +"`Mooncake Store Deployment Guide `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:246 +msgid "" +"`Mooncake Store Python API Reference `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:247 +msgid "" +"`Transfer Engine Documentation `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:248 +msgid "" +"`Build Instructions `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:249 +msgid "`GitHub Repository `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/mooncake.rst:250 +msgid "" +"`LMCache Integration Guide `_" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/nixl.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/nixl.po new file mode 100644 index 00000000000..9c335bfdb9c --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/nixl.po @@ -0,0 +1,203 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/nixl.rst:3 +msgid "Nixl" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:8 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:10 +msgid "" +"NIXL (NVIDIA Inference Xfer Library) is a high-performance library " +"designed for accelerating point to point communications in AI inference " +"frameworks. It provides an abstraction over various types of memory (CPU " +"and GPU) and storage through a modular plug-in architecture, enabling " +"efficient data transfer and coordination between different components of " +"the inference pipeline." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:12 +msgid "" +"LMCache supports using NIXL as a storage backend, allowing using NIXL to " +"save either GPU or CPU memory into storage." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:15 +msgid "Prerequisites" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:17 +msgid "**LMCache**: Install with ``pip install lmcache``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:18 +msgid "" +"**NIXL**: Install from `NIXL GitHub repository `_" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:19 +msgid "" +"**Model Access**: Valid Hugging Face token (HF_TOKEN) for Llama 3.1 8B " +"Instruct" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:22 +msgid "Ways to configure LMCache NIXL Offloading" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:24 +msgid "**Configuration File**:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:26 +msgid "Passed in through ``LMCACHE_CONFIG_FILE=lmcache-config.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:28 +msgid "Example ``lmcache-config.yaml`` for POSIX backend:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:42 +msgid "Key settings:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:44 +msgid "``nixl_buffer_size``: buffer size for NIXL transfers." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:46 +msgid "" +"``nixl_pool_size``: number of descriptors opened at init time for nixl " +"backend. Set to 0 for dynamic mode." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:48 +msgid "" +"``nixl_path``: directory under which the storage files will be saved " +"(e.g. /mnt/nixl/). Needed for NIXL backends that store to file." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:50 +msgid "" +"``nixl_buffer_device``: dictates where the memory managed by NIXL should " +"be on. \"cpu\" or \"cuda\" is supported for \"GDS\", \"GDS_MT\", and " +"\"OBJ\" backends - for \"POSIX\", \"HF3FS\" & \"AZURE_BLOB\", must be " +"\"cpu\"." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:52 +msgid "``nixl_backend``: configuration of which nixl backend to use for storage." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:56 +msgid "" +"Supported backends are: [\"GDS\", \"GDS_MT\", \"POSIX\", \"HF3FS\", " +"\"OBJ\", \"AZURE_BLOB\"]." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:58 +msgid "" +"Backend specific params should be provided via " +"``extra_config.nixl_backend_params``. Please refer to NIXL documentation " +"for specifics." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:60 +msgid "Example ``lmcache-config.yaml`` for OBJ backend using S3 API:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:78 +msgid "Example ``lmcache-config.yaml`` for POSIX backend using liburing:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:82 +msgid "" +"using POSIX backend with liburing requires NIXL to be built with liburing" +" support." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:98 +msgid "" +"Example ``lmcache-config.yaml`` for AZURE_BLOB backend to offload using " +"Azure Blob Storage API:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:115 +msgid "Per-Worker Endpoint Distribution" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:117 +msgid "" +"When using the OBJ backend with multiple tensor-parallel (TP) workers, " +"you can distribute workers across multiple object-storage endpoints by " +"providing a list of endpoints via ``nixl_endpoint_list``. Each worker " +"selects an endpoint in round-robin order based on its ``local_worker_id``" +" (the worker ID within its host)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:141 +msgid "" +"When ``nixl_endpoint_list`` is set, any ``endpoint_override`` value in " +"``nixl_backend_params`` is ignored (a warning is logged)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:145 +msgid "Dynamic Mode" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:147 +msgid "" +"Nixl Storage Backend also supports a dynamic mode, which creates nixl " +"storage descriptors on demand instead of at init time." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:149 +msgid "" +"In order to use dynamic mode, extra_config.nixl_pool_size should be set " +"to 0." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:152 +msgid "Restrictions" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:154 +msgid "" +"Dynamic mode is currently only supported for nixl OBJ and AZURE_BLOB " +"backends." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:155 +msgid "save_unfull_chunk must be set to False." +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:157 +msgid "Example ``lmcache-config.yaml`` for OBJ backend with dynamic mode:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/nixl.rst:184 +msgid "Example ``lmcache-config.yaml`` for AZURE_BLOB backend with dynamic mode:" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/redis.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/redis.po new file mode 100644 index 00000000000..c4b458f2b4e --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/redis.po @@ -0,0 +1,347 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/redis.rst:2 +msgid "Redis" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:9 +msgid "" +"Redis is an in-memory key-value store and is a supported option for " +"remote KV Cache offloading in LMCache. Some other remote backends are " +":doc:`Mooncake <./mooncake>`, :doc:`Valkey <./valkey>`, and " +":doc:`InfiniStore <./infinistore>`. This guide will mainly focus on " +"single-node Redis but also shows you how to set up Redis Sentinels and an" +" LMCache Server." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:14 +msgid "Two ways to configure LMCache Redis Offloading:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:16 +msgid "**1. Environment Variables:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:32 +msgid "**2. Configuration File**:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:34 +msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:36 +msgid "Example ``config.yaml``:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:53 +msgid "Remote Storage Explanation:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:55 +msgid "" +"LMCache's backend is obeys the natural memory hierarchy of prioritizing " +"CPU RAM offloading, then Local Storage offloading, and finally remote " +"offloading." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:58 +msgid "" +"For LMCache to know how to create a connector to a remote backend, you " +"must specify in ``remote_url`` a connector type followed by one or most " +"host:port pairs (depending on what connector type is used). If " +"``remote_url`` is set to ``None``, LMCache will not use any remote " +"storage." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:62 +msgid "Examples of ``remote_url``'s:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:73 +msgid "Remote Storage Example" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:77 +msgid "**Prerequisites:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:79 +msgid "" +"A Machine with at least one GPU. You can adjust the max model length of " +"your vllm instance depending on your GPU memory." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:81 +msgid "" +"vllm and lmcache installed (:doc:`Installation Guide " +"<../../getting_started/installation>`)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:83 +msgid "Hugging Face access to ``meta-llama/Meta-Llama-3.1-8B-Instruct``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:89 +msgid "**Step 0. Set up a directory for this example:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:96 +msgid "**Step 1. Start a Redis server:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:104 +msgid "Check if Redis is running:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:110 +msgid "Expected Response:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:116 +msgid "**Optional: Setting up Sentinels:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:118 +msgid "" +"To enable high availability with Redis, you can configure Redis sentinels" +" to monitor the master and automatically fail over to a replica if " +"needed." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:121 +msgid "**Step 1a. Start a Redis replica:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:127 +msgid "**Step 1b. Create Sentinel configuration files:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:129 +msgid "" +"Create three files: ``sentinel-26379.conf``, ``sentinel-26380.conf``, and" +" ``sentinel-26381.conf``, with contents like this:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:139 +msgid "**Step 1c. Start each Sentinel:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:147 +msgid "**Step 1d. Make sure the Sentinels are tracking the master:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:155 +msgid "**Step 1e. Verify everything is running:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:161 +msgid "You should see something like this (without the comments):" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:175 +msgid "**Alternative: Starting an LMCache Server:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:177 +msgid "" +"The ``lmcache_server`` CLI entrypoint starts a remote LMCache server and " +"comes with the ``lmcache`` package." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:186 +msgid "" +"Currently, the only supported device is \"cpu\" (which is the default, so" +" you don't need to specify it)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:189 +msgid "**Step 2. Start a vLLM server with remote offloading enabled:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:191 +msgid "Create a an lmcache configuration file called: ``redis-offload.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:202 +#: ../../source/kv_cache/storage_backends/redis.rst:230 +#: ../../source/kv_cache/storage_backends/redis.rst:255 +msgid "" +"If you don't want to use a config file, uncomment the first three " +"environment variables and then comment out the ``LMCACHE_CONFIG_FILE`` " +"below:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:220 +msgid "**Optional: Sentinels**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:222 +msgid "" +"Create a an lmcache configuration file called: ``redis-sentinel-" +"offload.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:245 +msgid "**Alternative: LMCache Server**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:247 +msgid "" +"Create a an lmcache configuration file called: ``lmcache-server-" +"offload.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:270 +msgid "**Step 3. Viewing and Managing LMCache Entries in Redis:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:272 +msgid "" +"If you would like to feel the TTFT speed up with offloading and KV Cache " +"reuse, feel free to use the same ``query-twice.py`` script and ``man-" +"bash.txt`` long context as in :doc:`CPU RAM <./cpu_ram>` and :doc:`Local " +"Storage <./local_storage>`." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:275 +msgid "" +"Here, we are instead going to demonstrate how to search for and modify " +"LMCache KV Chunk entries in Redis." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:277 +msgid "" +"Please note that the official LMCache way to achieve this redis-specific " +"functionality of viewing and modifying LMCache KV Chunks is available in " +":doc:`LMCache Controller <../../kv_cache_management/index>`." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:279 +msgid "Let's warm/populate LMCache first with ``curl`` this time:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:296 +msgid "" +"LMCache stores data in Redis using a structured key format. Each key " +"contains the following information in a delimited format:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:302 +msgid "`model_name`: Name of the language model" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:303 +msgid "`world_size`: Total number of workers in distributed deployment" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:304 +msgid "" +"`worker_id`: ID of the worker that created this cache entry, in the range" +" of [0, world_size - 1]" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:305 +msgid "`chunk_hash`: Hash of the token chunk (SHA-256 based)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:307 +msgid "For example, a typical key might look like:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:313 +msgid "**Using redis-cli to View LMCache Data**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:315 +msgid "To inspect and manage LMCache entries in Redis:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:321 +msgid "**Optional: If you are using sentinels, first find the master port:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:329 +msgid "**List LMCache keys:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:331 +msgid "" +"Notice (from the suffixes of the keys) that each LMCache KV Chunk has two" +" entries: ``kv_bytes`` and ``metadata``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:344 +msgid "**Delete LMCache entries:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:350 +msgid "Delete a specific LMCache entry:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:359 +msgid "**Check if a key exists:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:365 +msgid "**View memory usage for a key:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:367 +msgid "" +"Notice that the ``kv_bytes`` entry is what is exactly holding the KV " +"Chunk and is much larger than the ``metadata`` entry." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:377 +msgid "**Delete specific keys:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:391 +msgid "**Monitor Redis in real-time:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:397 +msgid "**Get Redis stats for LMCache:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:407 +msgid "" +"This tutorial utilized the ``redis-cli`` to directly peak into a remote " +"backend and manipualte KV Chunks." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:410 +msgid "" +"Once again, please refer to the :doc:`LMCache Controller " +"<../../kv_cache_management/index>` for the official LMCache way of " +"controlling and routing your KV Caches in your LMCache instances." +msgstr "" + +#: ../../source/kv_cache/storage_backends/redis.rst:413 +msgid "**Step 4. Clean up:**" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/resp.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/resp.po new file mode 100644 index 00000000000..5ad4b077ecb --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/resp.po @@ -0,0 +1,716 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/resp.rst:2 +msgid "RESP (Native Redis/Valkey)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:9 +msgid "" +"The RESP backend is a high-performance native C++ storage connector for " +"Redis and Valkey servers, using the RESP2 wire protocol over TCP. It is " +"designed for maximum throughput on KV cache store and retrieval " +"operations, achieving **6+ GB/s** on reads with optimal configuration." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:13 +msgid "Key advantages over the standard Redis connector:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:15 +msgid "" +"**Multi-threaded C++ I/O**: Worker threads operate in parallel with zero-" +"copy buffer passing and full GIL release" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:16 +msgid "" +"**Batched tiling**: Large batch operations are automatically split across" +" worker threads for maximum parallelism" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:17 +msgid "" +"**eventfd-based completions**: The kernel wakes Python on completion -- " +"no polling overhead" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:18 +msgid "" +"**Dual-mode support**: The same C++ connector works in both non-MP mode " +"(via ``ConnectorClientBase``) and MP mode (via " +"``NativeConnectorL2Adapter`` as an L2 adapter)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:20 +msgid "" +"The native C++ source lives in ``csrc/storage_backends/redis/``. See " +":doc:`Adding Native Connectors " +"<../../developer_guide/extending_lmcache/native_connectors>` for the full" +" architecture." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:24 +msgid "Prerequisites" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:26 +msgid "" +"LMCache installed from source (``pip install -e .``) to compile the C++ " +"extension" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:27 +msgid "" +"A Redis 8.2+ or Valkey server (Redis 8.2 recommended for IO threads " +"support)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:28 +msgid "A machine with at least one GPU for vLLM inference" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:32 +msgid "Redis Server Setup" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:35 +msgid "" +"Redis version and server configuration have a **major** impact on " +"throughput. Using Redis 8.2 with IO threads yields ~6 GB/s reads vs ~1.5 " +"GB/s with Redis 6.0 defaults." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:38 +msgid "**Build Redis 8.2 from source (recommended):**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:47 +msgid "**Start the server with IO threads enabled:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:58 +msgid "Recommended Server Flags" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:62 +msgid "Flag" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:63 +msgid "Why" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:64 +msgid "``--protected-mode no``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:65 +msgid "Allow connections from other hosts (use auth in production)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:66 +msgid "``--save '' --appendonly no``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:67 +msgid "Disable persistence -- KV cache is ephemeral, persistence wastes bandwidth" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:68 +msgid "``--io-threads 4``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:69 +msgid "Enable multi-threaded I/O for parallel read/write handling" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:70 +msgid "``--port 6379``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:71 +msgid "Default port (adjust if running multiple instances)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:74 +msgid "" +"The number of ``--io-threads`` should roughly match the number of " +"physical cores available to the Redis process. 4 is a good starting " +"point; benchmark with your hardware to find the optimum." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:80 +msgid "Chunk Size Selection and Throughput Tuning" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:82 +msgid "" +"The chunk size (in tokens) determines how many bytes each Redis key-value" +" pair holds. This is the **single most important parameter** for " +"throughput." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:85 +msgid "" +"**The sweet spot is ~4 MB per chunk.** Both smaller and larger chunks " +"degrade throughput:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:87 +msgid "Chunk Size vs Throughput (Redis 8.2, 8 workers)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:91 +msgid "Chunk Size" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:92 +msgid "Total Data" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:93 +msgid "SET Throughput" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:94 +msgid "GET Throughput" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:95 +msgid "1 MB (500 keys)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:96 +msgid "500 MB" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:97 +msgid "~3.5 GB/s" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:98 +msgid "~5.2 GB/s" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:99 +msgid "**4 MB (500 keys)**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:100 +msgid "**2 GB**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:101 +msgid "**~4.4 GB/s**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:102 +msgid "**~5.9 GB/s**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:103 +msgid "8 MB (200 keys)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:104 +msgid "1.6 GB" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:105 +msgid "~4.2 GB/s" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:106 +msgid "~1.4 GB/s" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:108 +msgid "**Why 4 MB?**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:110 +msgid "Below ~2 MB, per-key overhead (RESP framing, TCP round-trips) dominates" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:111 +msgid "" +"Above ~4 MB, Redis server-side memory allocation and TCP window sizes " +"become bottlenecks" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:112 +msgid "" +"At 4 MB, the balance between amortized overhead and memory pressure is " +"optimal" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:114 +msgid "**Calculating chunk size in tokens:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:116 +msgid "" +"The chunk size in bytes depends on the model's hidden dimension, number " +"of KV heads, number of layers, and dtype:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:123 +msgid "For ``meta-llama/Llama-3.1-8B-Instruct`` with BFloat16:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:135 +msgid "" +"The bytes-per-token calculation varies by model architecture. Larger " +"models (e.g., 70B) have more layers and larger hidden dimensions, so " +"fewer tokens are needed per chunk to reach the 4 MB sweet spot." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:141 +msgid "Throughput Sweep" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:143 +msgid "" +"To find the optimal configuration for your hardware, use the included " +"benchmark:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:165 +msgid "Expected output:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:181 +msgid "Environment Variable Configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:183 +msgid "" +"Sensitive credentials (and optionally host/port) can be provided via " +"environment variables instead of config files or CLI arguments. This " +"prevents secrets from appearing in logged configuration at startup." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:191 +msgid "Variable" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:192 +#: ../../source/kv_cache/storage_backends/resp.rst:322 +msgid "Description" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:193 +msgid "``LMCACHE_RESP_USERNAME``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:194 +msgid "" +"Redis ACL username. Used as default when ``username`` is not set in " +"config/JSON." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:195 +msgid "``LMCACHE_RESP_PASSWORD``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:196 +msgid "" +"Redis AUTH password. Used as default when ``password`` is not set in " +"config/JSON." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:197 +msgid "``LMCACHE_RESP_HOST``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:198 +msgid "" +"Redis hostname or IP. Used as default when ``host`` is not set in " +"config/JSON/URL." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:199 +msgid "``LMCACHE_RESP_PORT``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:200 +msgid "Redis port. Used as default when ``port`` is not set in config/JSON/URL." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:202 +msgid "" +"Config files (non-MP) and ``--l2-adapter`` JSON (MP) take precedence over" +" environment variables. Environment variables serve as defaults — they " +"are used when the corresponding config value is empty or unset. They are " +"read at adapter creation time inside the adapter itself, so they are " +"**never stored in the config object** and **never printed in startup " +"logs**." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:208 +msgid "**Example — MP mode with env vars:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:222 +msgid "**Example — Non-MP mode with env vars:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:236 +msgid "" +"For production deployments, always use environment variables for " +"credentials rather than embedding them in config files or CLI arguments." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:241 +msgid "Non-MP Mode (Single Process)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:243 +msgid "" +"In non-MP mode, the RESP connector is used directly as a remote storage " +"backend via the ``RESPClient`` asyncio wrapper." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:246 +msgid "**Configuration file** (``resp-config.yaml``):" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:254 +msgid "" +"Credentials can be set via environment variables (recommended) or in the " +"config file under ``extra_config`` (see `Environment Variable " +"Configuration`_ above)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:257 +msgid "**Launch vLLM:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:268 +msgid "" +"``save_unfull_chunk`` must be off (default) and chunk metadata saving " +"must be disabled for optimal throughput with the native RESP connector." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:273 +msgid "MP Mode (Multiprocess)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:275 +msgid "" +"In MP mode, LMCache runs as a separate server process communicating with " +"vLLM over ZMQ. The RESP connector serves as an L2 adapter with variable-" +"size chunk support." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:278 +msgid "**Step 1: Start Redis** (see `Redis Server Setup`_ above)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:280 +msgid "**Step 2: Launch LMCache MP Server:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:291 +msgid "**Step 3: Launch vLLM with LMCache MP Connector:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:311 +msgid "L2 Adapter Configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:313 +msgid "The ``--l2-adapter`` JSON accepts these fields:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:319 +msgid "Field" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:320 +msgid "Type" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:321 +msgid "Default" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:323 +msgid "``type``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:324 +#: ../../source/kv_cache/storage_backends/resp.rst:328 +#: ../../source/kv_cache/storage_backends/resp.rst:340 +#: ../../source/kv_cache/storage_backends/resp.rst:344 +msgid "str" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:325 +#: ../../source/kv_cache/storage_backends/resp.rst:329 +#: ../../source/kv_cache/storage_backends/resp.rst:333 +msgid "(required)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:326 +msgid "Must be ``\"resp\"``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:327 +msgid "``host``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:330 +msgid "Redis/Valkey hostname or IP" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:331 +msgid "``port``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:332 +#: ../../source/kv_cache/storage_backends/resp.rst:336 +msgid "int" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:334 +msgid "Redis/Valkey port" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:335 +msgid "``num_workers``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:337 +msgid "8" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:338 +msgid "C++ worker threads for parallel I/O" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:339 +msgid "``username``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:341 +#: ../../source/kv_cache/storage_backends/resp.rst:345 +msgid "``\"\"``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:342 +msgid "" +"Redis ACL username (leave empty for no auth). Falls back to " +"``LMCACHE_RESP_USERNAME`` env var if empty." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:343 +msgid "``password``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:346 +msgid "" +"Redis AUTH password (leave empty for no auth). Falls back to " +"``LMCACHE_RESP_PASSWORD`` env var if empty." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:347 +msgid "``max_capacity_gb``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:348 +msgid "float" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:349 +msgid "0" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:350 +msgid "" +"Maximum L2 storage capacity in GB for client-side usage tracking. " +"Required for L2 eviction. Set to 0 (default) to disable usage tracking." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:353 +msgid "L2 Eviction" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:355 +msgid "" +"To enable automatic eviction of least-recently-used keys when the Redis " +"backend fills up, set ``max_capacity_gb`` and add an ``\"eviction\"`` " +"block:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:378 +#, python-format +msgid "" +"This configures a 10 GB capacity limit. When usage exceeds 80% " +"(``trigger_watermark``), the eviction controller will delete the least-" +"recently-used ~20% of stored keys (``eviction_ratio``) using the Redis " +"``DEL`` command." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:383 +msgid "" +"``max_capacity_gb`` enables **client-side** size tracking. It does not " +"configure the Redis server's ``maxmemory`` setting. You should set " +"``max_capacity_gb`` to match or be slightly below your Redis server's " +"available memory." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:389 +msgid "Testing the Setup" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:391 +msgid "" +"Send the same prompt twice. The first request stores KV cache to Redis; " +"the second retrieves it." +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:408 +msgid "Verify data was stored:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:414 +msgid "Clear state between runs:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:422 +msgid "Best Practices" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:424 +msgid "**Server deployment:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:426 +msgid "Use Redis 8.2+ with ``--io-threads 4`` (or more, matching available cores)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:427 +msgid "Disable persistence (``--save '' --appendonly no``) for KV cache workloads" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:428 +msgid "Pin Redis to its own NUMA node if running on multi-socket systems" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:429 +msgid "" +"For production, enable authentication with ``--requirepass`` and supply " +"credentials via ``LMCACHE_RESP_USERNAME`` / ``LMCACHE_RESP_PASSWORD`` " +"environment variables to keep them out of logs" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:431 +msgid "**Client tuning:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:433 +msgid "" +"Start with ``num_workers: 8`` and increase if the server has spare CPU " +"and you're not saturating the network" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:434 +msgid "" +"More workers help when chunk sizes are smaller (more keys per batch = " +"more parallelism needed)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:435 +msgid "" +"On NUMA systems, ensure the LMCache process runs on the same NUMA node as" +" the NIC" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:437 +msgid "**Chunk size:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:439 +msgid "Target ~4 MB per chunk for maximum throughput" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:440 +msgid "" +"Calculate the token count using your model's per-token byte size (see " +"formula above)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:441 +msgid "" +"If unsure, run the benchmark sweep to find the optimum for your specific " +"hardware" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:443 +msgid "**Network:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:445 +msgid "Use localhost or loopback for single-machine deployments" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:446 +msgid "" +"For cross-machine setups, ensure low-latency networking (ideally <100 us " +"RTT)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:447 +msgid "" +"The RESP connector uses TCP; RDMA is not currently supported (consider " +":doc:`Mooncake <./mooncake>` for RDMA)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:451 +msgid "Additional Resources" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:453 +msgid "" +"Benchmark script: " +"``examples/kv_cache_reuse/remote_backends/resp/benchmark_resp_client.py``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:454 +msgid "C++ source: ``csrc/storage_backends/redis/``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:455 +msgid "Native connector architecture: ``csrc/storage_backends/README.md``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/resp.rst:456 +msgid "" +"Developer guide for adding new native connectors: :doc:`Adding Native " +"Connectors <../../developer_guide/extending_lmcache/native_connectors>`" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/s3.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/s3.po new file mode 100644 index 00000000000..2893d24920a --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/s3.po @@ -0,0 +1,231 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/s3.rst:2 +msgid "S3 Backend" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:5 +msgid "Example Configurations" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:8 +msgid "Basic S3 Configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:24 +msgid "S3 Express One Zone" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:42 +msgid "CoreWeave (S3-compatible)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:64 +msgid "" +"**Note**: `cwlota.com` is CoreWeave's S3-compatible Cloud Storage that " +"caches for GPU locality. You can set `disable_tls: True` for non-AWS " +"services." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:66 +msgid "" +"Check out the blog post between LMCache, Cohere, and CoreWeave: " +"https://blog.lmcache.ai/en/2025/10/29/breaking-the-memory-barrier-how-" +"lmcache-and-coreweave-power-efficient-llm-inference-for-cohere/" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:70 +msgid "Configuration Parameters" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:72 +msgid "**remote_url**: S3 bucket URL (`s3://bucket-name`)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:73 +msgid "" +"**save_unfull_chunk**: Save partial chunks (default: False, **must be " +"False for S3**)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:74 +msgid "**enable_async_loading**: Async loading (default: False)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:75 +msgid "**blocking_timeout_secs**: Timeout seconds (default: 10)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:78 +msgid "S3-Specific (in extra_config)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:80 +msgid "**s3_region**: AWS region for S3 client (required)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:81 +msgid "" +"**s3_num_io_threads**: Number of IO threads for the AWS CRT client to " +"spawn. Benefits taper out after exceeding the number of CPU cores. This " +"is also a way to restrict the number of outgoing requests in case your " +"S3-compatible object store has a rate-limiting gateway." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:82 +msgid "" +"**s3_prefer_http2**: Enable HTTP/2 with ALPN negotiation ([\"h2\", " +"\"http/1.1\"])" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:83 +msgid "" +"**s3_enable_s3express**: Enable S3 Express One Zone support in AWS CRT " +"client" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:84 +msgid "" +"**save_chunk_meta**: Whether to save chunk metadata in the object store " +"along with your data (False required for S3)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:85 +msgid "" +"**aws_access_key_id**: AWS access key ID (or log in with `aws configure` " +"in your environment)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:86 +msgid "" +"**aws_secret_access_key**: AWS secret access key (or log in with `aws " +"configure` in your environment)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:88 +msgid "**Tips:**::" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:92 +msgid "Consider S3 Express One Zone for less redundancy but better performance" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:96 +msgid "MP Mode Configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:98 +msgid "" +"In multi-process (MP) mode, S3 is configured as an L2 adapter via a JSON " +"spec passed to the LMCache server, rather than through ``remote_url`` + " +"``extra_config``. Each ``--l2-adapter`` argument takes a JSON object " +"whose ``\"type\": \"s3\"`` field selects the S3 adapter." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:122 +msgid "S3 L2 Adapter Fields" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:124 +msgid "**type** (required): must be ``\"s3\"``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:125 +msgid "" +"**s3_endpoint** (required): bucket URL. Accepts either ``s3://bucket`` or" +" the bare host form (e.g. ``bucket.s3.us-east-1.amazonaws.com``)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:127 +msgid "**s3_region** (required): AWS region for the S3 client." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:128 +msgid "**s3_num_io_threads**: number of CRT IO threads (default ``64``)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:129 +msgid "" +"**s3_prefer_http2**: attempt HTTP/2 via ALPN negotiation (default " +"``true``)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:130 +msgid "" +"**s3_enable_s3express**: enable S3 Express One Zone signing (default " +"``false``)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:131 +msgid "" +"**disable_tls**: bypass TLS, for non-AWS HTTP endpoints (default " +"``false``)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:132 +msgid "" +"**aws_access_key_id**, **aws_secret_access_key**: optional static " +"credentials. When omitted the adapter uses the AWS default credentials " +"chain (``aws configure``, environment variables, IRSA, etc.)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:135 +msgid "" +"**max_capacity_gb**: capacity used by ``get_usage()`` for watermark-based" +" L2 eviction. Set to ``0`` (default) to disable usage tracking — " +"``get_usage()`` then returns ``(-1.0, -1.0)`` and no automatic eviction " +"is triggered." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:138 +msgid "" +"**eviction**: optional sub-dict enabling the L2 eviction controller for " +"this adapter. See :class:`L2AdapterConfigBase` ``_parse_eviction_config``" +" for the full schema. When present, keys that are currently being loaded " +"(reference-counted by the lookup-and-lock path) are skipped by " +"``delete()``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:145 +msgid "Differences vs Non-MP S3" +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:147 +msgid "" +"The MP adapter honors first-class eviction: it implements ``delete()`` " +"(real S3 ``DeleteObject``), refcounted ``submit_unlock``, and " +"``get_usage()`` driven by ``max_capacity_gb``." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:150 +msgid "" +"Keys are identified by ``ObjectKey`` (``model_name`` + ``kv_rank`` + " +"``chunk_hash``) rather than ``CacheEngineKey``. The wire-format object " +"name is ``@@``, which is **not** " +"compatible with the non-MP naming. A bucket populated by non-MP LMCache " +"cannot be read directly by MP LMCache and vice versa." +msgstr "" + +#: ../../source/kv_cache/storage_backends/s3.rst:155 +msgid "Unfull chunks are rejected (same constraint as non-MP)." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/sagemaker_hyperpod.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/sagemaker_hyperpod.po new file mode 100644 index 00000000000..fbaa6786101 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/sagemaker_hyperpod.po @@ -0,0 +1,157 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:2 +msgid "SageMaker Hyperpod" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:5 +msgid "Prerequisites" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:7 +msgid "" +"Create an Amazon SageMaker HyperPod cluster with tiered storage enabled " +"by following the instructions at:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:9 +msgid "" +"https://docs.aws.amazon.com/sagemaker/latest/dg/managed-tier-" +"checkpointing-setup.html" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:11 +msgid "" +"This enables the ai-toolkit daemon that provides shared memory access for" +" LMCache." +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:14 +msgid "Example Configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:24 +msgid "Configuration Parameters" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:27 +msgid "SageMaker Hyperpod-Specific (in extra_config)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:29 +msgid "" +"**sagemaker_hyperpod_bucket**: Bucket name for KV storage namespace " +"(default: \"lmcache\")" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:30 +msgid "" +"**sagemaker_hyperpod_shared_memory_name**: Name of shared memory segment " +"(default: \"shared_memory\"). Set to None to disable shared memory." +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:31 +msgid "" +"**sagemaker_hyperpod_max_concurrent_requests**: Maximum concurrent HTTP " +"requests allowed in-flight at any moment (application-level throttling, " +"default: 100, minimum: 1). This limit is per LMCache engine instance. " +"With multiple workers (e.g., high TP), each worker creates its own engine" +" with separate limits." +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:32 +msgid "" +"**sagemaker_hyperpod_max_connections**: Maximum total TCP connections in " +"the connection pool per LMCache engine across all daemons (default: 256, " +"minimum: 1). For typical single-daemon setups, this effectively limits " +"connections from one engine to one daemon. With N workers per node, total" +" connections to the daemon = N × this value." +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:33 +msgid "" +"**sagemaker_hyperpod_max_connections_per_host**: Maximum TCP connections " +"per LMCache engine to a single daemon address (IP:port) (default: 128, " +"minimum: 1). \"Host\" refers to the daemon's network address, not the " +"client machine. For today's typical single-daemon setup, this has similar" +" effect as max_connections. This parameter enables future multi-daemon " +"configurations where one engine connects to multiple daemons for load " +"balancing. With N workers per node connecting to the same daemon, total " +"connections = N × this value. Reduce proportionally for high TP setups " +"(e.g., set to 16 for 8 workers to achieve ~128 total connections)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:34 +msgid "" +"**sagemaker_hyperpod_timeout_ms**: Timeout for lease acquisition requests" +" in milliseconds (default: 5000, minimum: 100)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:35 +msgid "" +"**sagemaker_hyperpod_lease_ttl_s**: Server-side lease timeout in seconds " +"(default: 30.0)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:36 +msgid "" +"**sagemaker_hyperpod_put_stream_chunk_bytes**: Chunk size for streaming " +"PUT requests in bytes (default: 65536, minimum: 1024)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:37 +msgid "" +"**sagemaker_hyperpod_use_https**: Enable HTTPS instead of HTTP (default: " +"False). **Note**: Ignored if ``remote_url`` already contains ``http://`` " +"or ``https://`` protocol." +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:38 +msgid "" +"**save_chunk_meta**: Whether to save chunk metadata with data (set False " +"for performance)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:41 +msgid "Kubernetes Deployment Requirements" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:44 +msgid "Environment Variable for Node IP" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:46 +msgid "" +"Add the ``NODE_IP`` environment variable to resolve the local node's IP " +"address:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:57 +msgid "/dev/shm Volume Configuration" +msgstr "" + +#: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:59 +msgid "" +"SageMaker Hyperpod requires /dev/shm for high-performance shared memory " +"operations:" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/valkey.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/valkey.po new file mode 100644 index 00000000000..ca886813b97 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/valkey.po @@ -0,0 +1,246 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/valkey.rst:2 +msgid "Valkey" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:5 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:7 +msgid "" +"Valkey is an open source (BSD) high-performance key/value datastore and " +"is a supported option for remote KV Cache offloading in LMCache. Some " +"other remote backends are :doc:`Mooncake <./mooncake>`, :doc:`Redis " +"<./redis>`, and :doc:`InfiniStore <./infinistore>`." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:11 +msgid "Prerequisites" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:13 +msgid "To use this connector, you need valkey-glide 2.0 or higher." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:21 +msgid "Configuration Reference" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:23 +msgid "The following ``extra_config`` keys are supported:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:29 +msgid "Key" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:30 +msgid "Default" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:31 +msgid "Description" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:32 +msgid "``valkey_num_workers``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:33 +msgid "``8``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:34 +msgid "Number of worker threads, each with its own GLIDE client connection." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:35 +msgid "``valkey_mode``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:36 +msgid "``\"standalone\"``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:37 +msgid "" +"``\"standalone\"`` or ``\"cluster\"``. Cluster mode auto-discovers " +"topology from a seed node." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:38 +msgid "``tls_enable``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:39 +msgid "``false``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:40 +msgid "Enable TLS. Required for ElastiCache Serverless." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:41 +msgid "``valkey_username``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:42 +#: ../../source/kv_cache/storage_backends/valkey.rst:45 +msgid "``\"\"``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:43 +msgid "Authentication username." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:44 +msgid "``valkey_password``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:46 +msgid "Authentication password." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:47 +msgid "``valkey_database``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:48 +msgid "None" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:49 +msgid "Database ID (standalone mode only, ignored in cluster mode)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:50 +msgid "``request_timeout``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:51 +msgid "``5.0``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:52 +msgid "" +"GLIDE request timeout in seconds. Also used as the Python-side Future " +"timeout." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:53 +msgid "``connection_timeout``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:54 +msgid "``10.0``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:55 +msgid "GLIDE connection timeout in seconds for initial client connections." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:58 +msgid "Example Configurations" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:61 +msgid "Standalone-mode" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:64 +msgid "Basic Valkey Configuration (Standalone-mode)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:76 +msgid "Standalone-mode Valkey Configuration with database" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:89 +msgid "Cluster-mode" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:92 +msgid "Cluster-mode Valkey Configuration (Endpoint)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:94 +msgid "For example, the configuration endpoint in ElastiCache is as follows:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:96 +msgid "..clustercfg..cache.amazonaws.com" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:98 +msgid "You need to add this DNS name in the ." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:113 +msgid "Cluster-mode Valkey Configuration (Nodes)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:115 +msgid "" +"Nodes are deployed directly and configured in cluster mode without " +"connecting them via DNS names (CNAME)." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:117 +msgid "In this scenario, you simply input multiple IP hosts and ports." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:119 +msgid "Example: 172.0.0.1:7001, 172.0.0.2:7002 ... 172.0.0.6:7006" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:132 +msgid "" +"Cluster-mode Valkey Configuration with numbered databases (Valkey 9.0+ " +"and Valkey-GLIDE 2.1+)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:134 +msgid "" +"Valkey connector supports numbered databases in both the Endpoint using " +"DNS and the Nodes method using IP and port pairs." +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:148 +msgid "TLS / ElastiCache Serverless" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:150 +msgid "ElastiCache Serverless requires TLS. Set ``tls_enable: true``:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:163 +msgid "Performance Tuning" +msgstr "" + +#: ../../source/kv_cache/storage_backends/valkey.rst:165 +msgid "" +"For large models (e.g., 70B with TP=8), increase the worker count for " +"higher throughput:" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/weka.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/weka.po new file mode 100644 index 00000000000..a4813b3efe5 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/weka.po @@ -0,0 +1,132 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache/storage_backends/weka.rst:2 +msgid "Weka" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:7 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:9 +msgid "" +"WekaFS is a high-performance, distributed filesystem and is a supported " +"option for KV Cache offloading in LMCache. Even though the local " +"filesystem backend can work with a WekaFS mount, this particular backend " +"is optimized for Weka's characteristics. It leverages GPUDirect Storage " +"for I/O and it allows data-sharing between multiple LMCache instances." +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:15 +msgid "Ways to configure LMCache WEKA Offloading" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:17 +msgid "**1. Environment Variables:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:33 +msgid "**2. Configuration File**:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:35 +msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:37 +msgid "Example ``config.yaml``:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:54 +msgid "GDS Buffer Size Explanation" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:56 +msgid "" +"The backend currently pre-registers buffer space to speed up GDS " +"operations. This buffer space is registered in VRAM so options like " +"``--gpu-memory-utilization`` from ``vllm`` should be considered when " +"setting it. For example, a good rule of thumb for H100 which generally " +"has 80GiBs of VRAM would be to start with 8GiB and set ``--gpu-memory-" +"utilization 0.85`` and depending on your workflow fine-tune it from " +"there." +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:64 +msgid "Setup Example" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:68 +msgid "**Prerequisites:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:70 +msgid "" +"A Machine with at least one GPU. You can adjust the max model length of " +"your vllm instance depending on your GPU memory." +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:72 +msgid "Weka already installed and mounted." +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:74 +msgid "" +"vllm and lmcache installed (:doc:`Installation Guide " +"<../../getting_started/installation>`)" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:76 +msgid "Hugging Face access to ``meta-llama/Llama-3.1-8B-Instruct``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:82 +msgid "**Step 1. Create cache directory under your Weka mount:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:84 +msgid "To find all your WekaFS mounts run:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:90 +msgid "For the sake of this example let's say that the above returns:" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:96 +msgid "Then create a directory under it (the name here is arbitrary):" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:102 +msgid "**Step 2. Start a vLLM server with Weka offloading enabled:**" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:104 +msgid "Create a an lmcache configuration file called: ``weka-offload.yaml``" +msgstr "" + +#: ../../source/kv_cache/storage_backends/weka.rst:115 +msgid "" +"If you don't want to use a config file, uncomment the first three " +"environment variables and then comment out the ``LMCACHE_CONFIG_FILE`` " +"below:" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/check_finish.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/check_finish.po new file mode 100644 index 00000000000..c95ce151b41 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/check_finish.po @@ -0,0 +1,29 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_management/check_finish.rst:4 +msgid "Check finish of a control event" +msgstr "" + +#: ../../source/kv_cache_management/check_finish.rst:6 +msgid "Coming soon..." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/clear.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/clear.po new file mode 100644 index 00000000000..30408c83f55 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/clear.po @@ -0,0 +1,79 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_management/clear.rst:4 +msgid "Clear the KV cache" +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:6 +msgid "The ``clear`` interface is defined as the following:" +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:12 +msgid "" +"The function removes the KV cache stored at ``location`` for the " +"specified ``instance_id``. It returns an ``event_id`` and the number of " +"tokens scheduled for clearing." +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:17 +msgid "Example usage:" +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:19 +msgid "" +"First, create a yaml file ``example.yaml`` to configure the lmcache " +"instance:" +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:37 +msgid "Start the vllm/lmcache instance at port 8000:" +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:44 +msgid "Start the lmcache controller at port 9000 and the monitor at port 9001:" +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:50 +msgid "Send a request to vllm:" +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:62 +msgid "Clear the KV cache in the system:" +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:74 +msgid "The controller responds with a message similar to:" +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:80 +msgid "" +"This indicates that the KV cache for 12 tokens has been scheduled for " +"clearing. We can verify the cache has been cleared by performing a " +"lookup:" +msgstr "" + +#: ../../source/kv_cache_management/clear.rst:91 +msgid "" +"The lookup should return an empty result, confirming that the KV cache " +"has been cleared for the given tokens." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/compress.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/compress.po new file mode 100644 index 00000000000..6c50ee49847 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/compress.po @@ -0,0 +1,94 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_management/compress.rst:4 +msgid "Compress and Decompress the KV cache" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:6 +msgid "The ``compress`` interface is defined as the following:" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:13 +msgid "" +"These 2 functions compresses/decompresses the KV cache chunks specified " +"by ``tokens`` using the given ``method`` in the storage ``location``. The" +" controller returns an ``event_id`` and the number of tokens scheduled " +"for compression or decompression." +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:17 +msgid "Example usage:" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:19 +msgid "" +"First, we need a yaml file ``example.yaml`` to properly configure the " +"lmcache instance:" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:37 +msgid "Second, we need to start the vllm/lmcache instance at port 8000:" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:43 +msgid "" +"Third, we need to start the lmcache controller at port 9000 and the " +"monitor at port 9001:" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:49 +msgid "Then we can send a request to vllm to see if it works properly:" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:61 +msgid "Now we send a request to tokenize the prompt:" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:72 +msgid "We should be able to see token ids in response:" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:78 +msgid "After all, we issue a ``compress`` request:" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:91 +#: ../../source/kv_cache_management/compress.rst:112 +msgid "The controller responds with a message similar to:" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:97 +msgid "" +"This indicates that 12 tokens are being compressed. The ``event_id`` can " +"be used to query the status of the operation." +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:99 +msgid "Once the kv cache is compressed, we can use cachegen to decompress" +msgstr "" + +#: ../../source/kv_cache_management/compress.rst:118 +msgid "" +"This indicates that 12 tokens are being decompressed. The ``event_id`` " +"can be used to query the status of the operation." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/health.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/health.po new file mode 100644 index 00000000000..60da02dcf7e --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/health.po @@ -0,0 +1,60 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_management/health.rst:4 +msgid "Check controller health" +msgstr "" + +#: ../../source/kv_cache_management/health.rst:6 +msgid "The ``health`` interface is defined as the following:" +msgstr "" + +#: ../../source/kv_cache_management/health.rst:12 +msgid "" +"The function returns an ``event_id`` and a dictionary mapping " +"``worker_id`` to ``error_code``. A value of ``0`` indicates a healthy " +"worker, while a non-zero value signals an error." +msgstr "" + +#: ../../source/kv_cache_management/health.rst:17 +msgid "Example usage:" +msgstr "" + +#: ../../source/kv_cache_management/health.rst:19 +msgid "" +"First, start the lmcache controller at port 9000 and the monitor at port " +"9001:" +msgstr "" + +#: ../../source/kv_cache_management/health.rst:25 +msgid "Then send a health check request:" +msgstr "" + +#: ../../source/kv_cache_management/health.rst:33 +msgid "The controller responds with a message similar to the following:" +msgstr "" + +#: ../../source/kv_cache_management/health.rst:39 +msgid "" +"Here ``error_codes`` lists each worker's ``error_code``. ``0`` represents" +" a healthy worker, while non-zero values indicate an error." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/index.po new file mode 100644 index 00000000000..7fbe7d24754 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/index.po @@ -0,0 +1,214 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_management/index.rst:2 +msgid "LMCache Controller" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:5 +msgid "Overview" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:6 +msgid "" +"The overall architecture of the LMCache Controller is shown in the " +"figure, mainly consisting of two parts: the Controller Manager and " +"LMCache Worker." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:9 +msgid "" +"The Controller Manager mainly consists of KV Controller, Reg Controller, " +"and Cluster Executor." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:11 +msgid "" +"KV Controller: The KV Controller handles the chunk information reported " +"by LMCache Workers, and lookup requests query chunk information from the " +"KV Controller." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:12 +msgid "" +"Reg Controller: The Reg Controller is responsible for handling " +"register/deregister/heartbeat requests from LMCache Workers." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:13 +msgid "" +"Cluster Executor: When the Controller Manager receives user requests, " +"such as Clear or Move, it sends the corresponding commands to LMCache " +"Workers through the Cluster Executor." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:15 +msgid "" +"The LMCache Worker is a thread within a rank process, which is " +"responsible for the following tasks:" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:17 +msgid "sends register, deregister, heartbeat to the Reg Controller." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:18 +msgid "" +"send chunk information to the KV Controller, which include admit and " +"evict message." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:19 +msgid "" +"listens on a port to receive commands from the Cluster Executor and " +"performs corresponding processing." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:21 +msgid "LMCache Controller Architecture Diagram" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:25 +msgid "P2P Related" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:27 +msgid "" +"If ``enable_p2p`` is enabled, the LMCache Controller must also be " +"enabled. The LMCache Controller serves as the central node and stores " +"information for each chunk. The ``P2PBackend`` queries chunk information " +"from the LMCache Controller and performs data transmission through NIXL." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:33 +msgid "Key Features" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:35 +msgid "Exposes a set of APIs for users and orchestrators to manage the KV cache." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:37 +msgid "Currently, the controller provides the following APIs:" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:39 +msgid ":ref:`Clear `: Clear the KV caches." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:40 +msgid ":ref:`Compress `: Compress the KV cache." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:41 +msgid ":ref:`Health `: Check the health status of cache workers." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:42 +msgid ":ref:`Lookup `: Lookup the KV cache for a given list of tokens." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:43 +msgid ":ref:`Move `: Move the KV cache to a different location." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:44 +msgid ":ref:`Pin `: Persist the KV cache to prevent it from being evicted." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:45 +msgid "" +":ref:`CheckFinish `: Check whether a (non-blocking) control" +" event has finished or not." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:46 +msgid ":ref:`QueryWorkerInfo `: Query the worker info." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:48 +msgid "Interacts with the LMCache worker." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:50 +msgid "Currently, the LMCache worker supports the following functions:" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:52 +msgid "register with the controller" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:53 +msgid "deregister from the controller" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:54 +msgid "heartbeat" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:55 +msgid "admit or evict chunk information(LocalCPUBackend or LocalDiskBackend)" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:59 +msgid "Quick Start" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:61 +msgid "**Start the Controller**" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:67 +msgid "Expected output:" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:82 +msgid "**Controller Configuration**" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:84 +msgid "--host: default is 0.0.0.0" +msgstr "" + +#: ../../source/kv_cache_management/index.rst:85 +msgid "" +"--port: default is 9000, the externally exposed port through which " +"interfaces like lookup can be accessed via this port." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:86 +msgid "" +"--monitor-port: default is 9001, the port through which LMCache Worker " +"communicates with Controller Manager (deprecated, indicates the pull port" +" in --monitor-ports, reply port is None)." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:87 +#, python-brace-format +msgid "" +"--monitor-ports: default is None, if configured, requires a JSON format " +"string input such as ``{\"pull\": 8300, \"reply\": 8400}``." +msgstr "" + +#: ../../source/kv_cache_management/index.rst:89 +msgid "**YAML Configuration**" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/lookup.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/lookup.po new file mode 100644 index 00000000000..c8c59154996 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/lookup.po @@ -0,0 +1,85 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_management/lookup.rst:4 +msgid "Lookup the KV cache" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:6 +msgid "The ``lookup`` interface is defined as the following:" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:12 +msgid "" +"The function takes a list of tokens as input and returns event_id and a " +"dictionary containing the layout information for each token. The layout " +"information is represented as a mapping between ``instance_id`` and a " +"tuple of ``(location, matched_prefix_length)``." +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:16 +msgid "Example usage:" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:18 +msgid "" +"First, we need a yaml file ``example.yaml`` to properly configure the " +"lmcache instance:" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:36 +msgid "Second, we need to start the vllm/lmcache instance at port 8000:" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:42 +msgid "" +"Third, we need to start the lmcache controller at port 9000 and the " +"monitor at port 9001:" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:48 +msgid "Then we can send a request to vllm to see if it works properly:" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:60 +msgid "Now we send a request to tokenize the prompt:" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:71 +msgid "We should be able to see token ids in response:" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:77 +msgid "Finally, we can send a ``lookup`` request to the lmcache controller:" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:87 +msgid "We should be able to see the response like this:" +msgstr "" + +#: ../../source/kv_cache_management/lookup.rst:93 +msgid "" +"The field ``lmcache_default_instance`` shows the instance ID, followed by" +" a tuple of ``(location, matched_prefix_length)`` indicating the cache " +"location within that instance and matched prefix length. ``event_id`` is " +"an identifier of the controller operation and can typically be ignored." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/move.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/move.po new file mode 100644 index 00000000000..043ddc6c4d5 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/move.po @@ -0,0 +1,85 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_management/move.rst:4 +msgid "Move the KV cache" +msgstr "" + +#: ../../source/kv_cache_management/move.rst:6 +msgid "The ``move`` interface is defined as the following:" +msgstr "" + +#: ../../source/kv_cache_management/move.rst:13 +msgid "" +"The function moves the KV cache chunks identified by ``tokens`` from " +"``old_position`` to ``new_position``. Each position is a tuple of " +"``(instance_id, location)``. Setting ``copy`` to ``True`` copies the KV " +"cache instead of moving it." +msgstr "" + +#: ../../source/kv_cache_management/move.rst:18 +msgid "" +"Note that NIXL is required to be installed for P2P transfer. We'll " +"support other transports later such as Python socket and Mooncake." +msgstr "" + +#: ../../source/kv_cache_management/move.rst:22 +msgid "Example usage:" +msgstr "" + +#: ../../source/kv_cache_management/move.rst:24 +msgid "" +"First, prepare two yaml files ``instance1.yaml`` and ``instance2.yaml`` " +"to configure two lmcache instances:" +msgstr "" + +#: ../../source/kv_cache_management/move.rst:70 +msgid "Start two vllm engines:" +msgstr "" + +#: ../../source/kv_cache_management/move.rst:82 +msgid "Start the lmcache controller at port 9000 and the monitor at port 9001:" +msgstr "" + +#: ../../source/kv_cache_management/move.rst:88 +msgid "Send a request to vllm engine 1:" +msgstr "" + +#: ../../source/kv_cache_management/move.rst:100 +msgid "Tokenize the prompt to obtain token ids:" +msgstr "" + +#: ../../source/kv_cache_management/move.rst:111 +msgid "" +"Move the KV cache from engine 1's CPU to engine 2's CPU using the token " +"ids:" +msgstr "" + +#: ../../source/kv_cache_management/move.rst:123 +msgid "The controller responds with a message similar to:" +msgstr "" + +#: ../../source/kv_cache_management/move.rst:129 +msgid "" +"``num_tokens`` indicates how many tokens' KV cache are being moved. The " +"returned ``event_id`` can be used to query the status of the operation." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/pin.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/pin.po new file mode 100644 index 00000000000..44902c1dd6c --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/pin.po @@ -0,0 +1,76 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_management/pin.rst:4 +msgid "Pin the KV cache" +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:6 +msgid "The ``pin`` interface is defined as the following:" +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:12 +msgid "" +"The function pins (persists) the KV cache chunks specified by ``tokens`` " +"in the given ``location`` of the ``instance_id``. The controller returns " +"an ``event_id`` and the number of tokens scheduled for pinning." +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:17 +msgid "Example usage:" +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:19 +msgid "" +"First, create a yaml file ``example.yaml`` to configure the lmcache " +"instance:" +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:38 +msgid "Start the vllm/lmcache instance at port 8000:" +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:45 +msgid "Start the lmcache controller at port 9000 and the monitor at port 9001:" +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:51 +msgid "Send a request to vllm:" +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:63 +msgid "Tokenize the prompt to obtain token ids:" +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:74 +msgid "Pin the KV cache in the system:" +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:86 +msgid "The controller responds with a message similar to:" +msgstr "" + +#: ../../source/kv_cache_management/pin.rst:92 +msgid "" +"``num_tokens`` indicates how many tokens' KV cache are pinned. The " +"returned ``event_id`` can be used to query the status of the operation." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/query_worker_info.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/query_worker_info.po new file mode 100644 index 00000000000..c405fcfb04a --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/query_worker_info.po @@ -0,0 +1,68 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_management/query_worker_info.rst:4 +msgid "Query Worker Info" +msgstr "" + +#: ../../source/kv_cache_management/query_worker_info.rst:6 +msgid "The ``query_worker_info`` interface is defined as the following:" +msgstr "" + +#: ../../source/kv_cache_management/query_worker_info.rst:12 +msgid "" +"The function get the info of the workers which specified by " +"``instance_id`` and ``worker_ids``. The controller returns an " +"``event_id`` and the worker infos." +msgstr "" + +#: ../../source/kv_cache_management/query_worker_info.rst:16 +msgid "Example usage:" +msgstr "" + +#: ../../source/kv_cache_management/query_worker_info.rst:18 +msgid "" +"First, create a yaml file ``example.yaml`` to configure the lmcache " +"instance:" +msgstr "" + +#: ../../source/kv_cache_management/query_worker_info.rst:37 +msgid "Start the vllm/lmcache instance at port 8000:" +msgstr "" + +#: ../../source/kv_cache_management/query_worker_info.rst:44 +msgid "Start the lmcache controller at port 9000 and the monitor at port 9001:" +msgstr "" + +#: ../../source/kv_cache_management/query_worker_info.rst:50 +msgid "Send a request to controller:" +msgstr "" + +#: ../../source/kv_cache_management/query_worker_info.rst:61 +msgid "The controller responds with a message similar to:" +msgstr "" + +#: ../../source/kv_cache_management/query_worker_info.rst:67 +msgid "" +"``worker_infos`` contains the queried worker information. returned " +"``event_id`` can be used to query the status of the operation." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/blending.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/blending.po new file mode 100644 index 00000000000..38ca824a692 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/blending.po @@ -0,0 +1,75 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_optimizations/blending.rst:2 +msgid "Blending" +msgstr "" + +#: ../../source/kv_cache_optimizations/blending.rst:4 +msgid "" +"CacheBlend enables KV cache reuse for non-prefix positions by recomputing" +" a subset of tokens at non-prefix positions. For example, CacheBlend can " +"combine multiple (pre-)computed KV caches, when their corresponding texts" +" are concatenated in the LLM input" +msgstr "" + +#: ../../source/kv_cache_optimizations/blending.rst:8 +msgid "Configuring CacheBlend in RAG scenarios" +msgstr "" + +#: ../../source/kv_cache_optimizations/blending.rst:10 +msgid "" +"Here, we will explain the code in our end-to-end `example " +"`_>." +msgstr "" + +#: ../../source/kv_cache_optimizations/blending.rst:12 +msgid "Below are some blending-related configurations (and explanations):" +msgstr "" + +#: ../../source/kv_cache_optimizations/blending.rst:37 +msgid "" +"Firstly, we preprocess texts into tokens, as tokenizing a concatenated " +"string may produce different tokens than concatenating the results of " +"tokenizing each string individually. For example, assume we have a system" +" prompt and three text chunks. We need to preprocess them into tokens " +"before sending to the LLM:" +msgstr "" + +#: ../../source/kv_cache_optimizations/blending.rst:59 +msgid "" +"Then, we can send the tokenized prompt to vLLM. Meanwhile, LMCache will " +"store the KV caches of different chunks according to the " +"``BLEND_SPECIAL_STR``." +msgstr "" + +#: ../../source/kv_cache_optimizations/blending.rst:65 +msgid "" +"Similarly, we build another prompt using the same chunks but with " +"different orders." +msgstr "" + +#: ../../source/kv_cache_optimizations/blending.rst:82 +msgid "" +"Even though the second prompt has a different order of chunks, LMCache " +"can still reuse the KV caches of chunk1, chunk2, and chunk3." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/cachegen.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/cachegen.po new file mode 100644 index 00000000000..104f559e68a --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/cachegen.po @@ -0,0 +1,53 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_optimizations/compression/cachegen.rst:4 +msgid "CacheGen" +msgstr "" + +#: ../../source/kv_cache_optimizations/compression/cachegen.rst:6 +msgid "" +"Cachegen leverages KV cache's distributional properties to encode a KV " +"cache into more compact bitstream representations with negligible " +"decoding overhead." +msgstr "" + +#: ../../source/kv_cache_optimizations/compression/cachegen.rst:10 +msgid "Configuring CacheGen in LMCache" +msgstr "" + +#: ../../source/kv_cache_optimizations/compression/cachegen.rst:12 +msgid "" +"The settings should be very similar to :ref:`naive KV cache sharing " +"`. Only minor configurations need to be done to enable " +"CacheGen." +msgstr "" + +#: ../../source/kv_cache_optimizations/compression/cachegen.rst:15 +msgid "To enable CacheGen in offline inference, we need to set:" +msgstr "" + +#: ../../source/kv_cache_optimizations/compression/cachegen.rst:22 +msgid "" +"To enable CacheGen in online inference, we need to set the " +"``remote_serde`` in the configuration yaml:" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/index.po new file mode 100644 index 00000000000..e4d65cccc18 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/index.po @@ -0,0 +1,39 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_optimizations/compression/index.rst:2 +msgid "Compression" +msgstr "" + +#: ../../source/kv_cache_optimizations/compression/index.rst:4 +msgid "" +"KV cache compression can greatly reduces the size of the cache, which can" +" be beneficial for both storage/memory usage and loading speed. " +"Currently, we support the following compression algorithms:" +msgstr "" + +#: ../../source/kv_cache_optimizations/compression/index.rst:7 +msgid "" +":ref:`CacheGen `: `CacheGen: KV Cache Compression and Streaming" +" for Fast Large Language Model Serving " +"`_" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/layerwise.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/layerwise.po new file mode 100644 index 00000000000..f8a96a41898 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/layerwise.po @@ -0,0 +1,299 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/kv_cache_optimizations/layerwise.rst:2 +msgid "Layerwise KV Transfer" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:4 +msgid "" +"The storage and loading of KV Cache on a layer granularity is a key " +"optimization that allows for forward pass to \"stagger\" through its " +"computation as each layer's KV Cache is received instead of only waiting " +"to begin after the entire loading" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:6 +msgid "" +"CacheBlend is implemented on top of the layerwise codepath in order to " +"pipeline recompute and loading to mask the latency of loading KV Cache." +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:8 +msgid "Basic Codepath" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:25 +msgid "Architecture Overview" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:27 +msgid "**CacheEngine**" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:28 +msgid "The main orchestrator containing two primary generators:" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:30 +msgid "" +"**Retrieval Generator** (N + 2 yields): Handles layer-by-layer KV cache " +"loading with on-demand memory allocation" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:31 +msgid "" +"**Storage Generator** (N + 1 yields): Manages layer-by-layer KV cache " +"saving with upfront CPU memory allocation" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:33 +msgid "**LayerwiseGPUConnector**" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:34 +msgid "Manages GPU-CPU memory transfers with dedicated CUDA streams:" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:36 +msgid "" +"**Load GPU Buffer**: Temporary GPU memory for CPU→GPU transfers " +"(``use_gpu: true``)" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:37 +msgid "" +"**Store GPU Buffer**: Temporary GPU memory for GPU→CPU transfers " +"(``use_gpu: true``)" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:38 +msgid "" +"**Nested Generators**: ``batched_to_gpu()`` and ``batched_from_gpu()`` " +"handle actual memory operations" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:40 +msgid "**StorageManager**" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:41 +msgid "Handles persistent storage operations:" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:43 +msgid "" +"``layerwise_batched_get()``: Asynchronous retrieval with ``.result()`` " +"for request-level concurrency" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:44 +msgid "``batched_put()``: Stores memory objects to persistent backends" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:47 +msgid "Execution Flow" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:49 +msgid "The layerwise pipeline follows a numbered execution sequence:" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:51 +msgid "**1. start_load_kv()**" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:52 +msgid "Initializes Retrieval Generator via ``lmcache_engine.retrieve_layer()``" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:53 +msgid "Performs setup (1st ``next()``) and loads layer 0 (2nd ``next()``)" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:54 +msgid "Creates ``layerwise_retrievers`` list for ongoing layer processing" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:56 +msgid "**2. wait_for_layer_load()** (repeated for each layer)" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:57 +msgid "Advances Retrieval Generator via ``next()`` to process layer i" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:58 +msgid "" +"Triggers ``StorageManager.layerwise_batched_get()`` for async cache " +"retrieval" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:59 +msgid "" +"Calls GPU Load Generator's ``batched_to_gpu()`` to transfer memory " +"objects to GPU" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:60 +msgid "" +"**Last request in batch**: Synchronizes " +"``current_stream.wait_stream(load_stream)``" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:62 +msgid "**3. save_kv_layer()** (repeated for each layer)" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:63 +msgid "" +"**First call only**: Creates Storage Generator with upfront CPU memory " +"allocation" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:64 +msgid "Advances Storage Generator via ``next()`` to process layer i" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:65 +msgid "" +"Calls GPU Store Generator's ``batched_from_gpu()`` to transfer GPU data " +"to CPU" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:66 +msgid "" +"**First request in batch**: Synchronizes " +"``store_stream.wait_stream(current_stream)``" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:68 +msgid "**4. wait_for_save()**" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:69 +msgid "Finalizes Storage Generator with last ``next()`` call" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:70 +msgid "Completes all ``StorageManager.batched_put()`` operations" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:71 +msgid "Performs GPU Store Generator cleanup" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:74 +msgid "Key Optimizations" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:76 +msgid "**Pipelined Memory Operations**" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:77 +msgid "The system overlaps layer N+1 computation with layer N storage." +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:79 +msgid "**Stream Synchronization**" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:80 +msgid "Three CUDA streams coordinate operations:" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:82 +msgid "``current_stream``: vLLM's forward pass computation" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:83 +msgid "``load_stream``: KV cache loading operations" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:84 +msgid "``store_stream``: KV cache storing operations" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:86 +msgid "**Batch-Level Coordination**" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:87 +msgid "Multiple requests are processed together with specialized synchronization:" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:89 +msgid "" +"**First request**: Provides store stream synchronization to prevent GPU " +"buffer corruption" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:90 +msgid "" +"**Last request**: Provides load stream synchronization to ensure KV cache" +" availability" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:92 +msgid "**Memory Allocation Strategies**" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:93 +msgid "**Retrieval**: Layer-by-layer allocation" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:94 +msgid "**Storage**: Upfront allocation for all layers" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:96 +msgid "**Cache Key Management**" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:97 +msgid "" +"Multi-layer cache engine keys use ``split_layers(N)`` to create per-layer" +" kubernetes_deployment" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:100 +msgid "Configuration" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:102 +msgid "Enable layerwise caching by setting:" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:108 +msgid "" +"The system automatically selects appropriate layerwise GPU connectors " +"based on configuration:" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:110 +msgid "``VLLMPagedMemLayerwiseGPUConnector``: For standard layerwise operations" +msgstr "" + +#: ../../source/kv_cache_optimizations/layerwise.rst:111 +msgid "``VLLMBufferLayerwiseGPUConnector``: When blending is enabled" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/architecture.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/architecture.po new file mode 100644 index 00000000000..41f0e04fb4f --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/architecture.po @@ -0,0 +1,792 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/architecture.rst:2 +msgid "Architecture & Developer Guide" +msgstr "" + +#: ../../source/mp/architecture.rst:4 +msgid "" +"This page describes the internal architecture of LMCache multiprocess " +"mode. It is aimed at developers who want to understand, debug, or extend " +"the system." +msgstr "" + +#: ../../source/mp/architecture.rst:12 +msgid "High-Level Architecture" +msgstr "" + +#: ../../source/mp/architecture.rst:43 +msgid "Server Variants" +msgstr "" + +#: ../../source/mp/architecture.rst:45 +msgid "" +"All three server entry points share the same ``MPCacheEngine`` and " +"``StorageManager`` core." +msgstr "" + +#: ../../source/mp/architecture.rst:48 +msgid "" +"**``server.py``** -- The default ZMQ-only server. Creates an " +"``MPCacheEngine`` and a ``MessageQueueServer``, registers handlers for " +"all core ``RequestType`` values, and blocks in a keep-alive loop." +msgstr "" + +#: ../../source/mp/architecture.rst:52 +msgid "" +"**``blend_server_v2.py``** -- Extends ``MPCacheEngine`` with " +"``BlendEngineV2``, which adds CacheBlend operations " +"(``CB_REGISTER_KV_CACHE``, ``CB_LOOKUP_PRE_COMPUTED``, " +"``CB_STORE_PRE_COMPUTED``, ``CB_RETRIEVE_PRE_COMPUTED``, " +"``CB_STORE_FINAL``). Enables non-prefix KV cache reuse across document " +"paragraphs." +msgstr "" + +#: ../../source/mp/architecture.rst:58 +msgid "" +"**``http_server.py``** -- Wraps ``run_cache_server()`` (from " +"``server.py``) inside a FastAPI application. Endpoints are contributed " +"by modules under ``http_apis/`` and auto-registered via " +"``HTTPAPIRegistry``: ``GET /`` (basic liveness), ``GET /healthcheck`` for" +" Kubernetes probes, ``POST /clear-cache`` for clearing all KV cache data " +"in L1 (CPU) memory, and ``GET /status`` for inspecting detailed internal " +"state. The ZMQ server runs as part of the same process, and any " +"configured runtime plugins are spawned by ``MPRuntimePluginLauncher`` " +"during FastAPI startup." +msgstr "" + +#: ../../source/mp/architecture.rst:68 +msgid "ZMQ Protocol" +msgstr "" + +#: ../../source/mp/architecture.rst:70 +msgid "Communication between vLLM and LMCache uses ZMQ (DEALER/ROUTER pattern)." +msgstr "" + +#: ../../source/mp/architecture.rst:72 +msgid "**RequestType enum** (defined in ``protocols/base.py``):" +msgstr "" + +#: ../../source/mp/architecture.rst:78 +msgid "Request Type" +msgstr "" + +#: ../../source/mp/architecture.rst:79 +msgid "Handler Type" +msgstr "" + +#: ../../source/mp/architecture.rst:80 +msgid "Description" +msgstr "" + +#: ../../source/mp/architecture.rst:81 +msgid "``REGISTER_KV_CACHE``" +msgstr "" + +#: ../../source/mp/architecture.rst:82 ../../source/mp/architecture.rst:85 +#: ../../source/mp/architecture.rst:117 ../../source/mp/architecture.rst:127 +#: ../../source/mp/architecture.rst:130 ../../source/mp/architecture.rst:133 +msgid "SYNC" +msgstr "" + +#: ../../source/mp/architecture.rst:83 +msgid "Register GPU KV cache tensors for a vLLM instance." +msgstr "" + +#: ../../source/mp/architecture.rst:84 +msgid "``UNREGISTER_KV_CACHE``" +msgstr "" + +#: ../../source/mp/architecture.rst:86 +msgid "Unregister KV cache tensors." +msgstr "" + +#: ../../source/mp/architecture.rst:87 +msgid "``STORE``" +msgstr "" + +#: ../../source/mp/architecture.rst:88 ../../source/mp/architecture.rst:91 +#: ../../source/mp/architecture.rst:94 ../../source/mp/architecture.rst:98 +#: ../../source/mp/architecture.rst:102 ../../source/mp/architecture.rst:107 +#: ../../source/mp/architecture.rst:111 ../../source/mp/architecture.rst:114 +#: ../../source/mp/architecture.rst:120 ../../source/mp/architecture.rst:123 +#: ../../source/mp/architecture.rst:136 ../../source/mp/architecture.rst:139 +#: ../../source/mp/architecture.rst:142 ../../source/mp/architecture.rst:145 +#: ../../source/mp/architecture.rst:148 ../../source/mp/architecture.rst:153 +msgid "BLOCKING" +msgstr "" + +#: ../../source/mp/architecture.rst:89 +msgid "Store KV cache chunks from GPU to L1 (CPU)." +msgstr "" + +#: ../../source/mp/architecture.rst:90 +msgid "``RETRIEVE``" +msgstr "" + +#: ../../source/mp/architecture.rst:92 +msgid "Copy KV cache chunks from L1 (CPU) back to GPU." +msgstr "" + +#: ../../source/mp/architecture.rst:93 +msgid "``LOOKUP``" +msgstr "" + +#: ../../source/mp/architecture.rst:95 +msgid "" +"Submit a prefix lookup; the prefetch job is tracked server-side by " +"request_id." +msgstr "" + +#: ../../source/mp/architecture.rst:97 +msgid "``QUERY_PREFETCH_STATUS``" +msgstr "" + +#: ../../source/mp/architecture.rst:99 +msgid "" +"Poll a prefetch job by request_id. Returns the loaded chunk count when " +"done, or ``None`` while the prefetch is still in progress." +msgstr "" + +#: ../../source/mp/architecture.rst:101 +msgid "``QUERY_PREFETCH_LOOKUP_HITS``" +msgstr "" + +#: ../../source/mp/architecture.rst:103 +msgid "" +"Query the lookup-phase hit chunk count by request_id, before the prefetch" +" finishes. Returns ``None`` while the lookup is still running." +msgstr "" + +#: ../../source/mp/architecture.rst:106 +msgid "``FREE_LOOKUP_LOCKS``" +msgstr "" + +#: ../../source/mp/architecture.rst:108 +msgid "Release read locks from a cancelled lookup without doing a full RETRIEVE." +msgstr "" + +#: ../../source/mp/architecture.rst:110 +msgid "``END_SESSION``" +msgstr "" + +#: ../../source/mp/architecture.rst:112 +msgid "Remove session state for a finished request." +msgstr "" + +#: ../../source/mp/architecture.rst:113 +msgid "``CLEAR``" +msgstr "" + +#: ../../source/mp/architecture.rst:115 +msgid "Clear all cached data." +msgstr "" + +#: ../../source/mp/architecture.rst:116 +msgid "``GET_CHUNK_SIZE``" +msgstr "" + +#: ../../source/mp/architecture.rst:118 +msgid "Return the server's chunk size." +msgstr "" + +#: ../../source/mp/architecture.rst:119 +msgid "``PING``" +msgstr "" + +#: ../../source/mp/architecture.rst:121 +msgid "Liveness ping; the handler always returns ``True``." +msgstr "" + +#: ../../source/mp/architecture.rst:122 +msgid "``REPORT_BLOCK_ALLOCATION``" +msgstr "" + +#: ../../source/mp/architecture.rst:124 +msgid "" +"Fire-and-forget channel for the vLLM scheduler to report GPU block " +"allocation events to the observability subsystem." +msgstr "" + +#: ../../source/mp/architecture.rst:126 +msgid "``NOOP``" +msgstr "" + +#: ../../source/mp/architecture.rst:128 +msgid "Debug heartbeat -- returns a confirmation string." +msgstr "" + +#: ../../source/mp/architecture.rst:129 +msgid "``CB_REGISTER_KV_CACHE``" +msgstr "" + +#: ../../source/mp/architecture.rst:131 +msgid "(Blend) Register CacheBlend KV buffer." +msgstr "" + +#: ../../source/mp/architecture.rst:132 +msgid "``CB_UNREGISTER_KV_CACHE``" +msgstr "" + +#: ../../source/mp/architecture.rst:134 +msgid "(Blend) Unregister CacheBlend KV buffer." +msgstr "" + +#: ../../source/mp/architecture.rst:135 +msgid "``CB_STORE_PRE_COMPUTED``" +msgstr "" + +#: ../../source/mp/architecture.rst:137 +msgid "(Blend) Store pre-computed paragraph chunks." +msgstr "" + +#: ../../source/mp/architecture.rst:138 +msgid "``CB_LOOKUP_PRE_COMPUTED``" +msgstr "" + +#: ../../source/mp/architecture.rst:140 +msgid "(Blend) Lookup pre-computed paragraph chunks." +msgstr "" + +#: ../../source/mp/architecture.rst:141 +msgid "``CB_RETRIEVE_PRE_COMPUTED``" +msgstr "" + +#: ../../source/mp/architecture.rst:143 +msgid "(Blend) Retrieve pre-computed paragraph chunks to GPU." +msgstr "" + +#: ../../source/mp/architecture.rst:144 +msgid "``CB_STORE_FINAL``" +msgstr "" + +#: ../../source/mp/architecture.rst:146 +msgid "(Blend) Store final blended chunks." +msgstr "" + +#: ../../source/mp/architecture.rst:147 +msgid "``CB_LOOKUP_PRE_COMPUTED_V2``" +msgstr "" + +#: ../../source/mp/architecture.rst:149 +msgid "" +"(Blend V2) Lookup pre-computed chunks; returns ``CBMatchResult`` entries " +"(with old/cur ranges and per-chunk hashes) so the retrieve step can skip " +"re-hashing." +msgstr "" + +#: ../../source/mp/architecture.rst:152 +msgid "``CB_RETRIEVE_PRE_COMPUTED_V2``" +msgstr "" + +#: ../../source/mp/architecture.rst:154 +msgid "" +"(Blend V2) Retrieve pre-computed chunks using the ``CBMatchResult`` list " +"returned by ``CB_LOOKUP_PRE_COMPUTED_V2``." +msgstr "" + +#: ../../source/mp/architecture.rst:157 +msgid "**Handler types:**" +msgstr "" + +#: ../../source/mp/architecture.rst:159 +msgid "**SYNC** -- Runs directly in the ZMQ main loop (fast, non-blocking)." +msgstr "" + +#: ../../source/mp/architecture.rst:160 +msgid "" +"**BLOCKING** -- Dispatched to a thread pool (may involve GPU copies or " +"I/O)." +msgstr "" + +#: ../../source/mp/architecture.rst:163 +msgid "Config System" +msgstr "" + +#: ../../source/mp/architecture.rst:165 +msgid "Each config module exposes a composable triple:" +msgstr "" + +#: ../../source/mp/architecture.rst:171 +msgid "``server.py:parse_args()`` composes them:" +msgstr "" + +#: ../../source/mp/architecture.rst:184 +msgid "" +"Both ``blend_server_v2.py`` and ``http_server.py`` reuse this pattern, " +"adding ``add_http_frontend_args()`` for the HTTP variant." +msgstr "" + +#: ../../source/mp/architecture.rst:188 +msgid "Distributed Storage" +msgstr "" + +#: ../../source/mp/architecture.rst:191 +msgid "StorageManager" +msgstr "" + +#: ../../source/mp/architecture.rst:193 ../../source/mp/architecture.rst:423 +msgid "``lmcache/v1/distributed/storage_manager.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:195 +msgid "" +"The top-level manager that wires together L1, L2, and all controllers. " +"Key methods:" +msgstr "" + +#: ../../source/mp/architecture.rst:198 +msgid "``reserve_write()`` / ``finish_write()`` -- Two-phase write into L1." +msgstr "" + +#: ../../source/mp/architecture.rst:199 +msgid "" +"``submit_prefetch_task()`` / ``query_prefetch_status()`` -- Async lookup " +"+ L2 prefetch." +msgstr "" + +#: ../../source/mp/architecture.rst:201 +msgid "" +"``read_prefetched_results()`` / ``finish_read_prefetched()`` -- Read " +"prefetched data from L1 with automatic lock management." +msgstr "" + +#: ../../source/mp/architecture.rst:205 +msgid "L1Manager" +msgstr "" + +#: ../../source/mp/architecture.rst:207 ../../source/mp/architecture.rst:427 +msgid "``lmcache/v1/distributed/l1_manager.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:209 +msgid "Manages objects in CPU memory with a state machine:" +msgstr "" + +#: ../../source/mp/architecture.rst:219 +msgid "" +"Each object has two ``TTLLock`` instances (read and write) with " +"configurable timeouts to prevent deadlocks from crashed clients." +msgstr "" + +#: ../../source/mp/architecture.rst:222 +msgid "" +"The ``L1MemoryManager`` handles the underlying memory allocation (lazy " +"growth up to ``--l1-size-gb``)." +msgstr "" + +#: ../../source/mp/architecture.rst:226 +msgid "L2 Adapters" +msgstr "" + +#: ../../source/mp/architecture.rst:228 +msgid "``lmcache/v1/distributed/l2_adapters/``" +msgstr "" + +#: ../../source/mp/architecture.rst:230 +msgid "" +"The ``L2AdapterInterface`` (in ``base.py``) defines three async task " +"methods:" +msgstr "" + +#: ../../source/mp/architecture.rst:232 +msgid "``submit_store_task(key, data)`` -- Push data to L2." +msgstr "" + +#: ../../source/mp/architecture.rst:233 +msgid "``submit_lookup_and_lock_task(keys)`` -- Check if keys exist in L2." +msgstr "" + +#: ../../source/mp/architecture.rst:234 +msgid "``submit_load_task(keys, layout_desc)`` -- Load data from L2 into L1." +msgstr "" + +#: ../../source/mp/architecture.rst:236 +msgid "" +"The factory function ``create_l2_adapter()`` (in ``__init__.py``) uses " +"``isinstance()`` on the config type to instantiate the correct adapter." +msgstr "" + +#: ../../source/mp/architecture.rst:239 +msgid "" +"New adapter types are registered via ``register_l2_adapter_type()`` in " +"``config.py``." +msgstr "" + +#: ../../source/mp/architecture.rst:243 +msgid "Controllers" +msgstr "" + +#: ../../source/mp/architecture.rst:245 +msgid "" +"**StoreController** (``storage_controllers/store_controller.py``): Event-" +"driven background thread that uses ``select.poll()`` on listener eventfd " +"and adapter store eventfds. When new objects appear in L1 (signaled via " +"``StoreListener``), it submits async store tasks to each L2 adapter based" +" on the ``StorePolicy``." +msgstr "" + +#: ../../source/mp/architecture.rst:251 +msgid "" +"**EvictionController** (``storage_controllers/eviction_controller.py``): " +"Periodically checks L1 memory usage against the watermark threshold. " +"When triggered, evicts objects using the configured policy (``LRU``, " +"``IsolatedLRU``, or ``noop``) until usage drops below the target. " +"``IsolatedLRU`` evicts per ``cache_salt`` against limits registered " +"through the ``/quota`` HTTP endpoints; see :ref:`mp-http-quota-api`." +msgstr "" + +#: ../../source/mp/architecture.rst:258 +msgid "" +"**PrefetchController** (``storage_controllers/prefetch_controller.py``): " +"Handles L2 lookup and load requests submitted by ``StorageManager`` " +"during ``LOOKUP`` RPCs. When keys are not in L1, it queries L2 adapters " +"and loads found data back into L1." +msgstr "" + +#: ../../source/mp/architecture.rst:264 +msgid "Request Flows" +msgstr "" + +#: ../../source/mp/architecture.rst:267 +msgid "LOOKUP Flow" +msgstr "" + +#: ../../source/mp/architecture.rst:285 +msgid "STORE Flow" +msgstr "" + +#: ../../source/mp/architecture.rst:304 +msgid "RETRIEVE Flow" +msgstr "" + +#: ../../source/mp/architecture.rst:320 +msgid "Observability Internals" +msgstr "" + +#: ../../source/mp/architecture.rst:322 +msgid "" +"**EventBus** (``lmcache/v1/mp_observability/event_bus.py``) is a global " +"singleton initialized at server startup by ``init_observability()``. " +"Producers (L1Manager, StorageManager, MPCacheEngine) publish ``Event`` " +"objects to a bounded queue (``--event-bus-queue-size``, default 10000, " +"tail-drop on overflow). A background drain thread dispatches each event " +"to all registered subscribers." +msgstr "" + +#: ../../source/mp/architecture.rst:329 +msgid "" +"**Subscribers** live under ``lmcache/v1/mp_observability/subscribers/`` " +"and are grouped by concern: ``metrics/`` (OTel counters and lifecycle " +"histograms), ``logging/`` (Python logging handlers, lookup-hash JSONL), " +"and ``tracing/`` (OTel spans built from START/END event pairs). " +"``init_observability()`` registers the set selected by CLI flags " +"(``--disable-metrics``, ``--disable-logging``, ``--enable-tracing``)." +msgstr "" + +#: ../../source/mp/architecture.rst:336 +msgid "" +"**OTel providers** are set up via ``otel_init.py`` before subscribers are" +" constructed, so module-level ``get_meter()`` / ``get_tracer()`` calls " +"bind to the real provider. Metrics are exported both to an in-process " +"Prometheus ``/metrics`` endpoint (``--prometheus-port``, default 9090) " +"and, when ``--otlp-endpoint`` is set, pushed to an OTel collector." +msgstr "" + +#: ../../source/mp/architecture.rst:344 +msgid "How to Extend" +msgstr "" + +#: ../../source/mp/architecture.rst:347 +msgid "Adding a new L2 adapter" +msgstr "" + +#: ../../source/mp/architecture.rst:349 +msgid "" +"Create a new ``*_l2_adapter.py`` module under " +"``lmcache/v1/distributed/l2_adapters/`` — ``__init__.py`` auto-discovers " +"modules matching that suffix via ``pkgutil`` and imports them lazily on " +"first use, so no other files need to be modified." +msgstr "" + +#: ../../source/mp/architecture.rst:354 +msgid "" +"Create a config class subclassing ``L2AdapterConfigBase`` with " +"``from_dict()`` and ``help()`` methods." +msgstr "" + +#: ../../source/mp/architecture.rst:356 +msgid "" +"Create an adapter class implementing ``L2AdapterInterface``, and a small " +"factory function ``(config, l1_memory_desc) -> L2AdapterInterface``." +msgstr "" + +#: ../../source/mp/architecture.rst:359 +msgid "At module level, self-register both the config and the factory:" +msgstr "" + +#: ../../source/mp/architecture.rst:366 +msgid "" +"See ``mock_l2_adapter.py`` or ``s3_l2_adapter.py`` for reference " +"implementations." +msgstr "" + +#: ../../source/mp/architecture.rst:370 +msgid "Adding an observability subscriber" +msgstr "" + +#: ../../source/mp/architecture.rst:372 +#, python-brace-format +msgid "" +"Create a subscriber class subclassing ``EventSubscriber`` (defined in " +"``lmcache/v1/mp_observability/event_bus.py``): implement " +"``get_subscriptions()`` to return an ``{EventType: callback}`` mapping; " +"optionally override ``shutdown()`` for cleanup." +msgstr "" + +#: ../../source/mp/architecture.rst:376 +msgid "" +"Place the class under the appropriate concern group " +"(``subscribers/metrics/``, ``subscribers/logging/``, or " +"``subscribers/tracing/``) and export it from that package's " +"``__init__.py``." +msgstr "" + +#: ../../source/mp/architecture.rst:380 +msgid "" +"Register the subscriber in ``init_observability()`` " +"(``lmcache/v1/mp_observability/config.py``) via " +"``bus.register_subscriber(...)`` inside the branch matching its concern " +"(metrics / logging / tracing), gated on the corresponding CLI flag if " +"needed." +msgstr "" + +#: ../../source/mp/architecture.rst:387 +msgid "Adding a new request type" +msgstr "" + +#: ../../source/mp/architecture.rst:389 +msgid "Add a new member to ``RequestType`` in ``protocols/base.py``." +msgstr "" + +#: ../../source/mp/architecture.rst:390 +msgid "" +"Create a ``ProtocolDefinition`` in the appropriate ``protocols/*.py`` " +"file (``engine``, ``controller``, ``observability``, ``debug``, " +"``blend``, or ``blend_v2``) and add the request name to that module's " +"``REQUEST_NAMES``." +msgstr "" + +#: ../../source/mp/architecture.rst:393 +msgid "Implement the handler method on ``MPCacheEngine`` (or ``BlendEngineV2``)." +msgstr "" + +#: ../../source/mp/architecture.rst:394 +msgid "" +"Register the handler in ``run_cache_server()`` via " +"``add_handler_helper()``." +msgstr "" + +#: ../../source/mp/architecture.rst:397 +msgid "Key Source Files" +msgstr "" + +#: ../../source/mp/architecture.rst:403 +msgid "File" +msgstr "" + +#: ../../source/mp/architecture.rst:404 +msgid "Purpose" +msgstr "" + +#: ../../source/mp/architecture.rst:405 +msgid "``lmcache/v1/multiprocess/server.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:406 +msgid "MPCacheEngine + ZMQ server entry point" +msgstr "" + +#: ../../source/mp/architecture.rst:407 +msgid "``lmcache/v1/multiprocess/config.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:408 +msgid "MPServerConfig, HTTPFrontendConfig" +msgstr "" + +#: ../../source/mp/architecture.rst:409 +msgid "``lmcache/v1/multiprocess/blend_server_v2.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:410 +msgid "BlendEngineV2 (extends MPCacheEngine)" +msgstr "" + +#: ../../source/mp/architecture.rst:411 +msgid "``lmcache/v1/multiprocess/http_server.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:412 +msgid "FastAPI wrapper with health check and many other useful APIs" +msgstr "" + +#: ../../source/mp/architecture.rst:413 +msgid "``lmcache/v1/multiprocess/http_api_registry.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:414 +msgid "``HTTPAPIRegistry`` that auto-discovers routers in ``http_apis/``" +msgstr "" + +#: ../../source/mp/architecture.rst:415 +msgid "``lmcache/v1/multiprocess/http_apis/``" +msgstr "" + +#: ../../source/mp/architecture.rst:416 +msgid "" +"Extensible HTTP endpoints (``/``, ``/healthcheck``, ``/clear-cache``, " +"``/status``)" +msgstr "" + +#: ../../source/mp/architecture.rst:418 +msgid "``lmcache/v1/multiprocess/mp_runtime_plugin_launcher.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:419 +msgid "" +"``MPRuntimePluginLauncher`` that spawns runtime plugins with the full " +"server config serialized into environment variables" +msgstr "" + +#: ../../source/mp/architecture.rst:421 +msgid "``lmcache/v1/multiprocess/protocols/base.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:422 +msgid "RequestType, HandlerType, ProtocolDefinition" +msgstr "" + +#: ../../source/mp/architecture.rst:424 +msgid "StorageManager (top-level manager)" +msgstr "" + +#: ../../source/mp/architecture.rst:425 +msgid "``lmcache/v1/distributed/config.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:426 +msgid "StorageManagerConfig hierarchy" +msgstr "" + +#: ../../source/mp/architecture.rst:428 +msgid "L1Manager (object state machine)" +msgstr "" + +#: ../../source/mp/architecture.rst:429 +msgid "``lmcache/v1/distributed/l2_adapters/config.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:430 +msgid "L2 adapter config registry" +msgstr "" + +#: ../../source/mp/architecture.rst:431 +msgid "``lmcache/v1/distributed/l2_adapters/base.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:432 +msgid "L2AdapterInterface" +msgstr "" + +#: ../../source/mp/architecture.rst:433 +msgid "``lmcache/v1/distributed/storage_controllers/store_controller.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:434 +msgid "StoreController (event-driven L1->L2)" +msgstr "" + +#: ../../source/mp/architecture.rst:435 +msgid "``lmcache/v1/distributed/storage_controllers/eviction_controller.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:436 +msgid "EvictionController (watermark-triggered)" +msgstr "" + +#: ../../source/mp/architecture.rst:437 +msgid "``lmcache/v1/distributed/storage_controllers/prefetch_controller.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:438 +msgid "PrefetchController (L2->L1 on miss)" +msgstr "" + +#: ../../source/mp/architecture.rst:439 +msgid "``lmcache/v1/mp_observability/config.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:440 +msgid "ObservabilityConfig + ``init_observability()`` entry point" +msgstr "" + +#: ../../source/mp/architecture.rst:441 +msgid "``lmcache/v1/mp_observability/event_bus.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:442 +msgid "EventBus singleton and ``EventSubscriber`` base class" +msgstr "" + +#: ../../source/mp/architecture.rst:443 +msgid "``lmcache/v1/mp_observability/event.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:444 +msgid "``Event`` / ``EventType`` definitions" +msgstr "" + +#: ../../source/mp/architecture.rst:445 +msgid "``lmcache/v1/mp_observability/otel_init.py``" +msgstr "" + +#: ../../source/mp/architecture.rst:446 +msgid "OTel metrics / tracing provider setup" +msgstr "" + +#: ../../source/mp/architecture.rst:447 +msgid "``lmcache/v1/mp_observability/subscribers/``" +msgstr "" + +#: ../../source/mp/architecture.rst:448 +msgid "Metrics, logging, and tracing subscribers" +msgstr "" + +#: ../../source/mp/architecture.rst:449 +msgid "``lmcache/v1/mp_observability/trace/``" +msgstr "" + +#: ../../source/mp/architecture.rst:450 +msgid "Trace recording (``--trace-level storage``) capture stack" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/configuration.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/configuration.po new file mode 100644 index 00000000000..7332d59f1d9 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/configuration.po @@ -0,0 +1,788 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/configuration.rst:2 +msgid "Configuration Reference" +msgstr "" + +#: ../../source/mp/configuration.rst:4 +msgid "" +"This page documents every CLI argument accepted by the LMCache " +"multiprocess server. Arguments are grouped by the config module that " +"defines them." +msgstr "" + +#: ../../source/mp/configuration.rst:12 +msgid "MP Server" +msgstr "" + +#: ../../source/mp/configuration.rst:14 ../../source/mp/configuration.rst:104 +msgid "Source: ``lmcache/v1/multiprocess/config.py``" +msgstr "" + +#: ../../source/mp/configuration.rst:20 ../../source/mp/configuration.rst:82 +#: ../../source/mp/configuration.rst:112 ../../source/mp/configuration.rst:131 +#: ../../source/mp/configuration.rst:158 ../../source/mp/configuration.rst:177 +#: ../../source/mp/configuration.rst:209 ../../source/mp/configuration.rst:368 +msgid "Argument" +msgstr "" + +#: ../../source/mp/configuration.rst:21 ../../source/mp/configuration.rst:83 +#: ../../source/mp/configuration.rst:113 ../../source/mp/configuration.rst:132 +#: ../../source/mp/configuration.rst:159 ../../source/mp/configuration.rst:178 +#: ../../source/mp/configuration.rst:210 ../../source/mp/configuration.rst:369 +msgid "Default" +msgstr "" + +#: ../../source/mp/configuration.rst:22 ../../source/mp/configuration.rst:84 +#: ../../source/mp/configuration.rst:114 ../../source/mp/configuration.rst:133 +#: ../../source/mp/configuration.rst:160 ../../source/mp/configuration.rst:179 +#: ../../source/mp/configuration.rst:211 ../../source/mp/configuration.rst:370 +#: ../../source/mp/configuration.rst:419 +msgid "Description" +msgstr "" + +#: ../../source/mp/configuration.rst:23 +msgid "``--host``" +msgstr "" + +#: ../../source/mp/configuration.rst:24 +msgid "``localhost``" +msgstr "" + +#: ../../source/mp/configuration.rst:25 +msgid "Host address to bind the ZMQ server." +msgstr "" + +#: ../../source/mp/configuration.rst:26 +msgid "``--port``" +msgstr "" + +#: ../../source/mp/configuration.rst:27 +msgid "``5555``" +msgstr "" + +#: ../../source/mp/configuration.rst:28 +msgid "Port to bind the ZMQ server." +msgstr "" + +#: ../../source/mp/configuration.rst:29 +msgid "``--chunk-size``" +msgstr "" + +#: ../../source/mp/configuration.rst:30 +msgid "``256``" +msgstr "" + +#: ../../source/mp/configuration.rst:31 +msgid "Chunk size for KV cache operations (in tokens)." +msgstr "" + +#: ../../source/mp/configuration.rst:32 +msgid "``--max-workers``" +msgstr "" + +#: ../../source/mp/configuration.rst:33 +msgid "``1``" +msgstr "" + +#: ../../source/mp/configuration.rst:34 +msgid "" +"Base number of worker threads. Sets the default for both the GPU " +"(affinity) pool and the CPU (normal) pool. Can be overridden per-pool " +"with ``--max-gpu-workers`` and ``--max-cpu-workers``." +msgstr "" + +#: ../../source/mp/configuration.rst:37 +msgid "``--max-gpu-workers``" +msgstr "" + +#: ../../source/mp/configuration.rst:38 ../../source/mp/configuration.rst:43 +msgid "(inherits ``--max-workers``)" +msgstr "" + +#: ../../source/mp/configuration.rst:39 +msgid "" +"Worker threads for the GPU affinity pool (STORE/RETRIEVE). Requests from " +"the same vLLM instance are always dispatched to the same thread, " +"eliminating GPU transfer lock contention." +msgstr "" + +#: ../../source/mp/configuration.rst:42 +msgid "``--max-cpu-workers``" +msgstr "" + +#: ../../source/mp/configuration.rst:44 +msgid "Worker threads for the normal CPU pool (LOOKUP, etc.)." +msgstr "" + +#: ../../source/mp/configuration.rst:45 +msgid "``--hash-algorithm``" +msgstr "" + +#: ../../source/mp/configuration.rst:46 +msgid "``blake3``" +msgstr "" + +#: ../../source/mp/configuration.rst:47 +msgid "" +"Hash algorithm for token-based operations. Choices: ``builtin``, " +"``sha256_cbor``, ``blake3``." +msgstr "" + +#: ../../source/mp/configuration.rst:49 +msgid "``--engine-type``" +msgstr "" + +#: ../../source/mp/configuration.rst:50 ../../source/mp/configuration.rst:213 +#: ../../source/mp/configuration.rst:221 +msgid "``default``" +msgstr "" + +#: ../../source/mp/configuration.rst:51 +msgid "" +"Cache engine backend type. ``default`` uses MPCacheEngine; ``blend`` uses" +" BlendEngineV2 for cross-request KV reuse. Choices: ``default``, " +"``blend``." +msgstr "" + +#: ../../source/mp/configuration.rst:55 +msgid "``--runtime-plugin-locations``" +msgstr "" + +#: ../../source/mp/configuration.rst:56 +msgid "``[]``" +msgstr "" + +#: ../../source/mp/configuration.rst:57 +msgid "" +"Zero or more paths to runtime plugin scripts or directories to launch " +"alongside the server. Plugins are spawned by ``MPRuntimePluginLauncher`` " +"and receive the full server config via the " +"``LMCACHE_RUNTIME_PLUGIN_CONFIG`` environment variable." +msgstr "" + +#: ../../source/mp/configuration.rst:61 +msgid "``--runtime-plugin-config``" +msgstr "" + +#: ../../source/mp/configuration.rst:62 +#, python-brace-format +msgid "``\"{}\"``" +msgstr "" + +#: ../../source/mp/configuration.rst:63 +#, python-brace-format +msgid "" +"JSON string of extra key-value config forwarded to runtime plugins via " +"``LMCACHE_RUNTIME_PLUGIN_EXTRA_CONFIG``. Example: " +"``'{\"plugin.frontend.heartbeat_url\": " +"\"http://localhost:5000/heartbeat\"}'``." +msgstr "" + +#: ../../source/mp/configuration.rst:68 +msgid "Lookup Hash Logging" +msgstr "" + +#: ../../source/mp/configuration.rst:70 +msgid "Source: ``lmcache/v1/mp_observability/subscribers/logging/lookup_hash.py``" +msgstr "" + +#: ../../source/mp/configuration.rst:72 +msgid "" +"When enabled, the server publishes chunk hashes computed during " +"``lookup()`` as ``MP_LOOKUP`` events on the EventBus. The " +"``LookupHashLoggingSubscriber`` writes these to rotating JSONL files for " +"offline analysis. Disabled by default. These arguments are part of the " +"Observability group." +msgstr "" + +#: ../../source/mp/configuration.rst:85 +msgid "``--lookup-hash-log-dir``" +msgstr "" + +#: ../../source/mp/configuration.rst:86 +msgid "``\"\"`` (disabled)" +msgstr "" + +#: ../../source/mp/configuration.rst:87 +msgid "" +"Directory to write lookup hash JSONL files. An empty string disables " +"logging." +msgstr "" + +#: ../../source/mp/configuration.rst:89 +msgid "``--lookup-hash-log-rotation-interval``" +msgstr "" + +#: ../../source/mp/configuration.rst:90 +msgid "``21600`` (6 h)" +msgstr "" + +#: ../../source/mp/configuration.rst:91 +msgid "Time interval in seconds before rotating to a new log file." +msgstr "" + +#: ../../source/mp/configuration.rst:92 +msgid "``--lookup-hash-log-rotation-max-size``" +msgstr "" + +#: ../../source/mp/configuration.rst:93 +msgid "``104857600`` (100 MB)" +msgstr "" + +#: ../../source/mp/configuration.rst:94 +msgid "" +"Max file size in bytes before rotating even if the time interval has not " +"elapsed." +msgstr "" + +#: ../../source/mp/configuration.rst:96 +msgid "``--lookup-hash-log-max-files``" +msgstr "" + +#: ../../source/mp/configuration.rst:97 +msgid "``100``" +msgstr "" + +#: ../../source/mp/configuration.rst:98 +msgid "" +"Max number of log files to keep. Oldest files are deleted when this " +"limit is exceeded." +msgstr "" + +#: ../../source/mp/configuration.rst:102 +msgid "HTTP Frontend" +msgstr "" + +#: ../../source/mp/configuration.rst:106 +msgid "The HTTP frontend is included when running ``lmcache server``." +msgstr "" + +#: ../../source/mp/configuration.rst:115 +msgid "``--http-host``" +msgstr "" + +#: ../../source/mp/configuration.rst:116 +msgid "``0.0.0.0``" +msgstr "" + +#: ../../source/mp/configuration.rst:117 +msgid "Host to bind the HTTP (FastAPI/uvicorn) server." +msgstr "" + +#: ../../source/mp/configuration.rst:118 +msgid "``--http-port``" +msgstr "" + +#: ../../source/mp/configuration.rst:119 +msgid "``8080``" +msgstr "" + +#: ../../source/mp/configuration.rst:120 +msgid "Port to bind the HTTP server." +msgstr "" + +#: ../../source/mp/configuration.rst:123 +msgid "L1 Memory Manager" +msgstr "" + +#: ../../source/mp/configuration.rst:125 ../../source/mp/configuration.rst:152 +#: ../../source/mp/configuration.rst:171 ../../source/mp/configuration.rst:203 +msgid "Source: ``lmcache/v1/distributed/config.py``" +msgstr "" + +#: ../../source/mp/configuration.rst:134 +msgid "``--l1-size-gb``" +msgstr "" + +#: ../../source/mp/configuration.rst:135 ../../source/mp/configuration.rst:181 +msgid "*required*" +msgstr "" + +#: ../../source/mp/configuration.rst:136 +msgid "Size of L1 memory in GB." +msgstr "" + +#: ../../source/mp/configuration.rst:137 +msgid "``--l1-use-lazy`` / ``--no-l1-use-lazy``" +msgstr "" + +#: ../../source/mp/configuration.rst:138 +msgid "``True``" +msgstr "" + +#: ../../source/mp/configuration.rst:139 +msgid "" +"Enable or disable lazy allocation for L1 memory. Pass ``--l1-use-lazy`` " +"to enable (default) or ``--no-l1-use-lazy`` to explicitly disable." +msgstr "" + +#: ../../source/mp/configuration.rst:142 +msgid "``--l1-init-size-gb``" +msgstr "" + +#: ../../source/mp/configuration.rst:143 +msgid "``20``" +msgstr "" + +#: ../../source/mp/configuration.rst:144 +msgid "Initial allocation size (GB) when using lazy allocation." +msgstr "" + +#: ../../source/mp/configuration.rst:145 +msgid "``--l1-align-bytes``" +msgstr "" + +#: ../../source/mp/configuration.rst:146 +msgid "``4096``" +msgstr "" + +#: ../../source/mp/configuration.rst:147 +msgid "Alignment size in bytes (default 4 KB)." +msgstr "" + +#: ../../source/mp/configuration.rst:150 +msgid "L1 Manager TTLs" +msgstr "" + +#: ../../source/mp/configuration.rst:161 +msgid "``--l1-write-ttl-seconds``" +msgstr "" + +#: ../../source/mp/configuration.rst:162 +msgid "``600``" +msgstr "" + +#: ../../source/mp/configuration.rst:163 +msgid "Time-to-live for each object's write lock (seconds)." +msgstr "" + +#: ../../source/mp/configuration.rst:164 +msgid "``--l1-read-ttl-seconds``" +msgstr "" + +#: ../../source/mp/configuration.rst:165 +msgid "``300``" +msgstr "" + +#: ../../source/mp/configuration.rst:166 +msgid "Time-to-live for each object's read lock (seconds)." +msgstr "" + +#: ../../source/mp/configuration.rst:169 +msgid "Eviction Policy" +msgstr "" + +#: ../../source/mp/configuration.rst:180 +msgid "``--eviction-policy``" +msgstr "" + +#: ../../source/mp/configuration.rst:182 +msgid "" +"Eviction policy. Choices: ``LRU``, ``IsolatedLRU``, ``noop``. Use " +"``noop`` for buffer-only mode where L1 acts as a pure write buffer (data " +"is deleted from L1 after L2 store). ``IsolatedLRU`` maintains one LRU " +"list per ``cache_salt`` and requires per-``cache_salt`` quotas to be " +"configured at runtime via the ``/quota`` HTTP endpoints (see :ref:`mp-" +"http-quota-api`); a ``cache_salt`` with no registered quota has an " +"effective limit of ``0`` bytes, so its data is evicted at the next " +"eviction cycle (allowlist semantics)." +msgstr "" + +#: ../../source/mp/configuration.rst:193 +msgid "``--eviction-trigger-watermark``" +msgstr "" + +#: ../../source/mp/configuration.rst:194 +msgid "``0.8``" +msgstr "" + +#: ../../source/mp/configuration.rst:195 +msgid "Memory usage ratio (0.0--1.0) that triggers eviction." +msgstr "" + +#: ../../source/mp/configuration.rst:196 +msgid "``--eviction-ratio``" +msgstr "" + +#: ../../source/mp/configuration.rst:197 +msgid "``0.2``" +msgstr "" + +#: ../../source/mp/configuration.rst:198 +msgid "Fraction of allocated memory to evict when triggered (0.0--1.0)." +msgstr "" + +#: ../../source/mp/configuration.rst:201 +msgid "L2 Policies" +msgstr "" + +#: ../../source/mp/configuration.rst:212 +msgid "``--l2-store-policy``" +msgstr "" + +#: ../../source/mp/configuration.rst:214 +msgid "" +"L2 store policy. Determines which adapters receive each key and whether " +"keys are deleted from L1 after L2 store. The ``default`` policy stores " +"all keys to all adapters and keeps L1. The ``skip_l1`` policy stores all " +"keys to all adapters and then deletes them from L1 (buffer-only mode). " +"Choices: ``default``, ``skip_l1``." +msgstr "" + +#: ../../source/mp/configuration.rst:220 +msgid "``--l2-prefetch-policy``" +msgstr "" + +#: ../../source/mp/configuration.rst:222 +msgid "" +"L2 prefetch policy. Determines which adapter loads each key when " +"multiple adapters have it. The ``default`` policy picks the first adapter" +" (lowest index). Prefetched keys are temporary (deleted after the reader " +"finishes). The ``retain`` policy uses the same load plan but keeps " +"prefetched keys permanently in L1. Choices: ``default``, ``retain``." +msgstr "" + +#: ../../source/mp/configuration.rst:229 +msgid "``--l2-prefetch-max-in-flight``" +msgstr "" + +#: ../../source/mp/configuration.rst:230 +msgid "``8``" +msgstr "" + +#: ../../source/mp/configuration.rst:231 +msgid "" +"Maximum number of concurrent prefetch (L2 load) requests. Limits how many" +" in-flight loads the PrefetchController may issue at once, preventing " +"excessive L1 memory pressure." +msgstr "" + +#: ../../source/mp/configuration.rst:236 +msgid "L2 Adapters" +msgstr "" + +#: ../../source/mp/configuration.rst:238 +msgid "Source: ``lmcache/v1/distributed/l2_adapters/config.py``" +msgstr "" + +#: ../../source/mp/configuration.rst:240 +msgid "" +"L2 adapters are configured via repeatable ``--l2-adapter `` " +"arguments. Each JSON object must include a ``\"type\"`` field that " +"selects the adapter type. The order of ``--l2-adapter`` arguments " +"determines the adapter order (cascade)." +msgstr "" + +#: ../../source/mp/configuration.rst:244 +msgid "" +"Registered adapter types: ``nixl_store``, ``nixl_store_dynamic``, ``fs``," +" ``fs_native``, ``mock``, ``mooncake_store``, ``s3``, ``resp``, " +"``plugin``, ``native_plugin``, ``raw_block``, ``dax``." +msgstr "" + +#: ../../source/mp/configuration.rst:249 +msgid "``nixl_store`` -- NIXL-based persistent storage" +msgstr "" + +#: ../../source/mp/configuration.rst:251 ../../source/mp/configuration.rst:287 +#: ../../source/mp/configuration.rst:307 ../../source/mp/configuration.rst:325 +msgid "Fields:" +msgstr "" + +#: ../../source/mp/configuration.rst:253 +msgid "" +"``backend`` *(required)*: One of ``POSIX``, ``GDS``, ``GDS_MT``, " +"``HF3FS``, ``OBJ``, ``AZURE_BLOB``." +msgstr "" + +#: ../../source/mp/configuration.rst:254 +msgid "" +"``backend_params`` *(required for file-based backends)*: Dict of string " +"key-value pairs. File-based backends (``GDS``, ``GDS_MT``, ``POSIX``, " +"``HF3FS``) require ``file_path`` and ``use_direct_io``." +msgstr "" + +#: ../../source/mp/configuration.rst:257 +msgid "" +"``pool_size`` *(required)*: Number of storage descriptors to pre-allocate" +" (> 0)." +msgstr "" + +#: ../../source/mp/configuration.rst:259 ../../source/mp/configuration.rst:294 +msgid "Examples:" +msgstr "" + +#: ../../source/mp/configuration.rst:283 +msgid "``fs`` -- File-system backed storage" +msgstr "" + +#: ../../source/mp/configuration.rst:285 +msgid "A pure file-system L2 adapter using async I/O." +msgstr "" + +#: ../../source/mp/configuration.rst:289 +msgid "``base_path`` *(required)*: Directory for storing KV cache files." +msgstr "" + +#: ../../source/mp/configuration.rst:290 +msgid "``relative_tmp_dir`` *(optional)*: Relative sub-dir for temp files." +msgstr "" + +#: ../../source/mp/configuration.rst:291 +msgid "" +"``read_ahead_size`` *(optional)*: Trigger read-ahead by reading this many" +" bytes first." +msgstr "" + +#: ../../source/mp/configuration.rst:292 +msgid "" +"``use_odirect`` *(optional)*: Bypass page cache via ``O_DIRECT`` (default" +" ``false``)." +msgstr "" + +#: ../../source/mp/configuration.rst:305 +msgid "``mock`` -- Mock adapter for testing" +msgstr "" + +#: ../../source/mp/configuration.rst:309 +msgid "``max_size_gb`` *(required)*: Maximum size of the adapter in GB (> 0)." +msgstr "" + +#: ../../source/mp/configuration.rst:310 +msgid "``mock_bandwidth_gb`` *(required)*: Simulated bandwidth in GB/sec (> 0)." +msgstr "" + +#: ../../source/mp/configuration.rst:312 ../../source/mp/configuration.rst:340 +msgid "Example:" +msgstr "" + +#: ../../source/mp/configuration.rst:319 +msgid "``s3`` -- S3-compatible object store" +msgstr "" + +#: ../../source/mp/configuration.rst:321 +msgid "" +"S3-backed L2 adapter using the AWS CRT (Common Runtime) for high-" +"throughput transfers to AWS S3 or any S3-compatible endpoint. See " +":doc:`l2_storage` for details." +msgstr "" + +#: ../../source/mp/configuration.rst:327 +msgid "" +"``s3_endpoint`` *(required)*: Bucket URL, either ``\"s3://\"`` or" +" the bare host form." +msgstr "" + +#: ../../source/mp/configuration.rst:329 +msgid "``s3_region`` *(required)*: AWS region string." +msgstr "" + +#: ../../source/mp/configuration.rst:330 +msgid "``s3_num_io_threads`` *(optional, default ``64``)*: CRT I/O threads." +msgstr "" + +#: ../../source/mp/configuration.rst:331 +msgid "" +"``s3_prefer_http2`` *(optional, default ``true``)*: Negotiate HTTP/2 via " +"ALPN." +msgstr "" + +#: ../../source/mp/configuration.rst:332 +msgid "" +"``s3_enable_s3express`` *(optional, default ``false``)*: Enable S3 " +"Express signing." +msgstr "" + +#: ../../source/mp/configuration.rst:333 +msgid "" +"``disable_tls`` *(optional, default ``false``)*: Bypass TLS (for non-AWS " +"HTTP endpoints)." +msgstr "" + +#: ../../source/mp/configuration.rst:335 +msgid "" +"``aws_access_key_id`` / ``aws_secret_access_key`` *(optional)*: Static " +"credentials; omit to use the default credential provider chain." +msgstr "" + +#: ../../source/mp/configuration.rst:337 +msgid "" +"``max_capacity_gb`` *(optional, default ``0.0``)*: Aggregate capacity " +"used by ``get_usage()``. A value of ``0`` disables aggregate eviction." +msgstr "" + +#: ../../source/mp/configuration.rst:347 +msgid "Multiple adapters (cascade)" +msgstr "" + +#: ../../source/mp/configuration.rst:349 +msgid "" +"Pass ``--l2-adapter`` multiple times. Adapters are used in the order " +"given:" +msgstr "" + +#: ../../source/mp/configuration.rst:357 +msgid "Observability" +msgstr "" + +#: ../../source/mp/configuration.rst:359 +msgid "Source: ``lmcache/v1/mp_observability/config.py``" +msgstr "" + +#: ../../source/mp/configuration.rst:361 +msgid "" +"See :doc:`observability` for full details on the three modes (metrics, " +"logging, tracing)." +msgstr "" + +#: ../../source/mp/configuration.rst:371 +msgid "``--disable-observability``" +msgstr "" + +#: ../../source/mp/configuration.rst:372 ../../source/mp/configuration.rst:375 +#: ../../source/mp/configuration.rst:378 ../../source/mp/configuration.rst:381 +msgid "off" +msgstr "" + +#: ../../source/mp/configuration.rst:373 +msgid "Master switch: disable the EventBus entirely." +msgstr "" + +#: ../../source/mp/configuration.rst:374 +msgid "``--disable-metrics``" +msgstr "" + +#: ../../source/mp/configuration.rst:376 +msgid "Skip metrics subscribers (no Prometheus endpoint)." +msgstr "" + +#: ../../source/mp/configuration.rst:377 +msgid "``--disable-logging``" +msgstr "" + +#: ../../source/mp/configuration.rst:379 +msgid "Skip logging subscribers." +msgstr "" + +#: ../../source/mp/configuration.rst:380 +msgid "``--enable-tracing``" +msgstr "" + +#: ../../source/mp/configuration.rst:382 +msgid "Register tracing subscribers. Requires ``--otlp-endpoint``." +msgstr "" + +#: ../../source/mp/configuration.rst:383 +msgid "``--event-bus-queue-size``" +msgstr "" + +#: ../../source/mp/configuration.rst:384 +msgid "``10000``" +msgstr "" + +#: ../../source/mp/configuration.rst:385 +msgid "Max events in the EventBus queue before tail-drop." +msgstr "" + +#: ../../source/mp/configuration.rst:386 +msgid "``--otlp-endpoint``" +msgstr "" + +#: ../../source/mp/configuration.rst:387 +msgid "*(none)*" +msgstr "" + +#: ../../source/mp/configuration.rst:388 +msgid "OTLP gRPC endpoint for exporting metrics and traces." +msgstr "" + +#: ../../source/mp/configuration.rst:389 +msgid "``--prometheus-port``" +msgstr "" + +#: ../../source/mp/configuration.rst:390 +msgid "``9090``" +msgstr "" + +#: ../../source/mp/configuration.rst:391 +msgid "Port for the Prometheus ``/metrics`` endpoint." +msgstr "" + +#: ../../source/mp/configuration.rst:392 +msgid "``--service-instance-id``" +msgstr "" + +#: ../../source/mp/configuration.rst:393 +msgid "*(unset, default UUID v4)*" +msgstr "" + +#: ../../source/mp/configuration.rst:394 +msgid "" +"Identifier for this MP server instance, attached as the OTel Resource " +"attribute ``service.instance.id`` on every metric and span. When the flag" +" is not passed, defaults to a random UUID v4. Pass ``--service-instance-" +"id=\"\"`` to force an empty value." +msgstr "" + +#: ../../source/mp/configuration.rst:400 +msgid "vLLM Client Configuration" +msgstr "" + +#: ../../source/mp/configuration.rst:402 +msgid "" +"On the vLLM side, specify the LMCache server host and port via the " +"``kv_connector_extra_config`` parameter:" +msgstr "" + +#: ../../source/mp/configuration.rst:412 +msgid "Environment Variables" +msgstr "" + +#: ../../source/mp/configuration.rst:418 +msgid "Variable" +msgstr "" + +#: ../../source/mp/configuration.rst:420 +msgid "``LMCACHE_LOG_LEVEL``" +msgstr "" + +#: ../../source/mp/configuration.rst:421 +msgid "" +"Log level for LMCache (``DEBUG``, ``INFO``, ``WARNING``, ``ERROR``). Set " +"to ``DEBUG`` to see L2 store activity, prefetch results, etc." +msgstr "" + +#: ../../source/mp/configuration.rst:423 +msgid "``PYTHONHASHSEED``" +msgstr "" + +#: ../../source/mp/configuration.rst:424 +msgid "" +"Set to a fixed value for reproducible hashing across processes (relevant " +"when using ``--hash-algorithm builtin``)." +msgstr "" + +#: ../../source/mp/configuration.rst:428 +msgid "Full Example" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/deployment.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/deployment.po new file mode 100644 index 00000000000..fc31ef2e146 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/deployment.po @@ -0,0 +1,254 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/deployment.rst:2 +msgid "Deployment Guide" +msgstr "" + +#: ../../source/mp/deployment.rst:4 +msgid "" +"This page covers deploying LMCache multiprocess mode in Docker and " +"Kubernetes environments, along with production best practices." +msgstr "" + +#: ../../source/mp/deployment.rst:12 +msgid "Docker" +msgstr "" + +#: ../../source/mp/deployment.rst:14 +msgid "**LMCache container:**" +msgstr "" + +#: ../../source/mp/deployment.rst:25 +msgid "**vLLM container:**" +msgstr "" + +#: ../../source/mp/deployment.rst:37 +msgid "Required Docker flags:" +msgstr "" + +#: ../../source/mp/deployment.rst:39 +msgid "" +"``--network host`` -- Allows the vLLM container to reach LMCache on " +"localhost." +msgstr "" + +#: ../../source/mp/deployment.rst:40 +msgid "" +"``--ipc host`` -- Required for CUDA IPC shared memory transfers between " +"containers." +msgstr "" + +#: ../../source/mp/deployment.rst:42 +msgid "" +"``--runtime nvidia --gpus all`` -- GPU access via the NVIDIA container " +"runtime." +msgstr "" + +#: ../../source/mp/deployment.rst:45 +msgid "**HTTP server variant:**" +msgstr "" + +#: ../../source/mp/deployment.rst:47 +msgid "" +"For health-check and cache management API support (useful with container " +"orchestrators), use the HTTP server entry point:" +msgstr "" + +#: ../../source/mp/deployment.rst:60 +msgid "Kubernetes" +msgstr "" + +#: ../../source/mp/deployment.rst:62 +msgid "" +"LMCache is designed for a **DaemonSet + Deployment** pattern: one LMCache" +" server per node (DaemonSet) shared by multiple vLLM pods (Deployment)." +msgstr "" + +#: ../../source/mp/deployment.rst:65 +msgid "Example YAML files are provided in ``examples/multi_process/``." +msgstr "" + +#: ../../source/mp/deployment.rst:68 +msgid "Prerequisites" +msgstr "" + +#: ../../source/mp/deployment.rst:70 +msgid "Kubernetes cluster with GPU support (NVIDIA GPU Operator installed)" +msgstr "" + +#: ../../source/mp/deployment.rst:71 +msgid "At least 4 GPUs per node" +msgstr "" + +#: ../../source/mp/deployment.rst:72 +msgid "``kubectl`` configured to access your cluster" +msgstr "" + +#: ../../source/mp/deployment.rst:75 +msgid "Step-by-Step" +msgstr "" + +#: ../../source/mp/deployment.rst:77 +msgid "**Step 1: Create namespace**" +msgstr "" + +#: ../../source/mp/deployment.rst:83 +msgid "**Step 2: Deploy LMCache DaemonSet**" +msgstr "" + +#: ../../source/mp/deployment.rst:89 +msgid "**Step 3: Deploy vLLM**" +msgstr "" + +#: ../../source/mp/deployment.rst:96 +msgid "" +"The default model is ``Qwen/Qwen3-14B``. For gated models (e.g., Llama)," +" create a Secret with your Hugging Face token:" +msgstr "" + +#: ../../source/mp/deployment.rst:105 +msgid "Then add the ``HF_TOKEN`` environment variable to the vLLM container spec." +msgstr "" + +#: ../../source/mp/deployment.rst:107 +msgid "**Step 4: Monitor deployment**" +msgstr "" + +#: ../../source/mp/deployment.rst:126 +msgid "**Step 5: Send test requests**" +msgstr "" + +#: ../../source/mp/deployment.rst:141 +msgid "Architecture Notes" +msgstr "" + +#: ../../source/mp/deployment.rst:143 +msgid "" +"**DaemonSet uses ``hostNetwork: true``** so vLLM pods discover the " +"LMCache server via ``status.hostIP``." +msgstr "" + +#: ../../source/mp/deployment.rst:145 +msgid "" +"**Both containers mount ``/dev/shm``** from the host to enable CUDA IPC " +"memory sharing." +msgstr "" + +#: ../../source/mp/deployment.rst:147 +msgid "" +"**GPUs are NOT requested in the DaemonSet** -- this allows GPUs to remain" +" exclusively allocated to vLLM pods. The NVIDIA container runtime " +"automatically provides GPU access for IPC-based memory transfers." +msgstr "" + +#: ../../source/mp/deployment.rst:150 +msgid "" +"**Multiple vLLM pods** on the same node automatically connect to the same" +" LMCache DaemonSet instance." +msgstr "" + +#: ../../source/mp/deployment.rst:154 +msgid "" +"LMCache pods on nodes without GPUs will crash with CUDA initialization " +"errors. This is expected -- LMCache only needs to run on GPU nodes where" +" vLLM pods are scheduled." +msgstr "" + +#: ../../source/mp/deployment.rst:159 +msgid "Health Checking (HTTP Server)" +msgstr "" + +#: ../../source/mp/deployment.rst:161 +msgid "" +"For Kubernetes liveness/readiness probes, deploy the HTTP server variant " +"instead. Use the ``/healthcheck`` endpoint:" +msgstr "" + +#: ../../source/mp/deployment.rst:180 +msgid "Monitoring Integration" +msgstr "" + +#: ../../source/mp/deployment.rst:182 +msgid "" +"Prometheus metrics are enabled by default on port 9090. Add a " +"``ServiceMonitor`` or Prometheus scrape annotation to collect metrics " +"from the LMCache DaemonSet pods. See :doc:`observability` for metric " +"details." +msgstr "" + +#: ../../source/mp/deployment.rst:187 +msgid "Cleanup" +msgstr "" + +#: ../../source/mp/deployment.rst:196 +msgid "Production Best Practices" +msgstr "" + +#: ../../source/mp/deployment.rst:198 +msgid "" +"**Worker count (``--max-workers``, ``--max-gpu-workers``, ``--max-cpu-" +"workers``):** ``--max-workers`` sets both the GPU affinity pool and CPU " +"normal pool sizes (default 1). Use ``--max-gpu-workers`` to override the" +" GPU pool independently --- set it to at least the number of vLLM " +"instances sharing the cache server so each instance gets its own " +"dedicated thread. Use ``--max-cpu-workers`` to override the CPU pool for" +" lookup and other non-GPU operations." +msgstr "" + +#: ../../source/mp/deployment.rst:205 +msgid "" +"**L1 memory sizing (``--l1-size-gb``):** Allocate as much CPU memory as " +"available after accounting for the OS and vLLM. A larger L1 cache means " +"fewer L2 round-trips." +msgstr "" + +#: ../../source/mp/deployment.rst:209 +msgid "**Eviction tuning:**" +msgstr "" + +#: ../../source/mp/deployment.rst:211 +#, python-format +msgid "" +"``--eviction-trigger-watermark 0.8`` (default) triggers eviction when L1 " +"is 80% full." +msgstr "" + +#: ../../source/mp/deployment.rst:213 +#, python-format +msgid "" +"``--eviction-ratio 0.2`` (default) frees 20% of allocated memory per " +"eviction cycle." +msgstr "" + +#: ../../source/mp/deployment.rst:215 +msgid "" +"Lower the watermark or increase the ratio if you observe frequent " +"evictions under steady load." +msgstr "" + +#: ../../source/mp/deployment.rst:218 +msgid "" +"**Logging:** Use ``LMCACHE_LOG_LEVEL=DEBUG`` during initial setup to " +"verify L2 store/load activity. Switch to ``INFO`` (default) for " +"production to reduce log volume." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/http_api.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/http_api.po new file mode 100644 index 00000000000..c2428abe861 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/http_api.po @@ -0,0 +1,772 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/http_api.rst:2 +msgid "HTTP API" +msgstr "" + +#: ../../source/mp/http_api.rst:4 +msgid "" +"When the MP server is started via ``lmcache server`` (the recommended " +"entry point), a FastAPI-based HTTP frontend is exposed alongside the ZMQ " +"socket used by vLLM. This HTTP API is intended for operators, " +"orchestrators (e.g. Kubernetes), and debugging tools — it is **not** on " +"the inference data path." +msgstr "" + +#: ../../source/mp/http_api.rst:10 +msgid "" +"New endpoints are registered automatically from " +"``lmcache/v1/multiprocess/http_apis/``: any module named ``*_api.py`` " +"that exposes a module-level ``router`` (a :class:`fastapi.APIRouter`) is " +"discovered at startup." +msgstr "" + +#: ../../source/mp/http_api.rst:15 +msgid "" +"A subset of routes defined under " +"``lmcache/v1/internal_api_server/common/`` is also exposed on this HTTP " +"server. The module ``lmcache/v1/multiprocess/http_apis/common_api.py`` " +"aggregates those routers (skipping modules listed in " +"``_MP_INCOMPATIBLE_MODULES``, such as ``run_script_api``) and forwards " +"them to the auto-discovery pipeline. Adding a new compatible module under" +" ``internal_api_server/common`` therefore requires no wiring changes on " +"the MP side." +msgstr "" + +#: ../../source/mp/http_api.rst:29 +msgid "Server Configuration" +msgstr "" + +#: ../../source/mp/http_api.rst:35 +msgid "Argument" +msgstr "" + +#: ../../source/mp/http_api.rst:36 +msgid "Default" +msgstr "" + +#: ../../source/mp/http_api.rst:37 +msgid "Description" +msgstr "" + +#: ../../source/mp/http_api.rst:38 +msgid "``--http-host``" +msgstr "" + +#: ../../source/mp/http_api.rst:39 +msgid "``0.0.0.0``" +msgstr "" + +#: ../../source/mp/http_api.rst:40 +msgid "Host to bind the HTTP server." +msgstr "" + +#: ../../source/mp/http_api.rst:41 +msgid "``--http-port``" +msgstr "" + +#: ../../source/mp/http_api.rst:42 +msgid "``8080``" +msgstr "" + +#: ../../source/mp/http_api.rst:43 +msgid "Port to bind the HTTP server." +msgstr "" + +#: ../../source/mp/http_api.rst:45 +msgid "Example:" +msgstr "" + +#: ../../source/mp/http_api.rst:53 +msgid "" +"All examples below assume the server is reachable at " +"``http://localhost:8080``." +msgstr "" + +#: ../../source/mp/http_api.rst:57 +msgid "Endpoints" +msgstr "" + +#: ../../source/mp/http_api.rst:59 +msgid "" +"The table below groups the routes by purpose. The operational surface " +"(health, status, cache control) is exposed at top-level paths. Routes " +"inherited from the shared ``internal_api_server`` package are kept at " +"their original paths for compatibility with the vLLM-embedded API server." +msgstr "" + +#: ../../source/mp/http_api.rst:69 +msgid "Method" +msgstr "" + +#: ../../source/mp/http_api.rst:70 +msgid "Path" +msgstr "" + +#: ../../source/mp/http_api.rst:71 +msgid "Purpose" +msgstr "" + +#: ../../source/mp/http_api.rst:72 ../../source/mp/http_api.rst:75 +#: ../../source/mp/http_api.rst:78 ../../source/mp/http_api.rst:84 +#: ../../source/mp/http_api.rst:90 ../../source/mp/http_api.rst:97 +#: ../../source/mp/http_api.rst:101 ../../source/mp/http_api.rst:104 +#: ../../source/mp/http_api.rst:107 ../../source/mp/http_api.rst:110 +#: ../../source/mp/http_api.rst:113 ../../source/mp/http_api.rst:116 +#: ../../source/mp/http_api.rst:122 ../../source/mp/http_api.rst:125 +#: ../../source/mp/http_api.rst:128 ../../source/mp/http_api.rst:131 +msgid "GET" +msgstr "" + +#: ../../source/mp/http_api.rst:73 +msgid "``/``" +msgstr "" + +#: ../../source/mp/http_api.rst:74 +msgid "Basic liveness ping." +msgstr "" + +#: ../../source/mp/http_api.rst:76 +msgid "``/healthcheck``" +msgstr "" + +#: ../../source/mp/http_api.rst:77 +msgid "K8s liveness/readiness probe." +msgstr "" + +#: ../../source/mp/http_api.rst:79 +msgid "``/status``" +msgstr "" + +#: ../../source/mp/http_api.rst:80 +msgid "Detailed engine status for inspection and debugging." +msgstr "" + +#: ../../source/mp/http_api.rst:81 ../../source/mp/http_api.rst:119 +msgid "POST" +msgstr "" + +#: ../../source/mp/http_api.rst:82 +msgid "``/clear-cache``" +msgstr "" + +#: ../../source/mp/http_api.rst:83 +msgid "Force-clear all KV data in L1 (CPU) memory." +msgstr "" + +#: ../../source/mp/http_api.rst:85 +msgid "``/quota``" +msgstr "" + +#: ../../source/mp/http_api.rst:86 +msgid "List every registered ``cache_salt`` quota with live usage." +msgstr "" + +#: ../../source/mp/http_api.rst:87 +msgid "PUT" +msgstr "" + +#: ../../source/mp/http_api.rst:88 ../../source/mp/http_api.rst:91 +#: ../../source/mp/http_api.rst:94 +#, python-brace-format +msgid "``/quota/{cache_salt}``" +msgstr "" + +#: ../../source/mp/http_api.rst:89 +msgid "Set or update the quota (in GB) for a ``cache_salt``." +msgstr "" + +#: ../../source/mp/http_api.rst:92 +msgid "Read the quota and live usage for a single ``cache_salt``." +msgstr "" + +#: ../../source/mp/http_api.rst:93 +msgid "DELETE" +msgstr "" + +#: ../../source/mp/http_api.rst:95 +msgid "Remove a ``cache_salt``'s quota entry (its data is evicted next cycle)." +msgstr "" + +#: ../../source/mp/http_api.rst:98 +msgid "``/conf``" +msgstr "" + +#: ../../source/mp/http_api.rst:99 +msgid "Dump merged server configurations (mp, storage_manager, observability)." +msgstr "" + +#: ../../source/mp/http_api.rst:102 +msgid "``/version``" +msgstr "" + +#: ../../source/mp/http_api.rst:103 +msgid "Full version descriptor (package version + commit id)." +msgstr "" + +#: ../../source/mp/http_api.rst:105 +msgid "``/lmc_version``" +msgstr "" + +#: ../../source/mp/http_api.rst:106 +msgid "LMCache package version string." +msgstr "" + +#: ../../source/mp/http_api.rst:108 +msgid "``/commit_id``" +msgstr "" + +#: ../../source/mp/http_api.rst:109 +msgid "Current build commit id." +msgstr "" + +#: ../../source/mp/http_api.rst:111 +msgid "``/env``" +msgstr "" + +#: ../../source/mp/http_api.rst:112 +msgid "Dump process environment variables (JSON, plain text)." +msgstr "" + +#: ../../source/mp/http_api.rst:114 +msgid "``/loglevel``" +msgstr "" + +#: ../../source/mp/http_api.rst:115 +msgid "List or inspect logger levels; also accepts ``level`` to mutate." +msgstr "" + +#: ../../source/mp/http_api.rst:117 +msgid "``/metrics``" +msgstr "" + +#: ../../source/mp/http_api.rst:118 +msgid "Prometheus exposition format." +msgstr "" + +#: ../../source/mp/http_api.rst:120 +msgid "``/metrics/reset``" +msgstr "" + +#: ../../source/mp/http_api.rst:121 +msgid "Reset all observability metrics to their initial state." +msgstr "" + +#: ../../source/mp/http_api.rst:123 +msgid "``/threads``" +msgstr "" + +#: ../../source/mp/http_api.rst:124 +msgid "Enumerate active Python threads and their stack traces." +msgstr "" + +#: ../../source/mp/http_api.rst:126 +msgid "``/periodic-threads``" +msgstr "" + +#: ../../source/mp/http_api.rst:127 +msgid "List registered periodic threads with summary counts." +msgstr "" + +#: ../../source/mp/http_api.rst:129 +#, python-brace-format +msgid "``/periodic-threads/{thread_name}``" +msgstr "" + +#: ../../source/mp/http_api.rst:130 +msgid "Detailed status for a single periodic thread." +msgstr "" + +#: ../../source/mp/http_api.rst:132 +msgid "``/periodic-threads-health``" +msgstr "" + +#: ../../source/mp/http_api.rst:133 +msgid "Quick health check for critical/high-level periodic threads." +msgstr "" + +#: ../../source/mp/http_api.rst:136 +msgid "``GET /``" +msgstr "" + +#: ../../source/mp/http_api.rst:138 +msgid "" +"Basic liveness check. Returns a static payload indicating the HTTP server" +" is running. Use ``/healthcheck`` instead for probes that also verify the" +" cache engine is initialized." +msgstr "" + +#: ../../source/mp/http_api.rst:142 ../../source/mp/http_api.rst:165 +#: ../../source/mp/http_api.rst:214 ../../source/mp/http_api.rst:279 +#: ../../source/mp/http_api.rst:334 ../../source/mp/http_api.rst:357 +#: ../../source/mp/http_api.rst:380 ../../source/mp/http_api.rst:394 +#: ../../source/mp/http_api.rst:415 ../../source/mp/http_api.rst:454 +#: ../../source/mp/http_api.rst:560 ../../source/mp/http_api.rst:617 +#: ../../source/mp/http_api.rst:657 +msgid "**Response** (``200 OK``):" +msgstr "" + +#: ../../source/mp/http_api.rst:151 ../../source/mp/http_api.rst:182 +#: ../../source/mp/http_api.rst:260 ../../source/mp/http_api.rst:296 +#: ../../source/mp/http_api.rst:344 ../../source/mp/http_api.rst:442 +#: ../../source/mp/http_api.rst:460 ../../source/mp/http_api.rst:471 +#: ../../source/mp/http_api.rst:482 ../../source/mp/http_api.rst:500 +#: ../../source/mp/http_api.rst:547 ../../source/mp/http_api.rst:566 +#: ../../source/mp/http_api.rst:591 ../../source/mp/http_api.rst:633 +#: ../../source/mp/http_api.rst:644 ../../source/mp/http_api.rst:684 +msgid "**Example:**" +msgstr "" + +#: ../../source/mp/http_api.rst:158 +msgid "``GET /healthcheck``" +msgstr "" + +#: ../../source/mp/http_api.rst:160 +msgid "" +"Health check endpoint suitable for Kubernetes liveness and readiness " +"probes. A ``200`` response implies the HTTP server is alive **and** the " +"MP cache engine is initialized. A ``503`` response indicates the engine " +"is not yet ready (still initializing, or failed to initialize)." +msgstr "" + +#: ../../source/mp/http_api.rst:173 ../../source/mp/http_api.rst:287 +msgid "**Response** (``503 Service Unavailable``):" +msgstr "" + +#: ../../source/mp/http_api.rst:188 +msgid "**Kubernetes probe snippet:**" +msgstr "" + +#: ../../source/mp/http_api.rst:206 +msgid "``GET /status``" +msgstr "" + +#: ../../source/mp/http_api.rst:208 +msgid "" +"Returns a detailed snapshot of the MP engine's internal state: L1 cache, " +"L2 adapters, registered GPU contexts, active sessions, and in-flight " +"prefetch jobs. Intended for operators and debugging, not for monitoring " +"(use Prometheus metrics for time-series data — see :doc:`observability`)." +msgstr "" + +#: ../../source/mp/http_api.rst:251 +msgid "" +"**Response** (``503 Service Unavailable``) when the engine has not yet " +"been initialized:" +msgstr "" + +#: ../../source/mp/http_api.rst:267 +msgid "``POST /clear-cache``" +msgstr "" + +#: ../../source/mp/http_api.rst:269 +msgid "Force-clears **all** KV cache data currently held in L1 (CPU) memory." +msgstr "" + +#: ../../source/mp/http_api.rst:273 +msgid "" +"This endpoint is destructive and bypasses read/write locks. In-flight " +"store or prefetch operations may be corrupted. Use only when the server " +"is idle, or when recovering from a known-bad cache state." +msgstr "" + +#: ../../source/mp/http_api.rst:277 +msgid "The request body is ignored." +msgstr "" + +#: ../../source/mp/http_api.rst:305 +msgid "``/quota`` — per-``cache_salt`` quota management" +msgstr "" + +#: ../../source/mp/http_api.rst:307 +msgid "" +"These endpoints manage the per-``cache_salt`` storage budgets consumed by" +" the ``IsolatedLRU`` eviction policy (selected via ``--eviction-policy " +"IsolatedLRU``). Quotas are **soft**: setting a limit does not reject " +"writes — any over-budget ``cache_salt`` is evicted at the next eviction " +"cycle (~1 s). A ``cache_salt`` with no registered quota has an effective " +"limit of ``0`` bytes, so its data is cleared next cycle (allowlist " +"semantics)." +msgstr "" + +#: ../../source/mp/http_api.rst:315 +msgid "" +"These endpoints are no-ops on engines that did not start with " +"``--eviction-policy IsolatedLRU``: the ``QuotaManager`` is still present," +" but the LRU policy ignores the registered quotas." +msgstr "" + +#: ../../source/mp/http_api.rst:319 +msgid "" +"**URL escaping for the empty salt.** ``cache_salt=\"\"`` (un-salted / " +"anonymous traffic) cannot appear in a URL path parameter, so the API " +"accepts the sentinel ``_default`` in its place. ``PUT /quota/_default`` " +"sets the quota for ``cache_salt=\"\"``. A user that legitimately stores " +"data with ``cache_salt=\"_default\"`` cannot be managed via this HTTP API" +" distinctly from anonymous traffic — both map to the same path parameter;" +" pick any other value (e.g. ``\"default\"``) to disambiguate." +msgstr "" + +#: ../../source/mp/http_api.rst:328 +#, python-brace-format +msgid "``PUT /quota/{cache_salt}``" +msgstr "" + +#: ../../source/mp/http_api.rst:330 +msgid "Create or update a quota." +msgstr "" + +#: ../../source/mp/http_api.rst:332 +#, python-brace-format +msgid "**Body:** ``{\"limit_gb\": }`` (required, finite, non-negative)." +msgstr "" + +#: ../../source/mp/http_api.rst:340 +msgid "" +"**Errors:** ``400`` for malformed JSON, missing ``limit_gb``, non-numeric" +" ``limit_gb``, ``nan`` / ``inf``, or negative values; ``503`` if the " +"engine is not initialized." +msgstr "" + +#: ../../source/mp/http_api.rst:353 +#, python-brace-format +msgid "``GET /quota/{cache_salt}``" +msgstr "" + +#: ../../source/mp/http_api.rst:355 +msgid "Read the current quota and live usage for one ``cache_salt``." +msgstr "" + +#: ../../source/mp/http_api.rst:368 +msgid "" +"``exists`` is ``false`` when no quota was ever registered for this " +"``cache_salt`` (``limit_gb`` is then ``0.0`` and ``current_usage_gb`` " +"reflects whatever bytes are currently cached for that salt — those bytes " +"will evict next cycle under ``IsolatedLRU``)." +msgstr "" + +#: ../../source/mp/http_api.rst:374 +#, python-brace-format +msgid "``DELETE /quota/{cache_salt}``" +msgstr "" + +#: ../../source/mp/http_api.rst:376 +msgid "" +"Remove a ``cache_salt``'s quota entry. Any bytes still cached under this " +"``cache_salt`` become over-budget on the next eviction cycle (effective " +"limit drops to ``0``) and will be evicted." +msgstr "" + +#: ../../source/mp/http_api.rst:386 +#, python-brace-format +msgid "" +"When no quota was registered for the given ``cache_salt``, the response " +"is ``{\"cache_salt\": \"...\", \"status\": \"not_found\"}`` (still ``200 " +"OK``)." +msgstr "" + +#: ../../source/mp/http_api.rst:390 +msgid "``GET /quota``" +msgstr "" + +#: ../../source/mp/http_api.rst:392 +msgid "List every registered quota alongside its live usage." +msgstr "" + +#: ../../source/mp/http_api.rst:406 +msgid "``GET /conf``" +msgstr "" + +#: ../../source/mp/http_api.rst:408 +msgid "" +"Returns every server-side configuration object registered on " +"``app.state.configs`` (typically ``mp``, ``storage_manager`` and " +"``observability``) as a single indented JSON document. Dataclasses are " +"serialized via ``safe_asdict``; other values go through " +"``make_json_safe``. Useful for confirming what the process actually " +"loaded — including environment overrides — without restarting." +msgstr "" + +#: ../../source/mp/http_api.rst:433 +msgid "" +"**Response** (``503 Service Unavailable``) when configs are not wired " +"onto ``app.state`` yet:" +msgstr "" + +#: ../../source/mp/http_api.rst:449 +msgid "``GET /version``" +msgstr "" + +#: ../../source/mp/http_api.rst:451 +msgid "" +"Returns the full version descriptor (package version combined with the " +"current commit id), formatted by ``lmcache.utils.get_version()``." +msgstr "" + +#: ../../source/mp/http_api.rst:467 +msgid "``GET /lmc_version``" +msgstr "" + +#: ../../source/mp/http_api.rst:469 +msgid "" +"Returns the raw LMCache package version string " +"(``lmcache.utils.VERSION``)." +msgstr "" + +#: ../../source/mp/http_api.rst:478 +msgid "``GET /commit_id``" +msgstr "" + +#: ../../source/mp/http_api.rst:480 +msgid "" +"Returns the git commit id baked into the build " +"(``lmcache.utils.COMMIT_ID``)." +msgstr "" + +#: ../../source/mp/http_api.rst:489 +msgid "``GET /env``" +msgstr "" + +#: ../../source/mp/http_api.rst:491 +msgid "" +"Dumps the process environment variables as a sorted, pretty-printed JSON " +"document. Response ``Content-Type`` is ``text/plain`` so it can be piped " +"directly to a terminal." +msgstr "" + +#: ../../source/mp/http_api.rst:497 +msgid "" +"The payload may contain secrets injected via environment variables. " +"Restrict network access to this endpoint in production." +msgstr "" + +#: ../../source/mp/http_api.rst:507 +msgid "``GET /loglevel``" +msgstr "" + +#: ../../source/mp/http_api.rst:509 +msgid "" +"Inspect or mutate Python logger levels at runtime. All responses are " +"``text/plain``. The endpoint has three modes driven by query parameters:" +msgstr "" + +#: ../../source/mp/http_api.rst:516 ../../source/mp/http_api.rst:583 +#: ../../source/mp/http_api.rst:608 +msgid "Query" +msgstr "" + +#: ../../source/mp/http_api.rst:517 ../../source/mp/http_api.rst:584 +#: ../../source/mp/http_api.rst:609 +msgid "Behavior" +msgstr "" + +#: ../../source/mp/http_api.rst:518 +msgid "(no params)" +msgstr "" + +#: ../../source/mp/http_api.rst:519 +msgid "List every logger registered with :mod:`logging` and its level." +msgstr "" + +#: ../../source/mp/http_api.rst:520 +msgid "``?logger_name=``" +msgstr "" + +#: ../../source/mp/http_api.rst:521 +msgid "Return the effective level of the named logger." +msgstr "" + +#: ../../source/mp/http_api.rst:522 +msgid "``?logger_name=&level=``" +msgstr "" + +#: ../../source/mp/http_api.rst:523 +msgid "" +"Set the named logger (and its handlers) to ``LEVEL`` " +"(``DEBUG``/``INFO``/``WARNING``/``ERROR``/``CRITICAL``). Returns ``400`` " +"on an unknown level." +msgstr "" + +#: ../../source/mp/http_api.rst:527 +msgid "**Examples:**" +msgstr "" + +#: ../../source/mp/http_api.rst:541 +msgid "``GET /metrics``" +msgstr "" + +#: ../../source/mp/http_api.rst:543 +msgid "" +"Prometheus exposition format for every metric registered on the default " +"``prometheus_client`` registry. Scrape this directly from Prometheus. See" +" :doc:`observability` for the list of exported metrics." +msgstr "" + +#: ../../source/mp/http_api.rst:554 +msgid "``POST /metrics/reset``" +msgstr "" + +#: ../../source/mp/http_api.rst:556 +msgid "" +"Resets all LMCache observability metrics to their initial state " +"(``reset_observability_metrics``). Intended for test harnesses and " +"benchmarks — not for production." +msgstr "" + +#: ../../source/mp/http_api.rst:573 +msgid "``GET /threads``" +msgstr "" + +#: ../../source/mp/http_api.rst:575 +msgid "" +"Enumerate active Python threads in the server process along with their " +"stack traces, plus a total-count summary. Useful for live debugging of " +"hangs or runaway workers." +msgstr "" + +#: ../../source/mp/http_api.rst:585 +msgid "``?name=``" +msgstr "" + +#: ../../source/mp/http_api.rst:586 +msgid "Keep only threads whose name contains ```` (case-insensitive)." +msgstr "" + +#: ../../source/mp/http_api.rst:588 +msgid "``?thread_id=``" +msgstr "" + +#: ../../source/mp/http_api.rst:589 +msgid "Keep only the thread with the matching ``ident``." +msgstr "" + +#: ../../source/mp/http_api.rst:598 +msgid "``GET /periodic-threads``" +msgstr "" + +#: ../../source/mp/http_api.rst:600 +msgid "" +"Returns a JSON snapshot of the " +":class:`~lmcache.v1.periodic_thread.PeriodicThreadRegistry`: counts by " +"level plus per-thread status (last run timestamp, latest summary, etc.)." +msgstr "" + +#: ../../source/mp/http_api.rst:610 +msgid "``?level=critical|high|medium|low``" +msgstr "" + +#: ../../source/mp/http_api.rst:611 +msgid "Only include threads at the given level. ``400`` on unknown." +msgstr "" + +#: ../../source/mp/http_api.rst:612 +msgid "``?running_only=true``" +msgstr "" + +#: ../../source/mp/http_api.rst:613 +msgid "Only include threads currently running." +msgstr "" + +#: ../../source/mp/http_api.rst:614 +msgid "``?active_only=true``" +msgstr "" + +#: ../../source/mp/http_api.rst:615 +msgid "Only include threads considered active (recent tick)." +msgstr "" + +#: ../../source/mp/http_api.rst:640 +#, python-brace-format +msgid "``GET /periodic-threads/{thread_name}``" +msgstr "" + +#: ../../source/mp/http_api.rst:642 +msgid "Detailed status for a single periodic thread (``404`` if not found)." +msgstr "" + +#: ../../source/mp/http_api.rst:651 +msgid "``GET /periodic-threads-health``" +msgstr "" + +#: ../../source/mp/http_api.rst:653 +msgid "" +"Fast health check covering only ``critical`` and ``high`` level periodic " +"threads. A thread is flagged unhealthy when it is marked running but has " +"not ticked within its expected interval." +msgstr "" + +#: ../../source/mp/http_api.rst:667 +msgid "When something is lagging:" +msgstr "" + +#: ../../source/mp/http_api.rst:691 +msgid "Adding New Endpoints" +msgstr "" + +#: ../../source/mp/http_api.rst:693 +msgid "" +"Endpoints are auto-discovered from " +"``lmcache/v1/multiprocess/http_apis/``. To add a new endpoint:" +msgstr "" + +#: ../../source/mp/http_api.rst:696 +msgid "Create a new module in that directory named ``_api.py``." +msgstr "" + +#: ../../source/mp/http_api.rst:697 +msgid "Define a module-level ``router = APIRouter()``." +msgstr "" + +#: ../../source/mp/http_api.rst:698 +msgid "Register handlers on ``router`` using FastAPI decorators." +msgstr "" + +#: ../../source/mp/http_api.rst:699 +msgid "" +"Access the engine via ``request.app.state.engine`` and guard for the " +"``None`` case (engine not yet initialized)." +msgstr "" + +#: ../../source/mp/http_api.rst:702 +msgid "" +"The :class:`~lmcache.v1.multiprocess.http_api_registry.HTTPAPIRegistry` " +"will pick the module up automatically at startup — no central " +"registration list to edit." +msgstr "" + +#: ../../source/mp/http_api.rst:706 +msgid "" +"If the route is generic enough to be shared with the vLLM-embedded API " +"server, add it under ``lmcache/v1/internal_api_server/common/`` instead. " +"It will be picked up on the MP side via ``common_api.py`` unless its " +"module name is listed in ``_MP_INCOMPATIBLE_MODULES`` there (used for " +"modules that require vLLM-specific ``app.state`` attributes, e.g. " +"``run_script_api``)." +msgstr "" + +#: ../../source/mp/http_api.rst:713 +msgid "" +"When adding a new endpoint, please also add a matching section to this " +"page documenting the endpoint's purpose, request/response schema, and an " +"example ``curl`` invocation." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/index.po new file mode 100644 index 00000000000..c958ec75daa --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/index.po @@ -0,0 +1,140 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/index.rst:54 +msgid "Contents" +msgstr "" + +#: ../../source/mp/index.rst:2 +msgid "Multiprocess Mode" +msgstr "" + +#: ../../source/mp/index.rst:4 +msgid "" +"LMCache multiprocess (MP) mode runs LMCache as a **standalone service** " +"that vLLM instances connect to over ZMQ. One LMCache server per node can" +" serve multiple vLLM pods, providing process isolation, shared caching, " +"and independent resource scaling." +msgstr "" + +#: ../../source/mp/index.rst:10 +msgid "Key Benefits" +msgstr "" + +#: ../../source/mp/index.rst:12 +msgid "" +"**Process isolation** -- LMCache and vLLM run in separate processes (or " +"containers), so a cache-related issue does not crash the inference " +"engine." +msgstr "" + +#: ../../source/mp/index.rst:14 +msgid "" +"**No GIL contention or Python overhead on the inference path** -- By " +"running LMCache in a separate process, its Python GIL and CPU work " +"(hashing, memory management, L2 I/O) do not compete with vLLM's inference" +" threads." +msgstr "" + +#: ../../source/mp/index.rst:17 +msgid "" +"**Shared caching across pods** -- Multiple vLLM instances on the same " +"node share a single L1 cache, maximizing KV reuse." +msgstr "" + +#: ../../source/mp/index.rst:19 +msgid "" +"**Independent resource scaling** -- Allocate CPU memory for caching " +"independently of GPU memory for inference." +msgstr "" + +#: ../../source/mp/index.rst:21 +msgid "" +"**Multi-tier storage (L1 + L2)** -- In-memory L1 cache backed by " +"persistent L2 storage via NIXL (GDS, POSIX, HF3FS, and more)." +msgstr "" + +#: ../../source/mp/index.rst:23 +msgid "" +"**Built-in observability** -- Prometheus metrics and a telemetry event " +"system out of the box." +msgstr "" + +#: ../../source/mp/index.rst:27 +msgid "Prerequisites" +msgstr "" + +#: ../../source/mp/index.rst:29 +msgid "**vLLM** latest version is recommended for best compatibility" +msgstr "" + +#: ../../source/mp/index.rst:30 +msgid "**LMCache** latest dev branch" +msgstr "" + +#: ../../source/mp/index.rst:33 +msgid "Server Variants" +msgstr "" + +#: ../../source/mp/index.rst:35 +msgid "LMCache ships three server entry points:" +msgstr "" + +#: ../../source/mp/index.rst:41 +msgid "Entry Point" +msgstr "" + +#: ../../source/mp/index.rst:42 +msgid "Description" +msgstr "" + +#: ../../source/mp/index.rst:43 +msgid "``lmcache server``" +msgstr "" + +#: ../../source/mp/index.rst:44 +msgid "" +"**Recommended.** ZMQ + FastAPI HTTP frontend (adds ``/healthcheck`` for " +"K8s probes, ``/clear-cache``, ``/status`` — see :doc:`http_api`). Use " +"``--engine-type blend`` to enable BlendEngineV2 for cross-request KV " +"reuse." +msgstr "" + +#: ../../source/mp/index.rst:48 +msgid "``python3 -m lmcache.v1.multiprocess.server``" +msgstr "" + +#: ../../source/mp/index.rst:49 +msgid "" +"(Legacy) ZMQ-only server using MPCacheEngine (no HTTP endpoints). Prefer " +"``lmcache server``." +msgstr "" + +#: ../../source/mp/index.rst:51 +msgid "``python3 -m lmcache.v1.multiprocess.blend_server_v2``" +msgstr "" + +#: ../../source/mp/index.rst:52 +msgid "" +"(Legacy) CacheBlend-enabled server. Prefer ``lmcache server --engine-type" +" blend``." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage.po new file mode 100644 index 00000000000..e920fa6fd86 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage.po @@ -0,0 +1,1272 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/l2_storage.rst:2 +msgid "L2 Storage (Persistent Cache)" +msgstr "" + +#: ../../source/mp/l2_storage.rst:4 +msgid "LMCache multiprocess mode supports a two-tier storage architecture:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:6 +msgid "" +"**L1 (in-memory)** -- Fast CPU memory managed by the L1 Manager. All KV " +"cache chunks live here during active use." +msgstr "" + +#: ../../source/mp/l2_storage.rst:8 +msgid "" +"**L2 (persistent)** -- Durable storage backends (NIXL-based or plain " +"file-system/raw-block). The StoreController asynchronously pushes data " +"from L1 to L2, and the PrefetchController loads data from L2 back into L1" +" on cache misses." +msgstr "" + +#: ../../source/mp/l2_storage.rst:18 +msgid "Data Flow" +msgstr "" + +#: ../../source/mp/l2_storage.rst:20 +msgid "**Write path (L1 -> L2):**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:22 +msgid "vLLM stores KV cache chunks into L1 via the ``STORE`` RPC." +msgstr "" + +#: ../../source/mp/l2_storage.rst:23 +msgid "" +"The ``StoreController`` detects new objects (via eventfd) and " +"asynchronously submits store tasks to each configured L2 adapter." +msgstr "" + +#: ../../source/mp/l2_storage.rst:25 +msgid "The L2 adapter writes the data to its backend (e.g., local SSD via GDS)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:27 +msgid "**Read path (L2 -> L1):**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:29 +msgid "A ``LOOKUP`` RPC checks L1 for prefix hits." +msgstr "" + +#: ../../source/mp/l2_storage.rst:30 +msgid "" +"For keys not found in L1, the ``PrefetchController`` submits lookup " +"requests to L2 adapters." +msgstr "" + +#: ../../source/mp/l2_storage.rst:32 +msgid "" +"If found in L2, the data is loaded back into L1 and read-locked for the " +"pending ``RETRIEVE`` RPC." +msgstr "" + +#: ../../source/mp/l2_storage.rst:36 +msgid "Adapter Types" +msgstr "" + +#: ../../source/mp/l2_storage.rst:39 +msgid "``nixl_store`` -- NIXL-based persistent storage" +msgstr "" + +#: ../../source/mp/l2_storage.rst:41 +msgid "" +"The primary production adapter. Uses NIXL (NVIDIA Interconnect Library) " +"for high-performance storage I/O." +msgstr "" + +#: ../../source/mp/l2_storage.rst:44 ../../source/mp/l2_storage.rst:118 +#: ../../source/mp/l2_storage.rst:170 ../../source/mp/l2_storage.rst:207 +#: ../../source/mp/l2_storage.rst:260 ../../source/mp/l2_storage.rst:343 +#: ../../source/mp/l2_storage.rst:512 +msgid "**Required fields:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:46 +msgid "" +"``backend``: Storage backend -- one of ``POSIX``, ``GDS``, ``GDS_MT``, " +"``HF3FS``, ``OBJ``, ``AZURE_BLOB``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:48 +msgid "" +"``pool_size``: Number of storage descriptors to pre-allocate (must be > " +"0)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:50 ../../source/mp/l2_storage.rst:123 +msgid "**Backend-specific parameters (``backend_params``):**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:52 +msgid "File-based backends (``GDS``, ``GDS_MT``, ``POSIX``, ``HF3FS``) require:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:54 +msgid "``file_path``: Directory path for storing L2 data." +msgstr "" + +#: ../../source/mp/l2_storage.rst:55 +msgid "" +"``use_direct_io``: ``\"true\"`` or ``\"false\"`` -- whether to use direct" +" I/O." +msgstr "" + +#: ../../source/mp/l2_storage.rst:57 +msgid "" +"The ``OBJ`` and ``AZURE_BLOB`` backends (object stores) do not require " +"``file_path``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:59 +msgid "**Backend descriptions:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:65 +msgid "Backend" +msgstr "" + +#: ../../source/mp/l2_storage.rst:66 ../../source/mp/l2_storage.rst:632 +#: ../../source/mp/l2_storage.rst:686 ../../source/mp/l2_storage.rst:726 +msgid "Description" +msgstr "" + +#: ../../source/mp/l2_storage.rst:67 +msgid "``POSIX``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:68 +msgid "Standard POSIX file I/O. Works on any file system. No direct I/O." +msgstr "" + +#: ../../source/mp/l2_storage.rst:69 +msgid "``GDS``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:70 +msgid "" +"NVIDIA GPU Direct Storage. Enables direct GPU-to-storage transfers " +"bypassing the CPU. Requires NVMe SSDs with GDS support." +msgstr "" + +#: ../../source/mp/l2_storage.rst:72 +msgid "``GDS_MT``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:73 +msgid "Multi-threaded variant of GDS for higher throughput." +msgstr "" + +#: ../../source/mp/l2_storage.rst:74 +msgid "``HF3FS``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:75 +msgid "Shared file system backend (e.g., for distributed/networked storage)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:76 +msgid "``OBJ``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:77 +msgid "Object store backend. No local file path required." +msgstr "" + +#: ../../source/mp/l2_storage.rst:78 +msgid "``AZURE_BLOB``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:79 +msgid "Object store backend for Azure Blob Storage. No local file path required." +msgstr "" + +#: ../../source/mp/l2_storage.rst:81 ../../source/mp/l2_storage.rst:140 +#: ../../source/mp/l2_storage.rst:183 ../../source/mp/l2_storage.rst:308 +#: ../../source/mp/l2_storage.rst:383 ../../source/mp/l2_storage.rst:533 +msgid "**Configuration examples:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:104 +msgid "``nixl_store_dynamic`` -- NIXL-based dynamic storage with persist/recover" +msgstr "" + +#: ../../source/mp/l2_storage.rst:106 +msgid "" +"A dynamic variant of the NIXL adapter that opens and registers files per-" +"operation instead of pre-allocating them at init. This enables:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:109 +msgid "**Persist/recover** -- cached KV metadata survives restarts." +msgstr "" + +#: ../../source/mp/l2_storage.rst:110 +msgid "" +"**No fd limits** -- files are opened and closed per transfer, so the " +"cache can grow beyond OS open-file-descriptor limits." +msgstr "" + +#: ../../source/mp/l2_storage.rst:115 +msgid "" +"Only file-based backends are supported (``POSIX``, ``GDS``, ``GDS_MT``, " +"``HF3FS``). The ``OBJ`` and ``AZURE_BLOB`` backends are not supported " +"yet." +msgstr "" + +#: ../../source/mp/l2_storage.rst:120 +msgid "" +"``backend``: Storage backend -- one of ``POSIX``, ``GDS``, ``GDS_MT``, " +"``HF3FS``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:125 +msgid "``file_path``: Directory path for storing L2 data files." +msgstr "" + +#: ../../source/mp/l2_storage.rst:126 +msgid "``use_direct_io``: ``\"true\"`` or ``\"false\"``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:127 +msgid "" +"``max_capacity_gb``: Maximum storage capacity in GB. The adapter rejects " +"stores when this limit is reached. Required for the eviction controller " +"to compute usage." +msgstr "" + +#: ../../source/mp/l2_storage.rst:131 +msgid "**Optional fields (for persist):**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:133 +msgid "" +"``persist_enabled`` (bool, default ``true``): If ``true``, data files are" +" kept on disk at shutdown. If ``false``, all data files are deleted on " +"shutdown." +msgstr "" + +#: ../../source/mp/l2_storage.rst:137 +msgid "" +"Lookup always checks secondary storage (disk) on miss and lazily " +"populates the in-memory index when a file is found." +msgstr "" + +#: ../../source/mp/l2_storage.rst:153 +msgid "**Persist / secondary lookup behaviour:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:155 +msgid "" +"On **shutdown**, the adapter keeps data files on disk by default " +"(``persist_enabled`` defaults to ``true``). If explicitly set to " +"``false``, all data files are deleted to avoid orphaned storage." +msgstr "" + +#: ../../source/mp/l2_storage.rst:158 +msgid "" +"On **startup**, the in-memory index is empty. Every lookup miss falls " +"through to a secondary lookup on disk: if the deterministic file exists, " +"it is treated as a hit and the in-memory index is populated lazily from " +"the file size." +msgstr "" + +#: ../../source/mp/l2_storage.rst:164 +msgid "``fs`` -- File-system backed storage" +msgstr "" + +#: ../../source/mp/l2_storage.rst:166 +msgid "" +"A pure file-system L2 adapter using async I/O (``aiofiles``). Each KV " +"cache object is stored as a raw ``.data`` file whose name encodes the " +"full ``ObjectKey``. Does **not** require NIXL -- works on any POSIX file" +" system." +msgstr "" + +#: ../../source/mp/l2_storage.rst:172 ../../source/mp/l2_storage.rst:262 +msgid "``base_path``: Directory for storing KV cache files." +msgstr "" + +#: ../../source/mp/l2_storage.rst:174 ../../source/mp/l2_storage.rst:215 +#: ../../source/mp/l2_storage.rst:264 ../../source/mp/l2_storage.rst:348 +#: ../../source/mp/l2_storage.rst:518 +msgid "**Optional fields:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:176 +msgid "" +"``relative_tmp_dir``: Relative sub-directory for temporary files during " +"writes (atomic rename on completion)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:178 +msgid "" +"``read_ahead_size``: Trigger file-system read-ahead by reading this many " +"bytes first (positive integer, optional)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:180 +msgid "" +"``use_odirect``: ``true`` or ``false`` (default ``false``) -- bypass the " +"page cache via ``O_DIRECT``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:197 +msgid "``dax`` -- Device-DAX fixed-slot storage" +msgstr "" + +#: ../../source/mp/l2_storage.rst:199 +msgid "" +"An L2 adapter that maps a single Device-DAX path, such as " +"``/dev/dax1.0``, and stores KV cache objects in fixed-size slots. This " +"adapter is intended for byte-addressable memory devices such as " +"persistent memory or CXL memory." +msgstr "" + +#: ../../source/mp/l2_storage.rst:203 +msgid "" +"The MP ``dax`` adapter is volatile in this release. It keeps the key " +"index in server memory and rebuilds an empty index on restart. Old bytes" +" may remain on the DAX device, but they are unreachable after the LMCache" +" server restarts." +msgstr "" + +#: ../../source/mp/l2_storage.rst:209 +msgid "``device_path``: Path to the mmap-able DAX device or test file." +msgstr "" + +#: ../../source/mp/l2_storage.rst:210 +msgid "``max_dax_size_gb``: Number of GiB to map from ``device_path``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:211 +msgid "" +"``slot_bytes``: Fixed slot size in bytes. This must be large enough for " +"one full LMCache chunk because MP memory descriptors do not expose the " +"non-MP full-chunk size." +msgstr "" + +#: ../../source/mp/l2_storage.rst:217 +msgid "``num_store_workers`` (int, default ``1``): Store worker threads." +msgstr "" + +#: ../../source/mp/l2_storage.rst:218 +msgid "``num_lookup_workers`` (int, default ``1``): Lookup worker threads." +msgstr "" + +#: ../../source/mp/l2_storage.rst:219 +msgid "" +"``num_load_workers`` (int, default ``min(4, os.cpu_count())``): Load " +"worker threads." +msgstr "" + +#: ../../source/mp/l2_storage.rst:221 +msgid "" +"``persist_enabled`` (bool): Accepted by common L2 config parsing but has " +"no effect for ``dax`` because restart recovery is not implemented." +msgstr "" + +#: ../../source/mp/l2_storage.rst:224 ../../source/mp/l2_storage.rst:461 +msgid "**Configuration example:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:243 +msgid "**Current limits:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:245 +msgid "" +"Uses one server-owned mapped DAX path. Per-TP partitions and multi-device" +" striping are not implemented." +msgstr "" + +#: ../../source/mp/l2_storage.rst:247 +msgid "" +"Only single-buffer objects are supported. Multi-tensor objects are " +"rejected." +msgstr "" + +#: ../../source/mp/l2_storage.rst:248 +msgid "" +"Capacity is slot-based, not payload-byte-based. L2 eviction and usage " +"metrics count occupied slots." +msgstr "" + +#: ../../source/mp/l2_storage.rst:250 +msgid "" +"Lookups acquire DAX-side external locks. ``submit_unlock`` releases those" +" locks after load/retrieve completes, making entries evictable again." +msgstr "" + +#: ../../source/mp/l2_storage.rst:253 +msgid "``fs_native`` -- Native C++ file-system connector" +msgstr "" + +#: ../../source/mp/l2_storage.rst:255 +msgid "" +"A file-system L2 adapter backed by the native C++ ``LMCacheFSClient`` " +"wrapped with ``NativeConnectorL2Adapter``. I/O is dispatched through a " +"C++ worker-thread pool with eventfd-driven completions, giving a true I/O" +" queue depth on a single Python thread." +msgstr "" + +#: ../../source/mp/l2_storage.rst:266 +msgid "" +"``num_workers`` (int, default ``4``, > 0): Number of C++ worker threads " +"inside the connector. This is the real I/O queue depth -- raise to push " +"throughput on filesystems whose aggregate BW exceeds per-stream BW." +msgstr "" + +#: ../../source/mp/l2_storage.rst:270 +msgid "" +"``relative_tmp_dir`` (str, default ``\"\"``): Relative sub-directory for " +"temporary files during writes (atomic rename on completion)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:272 +msgid "" +"``use_odirect`` (bool, default ``false``): Bypass the page cache via " +"``O_DIRECT``. Required to measure real disk bandwidth. See alignment " +"caveat below." +msgstr "" + +#: ../../source/mp/l2_storage.rst:275 +msgid "" +"``read_ahead_size`` (int, optional): Trigger filesystem readahead by " +"issuing a warm-up read of this many bytes at open time." +msgstr "" + +#: ../../source/mp/l2_storage.rst:277 +msgid "" +"``max_capacity_gb`` (float, default ``0``): Maximum L2 capacity in GB for" +" client-side usage tracking. Default ``0`` disables tracking." +msgstr "" + +#: ../../source/mp/l2_storage.rst:282 +msgid "``O_DIRECT`` has two independent alignment requirements:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:284 +#, python-format +msgid "" +"**Length alignment.** The transfer length must be a multiple of the " +"filesystem's block size. The connector queries the disk block size at " +"construction time and, on each operation, checks ``len % " +"disk_block_size``. If the length is **not** a multiple, the connector " +"silently falls back to a buffered open (no ``O_DIRECT``) for that " +"operation -- correctness is preserved but you do not get true direct I/O." +" To ensure ``O_DIRECT`` is actually used, choose ``--chunk-size`` so " +"that the resulting per-chunk byte size is a multiple of the FS block " +"size. GPFS and similar parallel filesystems often use large blocks (e.g." +" several MiB)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:296 +msgid "" +"**Memory-buffer alignment.** The I/O buffer pointer itself must also be " +"aligned (typically to 4096 bytes on local disks, or to the FS block size " +"on parallel filesystems). This is controlled by ``--l1-align-bytes`` " +"(default ``4096``) -- raise it to match the FS block size when running on" +" a filesystem with larger blocks. If the buffer is misaligned, the " +"underlying ``read``/``write`` syscall returns ``EINVAL`` (this is **not**" +" caught by the length-fallback path above and will surface as a runtime " +"error)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:305 +msgid "" +"If unsure, start with ``use_odirect: false`` and confirm correctness " +"before enabling ``O_DIRECT``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:321 +msgid "" +"**Buffer-only mode example.** L1 acts as a pure write buffer that " +"absorbs the peak burst of in-flight chunks while the C++ worker pool " +"drains them to disk; nothing is retained in L1 once a store completes:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:336 +msgid "``raw_block`` -- Raw block device backed persistent storage" +msgstr "" + +#: ../../source/mp/l2_storage.rst:338 +msgid "" +"A built-in L2 adapter that stores KV objects in fixed-size slots on a raw" +" block device or pre-sized file using the Rust raw-device I/O bindings. " +"It reuses the existing raw-block metadata checkpoint model and writes " +"directly into the caller-provided load buffers during prefetch." +msgstr "" + +#: ../../source/mp/l2_storage.rst:345 +msgid "``device_path``: Raw device path or pre-sized file path." +msgstr "" + +#: ../../source/mp/l2_storage.rst:346 +msgid "" +"``slot_bytes``: Fixed slot size in bytes. Must be aligned to " +"``block_align``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:350 +msgid "" +"``capacity_bytes``: Optional cap on the usable device bytes. Default " +"``0`` means use the full device/file size." +msgstr "" + +#: ../../source/mp/l2_storage.rst:352 +msgid "``use_odirect``: ``true`` or ``false`` (default ``true``)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:353 +msgid "``block_align``: Device alignment in bytes (default ``4096``)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:354 +msgid "``header_bytes``: Per-slot header reservation (default ``4096``)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:355 +msgid "" +"``meta_total_bytes``: Reserved metadata checkpoint region (default " +"``256MiB``)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:356 +msgid "" +"``meta_magic`` / ``meta_version``: Metadata checkpoint identity/version " +"knobs." +msgstr "" + +#: ../../source/mp/l2_storage.rst:357 +msgid "" +"``meta_checkpoint_interval_sec`` / ``meta_idle_quiet_ms`` / " +"``meta_enable_periodic`` / ``meta_verify_on_load``: Checkpoint and " +"recovery controls carried over from the legacy raw-block backend." +msgstr "" + +#: ../../source/mp/l2_storage.rst:360 +msgid "" +"``load_checkpoint_on_init``: Load an existing on-device metadata " +"checkpoint during startup (default ``true``). Set to ``false`` to start " +"with an empty in-memory index instead." +msgstr "" + +#: ../../source/mp/l2_storage.rst:363 +msgid "``enable_zero_copy``: Try aligned direct-buffer I/O when possible." +msgstr "" + +#: ../../source/mp/l2_storage.rst:364 +msgid "" +"``io_engine``: Rust raw-block I/O engine. Valid values are ``\"posix\"`` " +"(default synchronous ``pread``/``pwrite`` path), ``\"io_uring\"`` (direct" +" Rust io_uring syscall path)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:367 +msgid "``iouring_queue_depth``: Queue depth for ``io_engine=\"io_uring\"``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:368 +msgid "" +"``num_store_workers`` / ``num_lookup_workers`` / ``num_load_workers``: " +"Worker-thread counts for each operation type." +msgstr "" + +#: ../../source/mp/l2_storage.rst:371 +msgid "**Notes:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:373 +msgid "" +"``raw_block`` is a server-owned MP adapter. It does **not** support per-" +"TP device-path mappings in MP mode." +msgstr "" + +#: ../../source/mp/l2_storage.rst:375 +msgid "" +"``raw_block`` remains ``\"type\": \"raw_block\"`` for both supported " +"engines." +msgstr "" + +#: ../../source/mp/l2_storage.rst:376 +msgid "" +"``raw_block`` owns on-device slot allocation, checkpointing, and recovery" +" through ``RawBlockCore``. Slot reclamation is driven by the " +"shared/global L2 eviction controller or explicit ``delete()`` calls." +msgstr "" + +#: ../../source/mp/l2_storage.rst:379 +msgid "" +"If ``use_odirect`` is enabled, the server's ``--l1-align-bytes`` should " +"be at least ``block_align``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:381 +msgid "``persist_enabled`` must remain ``true`` for this adapter." +msgstr "" + +#: ../../source/mp/l2_storage.rst:394 +msgid "``mooncake_store`` -- Mooncake Store native connector" +msgstr "" + +#: ../../source/mp/l2_storage.rst:396 +msgid "" +"An L2 adapter backed by the native C++ Mooncake Store connector. Uses " +"`Mooncake `_ for high-performance" +" distributed KV cache storage with RDMA support." +msgstr "" + +#: ../../source/mp/l2_storage.rst:400 +msgid "" +"When Mooncake is configured with ``\"protocol\": \"rdma\"``, LMCache must" +" also have a valid contiguous L1 memory region available. The " +"distributed storage manager passes this L1 memory descriptor to the " +"adapter factory automatically in MP mode. If the descriptor is missing " +"or invalid, adapter creation fails with ``ValueError`` instead of " +"silently falling back to a non-RDMA path." +msgstr "" + +#: ../../source/mp/l2_storage.rst:406 +msgid "**Prerequisites -- Building with Mooncake support:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:408 +msgid "" +"The Mooncake extension is **not** built by default. You must explicitly " +"enable it:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:415 +msgid "The ``BUILD_MOONCAKE`` environment variable controls compilation:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:417 +msgid "``BUILD_MOONCAKE=1``: Enable the Mooncake C++ extension." +msgstr "" + +#: ../../source/mp/l2_storage.rst:418 +msgid "" +"``BUILD_MOONCAKE=0``: Force disable (highest priority), even if " +"``MOONCAKE_INCLUDE_DIR`` is set." +msgstr "" + +#: ../../source/mp/l2_storage.rst:420 +msgid "" +"**Not set**: Falls back to checking ``MOONCAKE_INCLUDE_DIR`` for backward" +" compatibility. If ``MOONCAKE_INCLUDE_DIR`` is also unset, the extension" +" is skipped." +msgstr "" + +#: ../../source/mp/l2_storage.rst:424 +msgid "" +"If the Mooncake headers are not installed in the system include path " +"(e.g., ``/usr/local/include``), you must point to them explicitly:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:434 +msgid "**LMCache-specific fields:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:436 +msgid "" +"``num_workers``: Number of C++ worker threads for the shared pool " +"(default ``4``, must be > 0)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:439 +msgid "" +"``per_op_workers`` (``dict[str, int]``, optional): A dict mapping lane " +"keys to dedicated worker thread counts. Supported keys:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:442 +msgid "``\"lookup\"`` — threads for ``EXISTS`` operations." +msgstr "" + +#: ../../source/mp/l2_storage.rst:443 +msgid "``\"retrieve\"`` — threads for ``GET`` / load operations." +msgstr "" + +#: ../../source/mp/l2_storage.rst:444 +msgid "``\"store\"`` — threads for ``SET`` / put operations." +msgstr "" + +#: ../../source/mp/l2_storage.rst:445 +msgid "``\"delete\"`` — threads for ``DELETE`` operations." +msgstr "" + +#: ../../source/mp/l2_storage.rst:447 +msgid "" +"Operations whose lane key is **not** present in the dict use the shared " +"``num_workers`` pool. There is no requirement to set all keys — you can " +"configure only the lanes that need dedicated pools." +msgstr "" + +#: ../../source/mp/l2_storage.rst:451 +msgid "**Mooncake fields:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:453 +msgid "" +"All other keys in the JSON config (except ``type``, ``num_workers``, " +"``per_op_workers``, and ``eviction``) are forwarded **as-is** to " +"Mooncake's ``setup_internal(ConfigDict)``. Refer to the `Mooncake " +"documentation `_ for available " +"setup keys (e.g., ``local_hostname``, ``metadata_server``, " +"``master_server_addr``, ``protocol``, ``rdma_devices``, " +"``global_segment_size``)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:491 +msgid "" +"For full Mooncake setup instructions (master service, metadata server, " +"etc.), see `Mooncake `_ ." +msgstr "" + +#: ../../source/mp/l2_storage.rst:494 +msgid "**RDMA notes:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:496 +msgid "``protocol: \"rdma\"`` requires a valid LMCache L1 memory descriptor." +msgstr "" + +#: ../../source/mp/l2_storage.rst:497 +msgid "" +"When using ``protocol: \"rdma\"``, it is recommended to disable lazy L1 " +"allocation with ``--no-l1-use-lazy`` so the L1 buffer is fully allocated " +"before Mooncake registers it." +msgstr "" + +#: ../../source/mp/l2_storage.rst:500 +msgid "``protocol: \"tcp\"`` does not require L1 preregistration." +msgstr "" + +#: ../../source/mp/l2_storage.rst:501 +msgid "" +"If Mooncake RDMA initialization fails at adapter creation time, verify " +"that LMCache L1 memory is enabled and that the descriptor has a non-zero " +"pointer and size." +msgstr "" + +#: ../../source/mp/l2_storage.rst:506 +msgid "``s3`` -- S3-compatible object store" +msgstr "" + +#: ../../source/mp/l2_storage.rst:508 +msgid "" +"An L2 adapter that stores KV cache objects as S3 objects using the AWS " +"Common Runtime (CRT). Works with AWS S3, S3 Express One Zone, and any " +"S3-compatible endpoint (MinIO, Ceph RGW, etc.)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:514 +msgid "" +"``s3_endpoint``: Bucket URL -- either ``\"s3://\"`` or the bare " +"host form (used for non-AWS endpoints)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:516 +msgid "``s3_region``: AWS region string (e.g. ``\"us-west-2\"``)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:520 +msgid "``s3_num_io_threads`` (int, default ``64``): Number of CRT I/O threads." +msgstr "" + +#: ../../source/mp/l2_storage.rst:521 +msgid "``s3_prefer_http2`` (bool, default ``true``): Negotiate HTTP/2 via ALPN." +msgstr "" + +#: ../../source/mp/l2_storage.rst:522 +msgid "" +"``s3_enable_s3express`` (bool, default ``false``): Enable S3 Express " +"signing for S3 Express One Zone buckets." +msgstr "" + +#: ../../source/mp/l2_storage.rst:524 +msgid "" +"``disable_tls`` (bool, default ``false``): Bypass TLS when pointing at a " +"plain-HTTP endpoint (e.g. a local MinIO)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:526 +msgid "" +"``aws_access_key_id`` / ``aws_secret_access_key`` (string): Static " +"credentials; omit both to use the AWS default credential provider chain " +"(environment, EC2 instance profile, etc.)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:529 +msgid "" +"``max_capacity_gb`` (float, default ``0.0``): Aggregate capacity used by " +"``get_usage()``. A value of ``0`` disables aggregate eviction " +"(``usage_fraction == -1.0``)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:547 +msgid "``mock`` -- Mock adapter for testing" +msgstr "" + +#: ../../source/mp/l2_storage.rst:549 +msgid "" +"Simulates L2 storage with configurable size and bandwidth. Useful for " +"testing the L2 pipeline without real storage hardware." +msgstr "" + +#: ../../source/mp/l2_storage.rst:552 +msgid "**Fields:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:554 +msgid "``max_size_gb``: Maximum size in GB (> 0)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:555 +msgid "``mock_bandwidth_gb``: Simulated bandwidth in GB/sec (> 0)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:562 +msgid "Multiple Adapters (Cascade)" +msgstr "" + +#: ../../source/mp/l2_storage.rst:564 +msgid "" +"You can configure multiple L2 adapters by repeating the ``--l2-adapter`` " +"argument. Adapters are used in the order they are specified. The " +"``StoreController`` pushes data to all configured adapters, and the " +"``PrefetchController`` queries adapters in order during lookups." +msgstr "" + +#: ../../source/mp/l2_storage.rst:576 +msgid "Store and Prefetch Policies" +msgstr "" + +#: ../../source/mp/l2_storage.rst:578 +msgid "" +"The **store policy** controls how keys flow from L1 to L2: which adapters" +" receive each key and whether keys are deleted from L1 after a successful" +" L2 store. The **prefetch policy** controls how keys flow from L2 back " +"to L1: when multiple adapters have the same key, the policy decides which" +" adapter loads it." +msgstr "" + +#: ../../source/mp/l2_storage.rst:584 +msgid "Select policies via CLI:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:591 +msgid "**Built-in policies:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:597 ../../source/mp/l2_storage.rst:630 +#: ../../source/mp/l2_storage.rst:684 +msgid "Flag" +msgstr "" + +#: ../../source/mp/l2_storage.rst:598 +msgid "Name" +msgstr "" + +#: ../../source/mp/l2_storage.rst:599 +msgid "Behaviour" +msgstr "" + +#: ../../source/mp/l2_storage.rst:600 ../../source/mp/l2_storage.rst:603 +msgid "``--l2-store-policy``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:601 ../../source/mp/l2_storage.rst:609 +msgid "``default``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:602 +msgid "Store all keys to all adapters. Never delete from L1." +msgstr "" + +#: ../../source/mp/l2_storage.rst:604 +msgid "``skip_l1``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:605 +msgid "" +"Buffer-only mode. Store all keys to all adapters, then **delete them " +"from L1** immediately. Pair with ``--eviction-policy noop`` to avoid " +"useless LRU overhead." +msgstr "" + +#: ../../source/mp/l2_storage.rst:608 ../../source/mp/l2_storage.rst:612 +msgid "``--l2-prefetch-policy``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:610 +msgid "" +"For each key, pick the first (lowest-indexed) adapter that has it. " +"Prefetched keys are **temporary** (deleted after the reader finishes)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:613 +msgid "``retain``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:614 +msgid "" +"Same load plan as ``default``, but prefetched keys are **retained** " +"permanently in L1. Useful when prefetched data is likely reused by " +"subsequent requests (e.g. shared system-prompt chunks)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:619 +msgid "Prefetch Concurrency" +msgstr "" + +#: ../../source/mp/l2_storage.rst:621 +msgid "" +"The ``--l2-prefetch-max-in-flight`` flag limits the number of concurrent " +"prefetch requests that the ``PrefetchController`` can have in flight at " +"any time. A higher value increases L2-to-L1 throughput but also " +"increases L1 memory pressure from in-flight data." +msgstr "" + +#: ../../source/mp/l2_storage.rst:631 ../../source/mp/l2_storage.rst:685 +#: ../../source/mp/l2_storage.rst:725 +msgid "Default" +msgstr "" + +#: ../../source/mp/l2_storage.rst:633 +msgid "``--l2-prefetch-max-in-flight``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:634 +msgid "``8``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:635 +msgid "Maximum number of concurrent prefetch requests." +msgstr "" + +#: ../../source/mp/l2_storage.rst:638 +msgid "Buffer-Only Mode" +msgstr "" + +#: ../../source/mp/l2_storage.rst:640 +msgid "" +"When L1 is used purely as a write buffer (all data lives in L2), use " +"``--l2-store-policy skip_l1`` together with ``--eviction-policy noop``. " +"This combination deletes keys from L1 as soon as they are stored to L2 " +"and disables the LRU eviction tracker entirely, reducing memory and CPU " +"overhead." +msgstr "" + +#: ../../source/mp/l2_storage.rst:652 +msgid "" +"Policies are extensible -- new policies can be added by creating a file " +"in ``storage_controllers/`` and calling ``register_store_policy()`` or " +"``register_prefetch_policy()`` at import time. See the design doc " +"``l2_adapters/design_docs/overall.md`` for details." +msgstr "" + +#: ../../source/mp/l2_storage.rst:658 +msgid "Serde (compression / quantization)" +msgstr "" + +#: ../../source/mp/l2_storage.rst:660 +msgid "" +"Each adapter can optionally run a **serde** (serializer / deserializer) " +"that transforms data on the way in and out of L2 — e.g. fp8 quantization " +"for disk backends, or encryption for remote adapters. See :doc:`serde` " +"for details and configuration." +msgstr "" + +#: ../../source/mp/l2_storage.rst:666 +msgid "Eviction" +msgstr "" + +#: ../../source/mp/l2_storage.rst:668 +msgid "" +"LMCache supports eviction at both storage tiers so that each tier can " +"operate within a fixed capacity budget." +msgstr "" + +#: ../../source/mp/l2_storage.rst:672 +msgid "L1 Eviction" +msgstr "" + +#: ../../source/mp/l2_storage.rst:674 +msgid "" +"L1 eviction runs a single background thread that monitors overall L1 " +"memory usage. When usage exceeds ``trigger_watermark``, the eviction " +"policy evicts a fraction of the least-recently-used keys." +msgstr "" + +#: ../../source/mp/l2_storage.rst:678 +msgid "**CLI flags:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:687 +msgid "``--eviction-policy``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:688 ../../source/mp/l2_storage.rst:728 +msgid "*(required)*" +msgstr "" + +#: ../../source/mp/l2_storage.rst:689 +msgid "Policy name: ``LRU`` or ``noop``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:690 +msgid "``--eviction-trigger-watermark``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:691 ../../source/mp/l2_storage.rst:731 +msgid "``0.8``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:692 +msgid "L1 usage fraction [0, 1] above which eviction is triggered." +msgstr "" + +#: ../../source/mp/l2_storage.rst:693 +msgid "``--eviction-ratio``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:694 ../../source/mp/l2_storage.rst:734 +msgid "``0.2``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:695 +msgid "Fraction of currently allocated L1 memory to evict per cycle." +msgstr "" + +#: ../../source/mp/l2_storage.rst:697 +msgid "**Example:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:706 +msgid "L2 Eviction" +msgstr "" + +#: ../../source/mp/l2_storage.rst:708 +msgid "" +"L2 eviction is **per-adapter** and **opt-in**. Each adapter can " +"independently declare an eviction policy by adding an ``\"eviction\"`` " +"sub-object to its ``--l2-adapter`` JSON spec. Adapters without an " +"``\"eviction\"`` key have no eviction controller." +msgstr "" + +#: ../../source/mp/l2_storage.rst:713 +msgid "" +"When L2 eviction is enabled for an adapter, a dedicated background thread" +" monitors that adapter's ``get_usage()`` value. Once usage exceeds " +"``trigger_watermark``, the policy evicts keys until usage drops by " +"``eviction_ratio``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:718 +msgid "**``\"eviction\"`` sub-object fields:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:724 +msgid "Field" +msgstr "" + +#: ../../source/mp/l2_storage.rst:727 +msgid "``eviction_policy``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:729 +msgid "Policy name: ``\"LRU\"`` or ``\"noop\"``." +msgstr "" + +#: ../../source/mp/l2_storage.rst:730 +msgid "``trigger_watermark``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:732 +msgid "Adapter usage fraction [0, 1] above which eviction is triggered." +msgstr "" + +#: ../../source/mp/l2_storage.rst:733 +msgid "``eviction_ratio``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:735 +msgid "Fraction of used capacity to evict per cycle." +msgstr "" + +#: ../../source/mp/l2_storage.rst:737 +msgid "**Example — nixl_store with LRU eviction:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:753 +msgid "**Adapter support:**" +msgstr "" + +#: ../../source/mp/l2_storage.rst:759 +msgid "Adapter" +msgstr "" + +#: ../../source/mp/l2_storage.rst:760 +msgid "L2 Eviction Support" +msgstr "" + +#: ../../source/mp/l2_storage.rst:761 +msgid "``nixl_store``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:762 +msgid "" +"Full support. ``delete`` frees pool slots; pinned keys (in-flight loads) " +"are skipped and retried on the next cycle." +msgstr "" + +#: ../../source/mp/l2_storage.rst:764 +msgid "``nixl_store_dynamic``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:765 +msgid "" +"Full support. ``delete`` removes data files from disk; pinned keys are " +"skipped. ``get_usage`` is byte-based (``_total_bytes / " +"max_capacity_bytes``)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:768 +msgid "``mock``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:769 +msgid "" +"Full support. Useful for testing eviction behaviour without real storage " +"hardware." +msgstr "" + +#: ../../source/mp/l2_storage.rst:771 +msgid "``raw_block``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:772 +msgid "" +"Full shared/global eviction support. ``delete`` recycles raw-block slots;" +" locked entries are skipped and retried on the next cycle." +msgstr "" + +#: ../../source/mp/l2_storage.rst:774 +msgid "``s3``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:775 +msgid "" +"``delete`` removes objects from the bucket and frees aggregate byte " +"accounting. ``get_usage`` reports ``usage_fraction == -1.0`` when " +"``max_capacity_gb`` is ``0`` (disabled); set a non-zero " +"``max_capacity_gb`` to enable the watermark-triggered eviction " +"controller." +msgstr "" + +#: ../../source/mp/l2_storage.rst:780 +msgid "``dax``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:781 +msgid "" +"Full support. ``delete`` removes unlocked keys from the in-memory index " +"immediately and recycles fixed slots once active read borrows drain. " +"Usage is slot-based." +msgstr "" + +#: ../../source/mp/l2_storage.rst:784 +msgid "``mooncake_store``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:785 +msgid "No eviction support (native connector adapter)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:786 +msgid "``fs``" +msgstr "" + +#: ../../source/mp/l2_storage.rst:787 +msgid "No eviction support (``delete`` and ``get_usage`` are no-ops)." +msgstr "" + +#: ../../source/mp/l2_storage.rst:788 +msgid "native connectors" +msgstr "" + +#: ../../source/mp/l2_storage.rst:789 +msgid "No eviction support." +msgstr "" + +#: ../../source/mp/l2_storage.rst:793 +msgid "" +"Each L2 adapter instance gets its own independent eviction controller and" +" policy. Two adapters of the same type can have different watermarks or " +"policies." +msgstr "" + +#: ../../source/mp/l2_storage.rst:798 +msgid "Combined L1 + L2 Eviction Example" +msgstr "" + +#: ../../source/mp/l2_storage.rst:818 +msgid "In this setup:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:820 +#, python-format +msgid "" +"L1 evicts from memory when it is 80 % full, reclaiming 20 % of allocated " +"memory per cycle." +msgstr "" + +#: ../../source/mp/l2_storage.rst:822 +#, python-format +msgid "" +"L2 (NIXL/GDS) evicts from the storage pool when 90 % of pool slots are " +"occupied, reclaiming 10 % per cycle." +msgstr "" + +#: ../../source/mp/l2_storage.rst:824 +msgid "" +"Both tiers use independent LRU policies, so each evicts its own least-" +"recently-used keys." +msgstr "" + +#: ../../source/mp/l2_storage.rst:828 +msgid "Verifying L2 Storage" +msgstr "" + +#: ../../source/mp/l2_storage.rst:830 +msgid "Set ``LMCACHE_LOG_LEVEL=DEBUG`` to see L2 activity in the server logs:" +msgstr "" + +#: ../../source/mp/l2_storage.rst:838 +msgid "Expected log messages when L2 is active:" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/observability.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/observability.po new file mode 100644 index 00000000000..153ff743e85 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/observability.po @@ -0,0 +1,1458 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/observability.rst:2 +msgid "Observability" +msgstr "" + +#: ../../source/mp/observability.rst:4 +msgid "" +"LMCache multiprocess mode provides three complementary observability " +"modes: **metrics** (Prometheus counters via OTel), **logging** (Python " +"logging with optional OTel log forwarding), and **tracing** (OTel spans " +"for per-request latency)." +msgstr "" + +#: ../../source/mp/observability.rst:9 +msgid "" +"All three modes are powered by an internal **EventBus** that decouples " +"producers (L1Manager, StorageManager, MPCacheEngine) from subscribers." +msgstr "" + +#: ../../source/mp/observability.rst:17 +msgid "Quick Start" +msgstr "" + +#: ../../source/mp/observability.rst:19 +msgid "" +"By default, **metrics** and **logging** are enabled; **tracing** is " +"disabled. No extra flags are needed:" +msgstr "" + +#: ../../source/mp/observability.rst:27 +msgid "To enable tracing, supply an OTLP endpoint:" +msgstr "" + +#: ../../source/mp/observability.rst:36 +msgid "Configuration" +msgstr "" + +#: ../../source/mp/observability.rst:42 +msgid "Argument" +msgstr "" + +#: ../../source/mp/observability.rst:43 ../../source/mp/observability.rst:98 +msgid "Default" +msgstr "" + +#: ../../source/mp/observability.rst:44 ../../source/mp/observability.rst:99 +#: ../../source/mp/observability.rst:152 ../../source/mp/observability.rst:181 +#: ../../source/mp/observability.rst:218 ../../source/mp/observability.rst:253 +#: ../../source/mp/observability.rst:275 ../../source/mp/observability.rst:338 +#: ../../source/mp/observability.rst:384 ../../source/mp/observability.rst:436 +#: ../../source/mp/observability.rst:470 ../../source/mp/observability.rst:507 +#: ../../source/mp/observability.rst:529 ../../source/mp/observability.rst:560 +#: ../../source/mp/observability.rst:618 ../../source/mp/observability.rst:738 +msgid "Description" +msgstr "" + +#: ../../source/mp/observability.rst:45 +msgid "``--disable-observability``" +msgstr "" + +#: ../../source/mp/observability.rst:46 ../../source/mp/observability.rst:50 +#: ../../source/mp/observability.rst:53 ../../source/mp/observability.rst:56 +msgid "off" +msgstr "" + +#: ../../source/mp/observability.rst:47 +msgid "" +"Master switch: disable the EventBus entirely (no metrics, logging, or " +"tracing subscribers are registered)." +msgstr "" + +#: ../../source/mp/observability.rst:49 +msgid "``--disable-metrics``" +msgstr "" + +#: ../../source/mp/observability.rst:51 +msgid "Skip metrics subscribers (Prometheus endpoint is not started)." +msgstr "" + +#: ../../source/mp/observability.rst:52 +msgid "``--disable-logging``" +msgstr "" + +#: ../../source/mp/observability.rst:54 +msgid "Skip logging subscribers." +msgstr "" + +#: ../../source/mp/observability.rst:55 +msgid "``--enable-tracing``" +msgstr "" + +#: ../../source/mp/observability.rst:57 +msgid "Register tracing subscribers. Requires ``--otlp-endpoint``." +msgstr "" + +#: ../../source/mp/observability.rst:58 +msgid "``--event-bus-queue-size``" +msgstr "" + +#: ../../source/mp/observability.rst:59 +msgid "``10000``" +msgstr "" + +#: ../../source/mp/observability.rst:60 +msgid "Maximum events in the EventBus queue before tail-drop." +msgstr "" + +#: ../../source/mp/observability.rst:61 +msgid "``--otlp-endpoint``" +msgstr "" + +#: ../../source/mp/observability.rst:62 ../../source/mp/observability.rst:80 +#: ../../source/mp/observability.rst:86 +msgid "*(none)*" +msgstr "" + +#: ../../source/mp/observability.rst:63 +msgid "" +"OTLP gRPC endpoint (e.g. ``http://localhost:4317``). Used for exporting " +"metrics (push mode) and traces." +msgstr "" + +#: ../../source/mp/observability.rst:65 +msgid "``--prometheus-port``" +msgstr "" + +#: ../../source/mp/observability.rst:66 +msgid "``9090``" +msgstr "" + +#: ../../source/mp/observability.rst:67 +msgid "Port for the Prometheus ``/metrics`` HTTP endpoint." +msgstr "" + +#: ../../source/mp/observability.rst:68 +msgid "``--service-instance-id``" +msgstr "" + +#: ../../source/mp/observability.rst:69 +msgid "*(unset, default UUID v4)*" +msgstr "" + +#: ../../source/mp/observability.rst:70 +msgid "" +"Identifier for this MP server instance. Attached as the OTel Resource " +"attribute ``service.instance.id`` on every metric and span. When the flag" +" is not passed, defaults to a random UUID v4 minted at startup. Pass " +"``--service-instance-id=\"\"`` to force an explicit empty value. See :ref" +":`mp-observability-resource`." +msgstr "" + +#: ../../source/mp/observability.rst:75 +msgid "``--metrics-sample-rate``" +msgstr "" + +#: ../../source/mp/observability.rst:76 +msgid "``0.01``" +msgstr "" + +#: ../../source/mp/observability.rst:77 +msgid "" +"Fraction of chunks/blocks to track for lifecycle histograms (0, 1.0]. " +"Counters always count all events. Default is 1%." +msgstr "" + +#: ../../source/mp/observability.rst:79 +msgid "``--trace-level``" +msgstr "" + +#: ../../source/mp/observability.rst:81 +msgid "" +"Enable trace recording at the given level. Currently only ``storage`` is " +"supported (records ``StorageManager`` public-API calls for offline " +"replay). When unset, trace recording is off. See :ref:`trace-recording` " +"for details." +msgstr "" + +#: ../../source/mp/observability.rst:85 +msgid "``--trace-output``" +msgstr "" + +#: ../../source/mp/observability.rst:87 +msgid "" +"Path to write the trace file. If omitted while ``--trace-level`` is set, " +"a timestamped file under ``$TMPDIR`` is minted (``lmcache-" +"trace--.lct``) and its path is logged at INFO." +msgstr "" + +#: ../../source/mp/observability.rst:91 +msgid "**Environment variables:**" +msgstr "" + +#: ../../source/mp/observability.rst:97 +msgid "Variable" +msgstr "" + +#: ../../source/mp/observability.rst:100 +msgid "``LMCACHE_LOG_LEVEL``" +msgstr "" + +#: ../../source/mp/observability.rst:101 +msgid "``INFO``" +msgstr "" + +#: ../../source/mp/observability.rst:102 +msgid "" +"Controls the log level for all LMCache loggers. Valid values: ``DEBUG``, " +"``INFO``, ``WARNING``, ``ERROR``, ``CRITICAL``." +msgstr "" + +#: ../../source/mp/observability.rst:106 +msgid "Metrics" +msgstr "" + +#: ../../source/mp/observability.rst:108 +msgid "" +"Metrics are collected via OpenTelemetry counters and exported through an " +"in-process **Prometheus** ``/metrics`` HTTP endpoint (default port 9090)." +" When ``--otlp-endpoint`` is set, metrics are also pushed to the OTel " +"collector." +msgstr "" + +#: ../../source/mp/observability.rst:113 +msgid "" +"All metrics use the ``lmcache_mp.`` prefix (multiprocess). On Prometheus," +" dots are converted to underscores and counters get a ``_total`` suffix " +"(e.g. ``lmcache_mp_l1_read_keys_total``)." +msgstr "" + +#: ../../source/mp/observability.rst:120 +msgid "Global Resource Attributes" +msgstr "" + +#: ../../source/mp/observability.rst:122 +msgid "" +"Every metric and span exported by an MP server carries Resource-level " +"attributes built at startup. These identify the process producing the " +"telemetry and are orthogonal to per-metric attributes such as " +"``cache_salt``." +msgstr "" + +#: ../../source/mp/observability.rst:131 ../../source/mp/observability.rst:736 +msgid "Attribute" +msgstr "" + +#: ../../source/mp/observability.rst:132 +msgid "CLI flag / config" +msgstr "" + +#: ../../source/mp/observability.rst:133 +msgid "Default when unset" +msgstr "" + +#: ../../source/mp/observability.rst:134 +msgid "``service.instance.id``" +msgstr "" + +#: ../../source/mp/observability.rst:135 +msgid "``--service-instance-id`` / ``ObservabilityConfig.service_instance_id``" +msgstr "" + +#: ../../source/mp/observability.rst:136 +msgid "Random UUID v4 minted at startup." +msgstr "" + +#: ../../source/mp/observability.rst:138 +msgid "" +"Resource attributes attach to the ``MeterProvider`` / ``TracerProvider`` " +"and propagate to every exported datapoint via OTLP. On Prometheus, SDK " +"resource attributes surface on the ``target_info`` series rather than on " +"each time-series — this is standard OTel behavior." +msgstr "" + +#: ../../source/mp/observability.rst:144 +msgid "StorageManager Metrics" +msgstr "" + +#: ../../source/mp/observability.rst:150 ../../source/mp/observability.rst:179 +#: ../../source/mp/observability.rst:216 ../../source/mp/observability.rst:251 +#: ../../source/mp/observability.rst:273 ../../source/mp/observability.rst:336 +#: ../../source/mp/observability.rst:382 ../../source/mp/observability.rst:434 +#: ../../source/mp/observability.rst:468 ../../source/mp/observability.rst:505 +#: ../../source/mp/observability.rst:527 ../../source/mp/observability.rst:558 +#: ../../source/mp/observability.rst:616 +msgid "Metric" +msgstr "" + +#: ../../source/mp/observability.rst:151 ../../source/mp/observability.rst:180 +#: ../../source/mp/observability.rst:217 ../../source/mp/observability.rst:252 +#: ../../source/mp/observability.rst:274 ../../source/mp/observability.rst:337 +#: ../../source/mp/observability.rst:383 ../../source/mp/observability.rst:435 +#: ../../source/mp/observability.rst:469 ../../source/mp/observability.rst:506 +#: ../../source/mp/observability.rst:528 ../../source/mp/observability.rst:559 +#: ../../source/mp/observability.rst:617 +msgid "Type" +msgstr "" + +#: ../../source/mp/observability.rst:153 +msgid "``lmcache_mp.sm_read_requests``" +msgstr "" + +#: ../../source/mp/observability.rst:154 ../../source/mp/observability.rst:157 +#: ../../source/mp/observability.rst:160 ../../source/mp/observability.rst:163 +#: ../../source/mp/observability.rst:166 ../../source/mp/observability.rst:169 +#: ../../source/mp/observability.rst:183 ../../source/mp/observability.rst:186 +#: ../../source/mp/observability.rst:189 ../../source/mp/observability.rst:192 +#: ../../source/mp/observability.rst:196 ../../source/mp/observability.rst:277 +#: ../../source/mp/observability.rst:280 ../../source/mp/observability.rst:286 +#: ../../source/mp/observability.rst:289 ../../source/mp/observability.rst:292 +#: ../../source/mp/observability.rst:295 ../../source/mp/observability.rst:298 +#: ../../source/mp/observability.rst:301 ../../source/mp/observability.rst:304 +#: ../../source/mp/observability.rst:307 ../../source/mp/observability.rst:310 +#: ../../source/mp/observability.rst:340 ../../source/mp/observability.rst:346 +#: ../../source/mp/observability.rst:355 +msgid "Counter" +msgstr "" + +#: ../../source/mp/observability.rst:155 +msgid "Number of read (prefetch) requests received by the StorageManager." +msgstr "" + +#: ../../source/mp/observability.rst:156 +msgid "``lmcache_mp.sm_read_succeed_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:158 +msgid "Number of keys successfully read from LMCache." +msgstr "" + +#: ../../source/mp/observability.rst:159 +msgid "``lmcache_mp.sm_read_failed_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:161 +msgid "Number of keys that failed to read." +msgstr "" + +#: ../../source/mp/observability.rst:162 +msgid "``lmcache_mp.sm_write_requests``" +msgstr "" + +#: ../../source/mp/observability.rst:164 +msgid "Number of write (reserve) requests." +msgstr "" + +#: ../../source/mp/observability.rst:165 +msgid "``lmcache_mp.sm_write_succeed_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:167 +msgid "Number of keys successfully reserved for write." +msgstr "" + +#: ../../source/mp/observability.rst:168 +msgid "``lmcache_mp.sm_write_failed_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:170 +msgid "Number of keys that failed to reserve (OOM, write conflict)." +msgstr "" + +#: ../../source/mp/observability.rst:173 +msgid "L1 Metrics" +msgstr "" + +#: ../../source/mp/observability.rst:182 +msgid "``lmcache_mp.l1_read_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:184 +msgid "Number of keys read from L1." +msgstr "" + +#: ../../source/mp/observability.rst:185 +msgid "``lmcache_mp.l1_write_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:187 +msgid "Number of keys written to L1." +msgstr "" + +#: ../../source/mp/observability.rst:188 +msgid "``lmcache_mp.l1_evicted_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:190 +msgid "Number of keys evicted by the EvictionController." +msgstr "" + +#: ../../source/mp/observability.rst:191 +msgid "``lmcache_mp.l1_eviction_loop_ticks``" +msgstr "" + +#: ../../source/mp/observability.rst:193 +msgid "" +"L1 eviction-loop iterations (every cycle, regardless of whether the " +"watermark was crossed). Driven by ``L1_EVICTION_LOOP_TICK``." +msgstr "" + +#: ../../source/mp/observability.rst:195 +msgid "``lmcache_mp.l1_eviction_loop_triggered``" +msgstr "" + +#: ../../source/mp/observability.rst:197 +msgid "" +"L1 eviction-loop iterations where ``usage >= watermark`` and the eviction" +" policy actually ran. The two counters distinguish \"loop is alive\" from" +" \"eviction fired\" — important when debugging short-lived benchmarks " +"that complete faster than the 1 Hz polling cycle." +msgstr "" + +#: ../../source/mp/observability.rst:204 +msgid "L1 Chunk Lifecycle Histograms" +msgstr "" + +#: ../../source/mp/observability.rst:206 +msgid "" +"Sampled (default 1%) chunk-level lifecycle tracking via " +"``L1LifecycleSubscriber``. Only sampled chunks contribute to histograms; " +"counters above always count all events. Sampling is deterministic (hash-" +"based), so the same key always gets the same decision with zero memory " +"overhead." +msgstr "" + +#: ../../source/mp/observability.rst:219 +msgid "``lmcache_mp.l1_chunk_lifetime_seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:220 ../../source/mp/observability.rst:223 +#: ../../source/mp/observability.rst:226 ../../source/mp/observability.rst:229 +#: ../../source/mp/observability.rst:438 ../../source/mp/observability.rst:441 +#: ../../source/mp/observability.rst:444 ../../source/mp/observability.rst:472 +#: ../../source/mp/observability.rst:475 ../../source/mp/observability.rst:509 +#: ../../source/mp/observability.rst:512 +msgid "Histogram" +msgstr "" + +#: ../../source/mp/observability.rst:221 +msgid "Time from allocation to eviction per sampled chunk." +msgstr "" + +#: ../../source/mp/observability.rst:222 +msgid "``lmcache_mp.l1_chunk_idle_before_evict_seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:224 +msgid "Time from last access to eviction per sampled chunk." +msgstr "" + +#: ../../source/mp/observability.rst:225 +msgid "``lmcache_mp.l1_chunk_reuse_gap_seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:227 +msgid "Time gap between consecutive touches (read or write) of the same chunk." +msgstr "" + +#: ../../source/mp/observability.rst:228 +msgid "``lmcache_mp.l1_chunk_evict_reuse_gap_seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:230 +msgid "Time from eviction to next reuse (capped at 300 s)." +msgstr "" + +#: ../../source/mp/observability.rst:233 +msgid "StorageManager Real-Reuse Metrics" +msgstr "" + +#: ../../source/mp/observability.rst:235 +msgid "" +"Workload-level reuse histograms emitted by ``SMLifecycleSubscriber``, " +"driven by caller-facing StorageManager events " +"(``SM_READ_PREFETCHED_FINISHED``, ``SM_WRITE_FINISHED``). Internal read-" +"lock releases by the store/prefetch controllers are excluded so the " +"signal reflects user-driven access only." +msgstr "" + +#: ../../source/mp/observability.rst:241 +msgid "" +"Both histograms are tagged with ``cache_salt`` for per-tenant isolation." +" The per-salt access counter advances on every read and write of every " +"chunk (regardless of sampling) so the chunks-gap reflects true storage " +"volume; the histogram itself records gaps only for chunks that pass the " +"(deterministic, hash-based) sampling gate." +msgstr "" + +#: ../../source/mp/observability.rst:254 +msgid "``lmcache_mp.real_reuse_gap_seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:255 ../../source/mp/observability.rst:260 +msgid "Histogram (tag: ``cache_salt``)" +msgstr "" + +#: ../../source/mp/observability.rst:256 +msgid "" +"Time gap between a chunk's last access (read or write) and its next read." +" Captures storage cost — how long a stored chunk sat between accesses. " +"Emitted only on read events." +msgstr "" + +#: ../../source/mp/observability.rst:259 +msgid "``lmcache_mp.real_reuse_gap_chunks``" +msgstr "" + +#: ../../source/mp/observability.rst:261 +msgid "" +"Per-``cache_salt`` access-counter gap between two reads of the same " +"chunk. Captures storage volume — how many chunk-accesses occurred while " +"this chunk waited for its next read. Emitted on read events for sampled " +"chunks." +msgstr "" + +#: ../../source/mp/observability.rst:267 +msgid "L2 Metrics" +msgstr "" + +#: ../../source/mp/observability.rst:276 +msgid "``lmcache_mp.l2_store_tasks``" +msgstr "" + +#: ../../source/mp/observability.rst:278 +msgid "Number of L2 store tasks submitted." +msgstr "" + +#: ../../source/mp/observability.rst:279 +msgid "``lmcache_mp.l2_store_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:281 +msgid "Number of keys submitted for L2 store." +msgstr "" + +#: ../../source/mp/observability.rst:282 +msgid "``lmcache_mp.l2_store_completed``" +msgstr "" + +#: ../../source/mp/observability.rst:283 ../../source/mp/observability.rst:313 +msgid "Counter (attr: ``l2_name``)" +msgstr "" + +#: ../../source/mp/observability.rst:284 +msgid "Number of L2 store tasks completed, labeled by adapter type." +msgstr "" + +#: ../../source/mp/observability.rst:285 +msgid "``lmcache_mp.l2_store_succeeded_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:287 +msgid "Number of keys successfully stored to L2." +msgstr "" + +#: ../../source/mp/observability.rst:288 +msgid "``lmcache_mp.l2_store_failed_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:290 +msgid "Number of keys that failed to store to L2." +msgstr "" + +#: ../../source/mp/observability.rst:291 +msgid "``lmcache_mp.l2_prefetch_lookups``" +msgstr "" + +#: ../../source/mp/observability.rst:293 +msgid "Number of L2 prefetch lookup requests." +msgstr "" + +#: ../../source/mp/observability.rst:294 +msgid "``lmcache_mp.l2_prefetch_lookup_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:296 +msgid "Number of keys submitted for L2 prefetch lookup." +msgstr "" + +#: ../../source/mp/observability.rst:297 +msgid "``lmcache_mp.l2_prefetch_hit_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:299 +msgid "Number of prefix keys found in L2 lookup." +msgstr "" + +#: ../../source/mp/observability.rst:300 +msgid "``lmcache_mp.l2_prefetch_load_tasks``" +msgstr "" + +#: ../../source/mp/observability.rst:302 +msgid "Number of L2 prefetch load tasks submitted." +msgstr "" + +#: ../../source/mp/observability.rst:303 +msgid "``lmcache_mp.l2_prefetch_load_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:305 +msgid "Number of keys submitted for L2 load." +msgstr "" + +#: ../../source/mp/observability.rst:306 +msgid "``lmcache_mp.l2_prefetch_loaded_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:308 +msgid "Number of keys successfully loaded from L2." +msgstr "" + +#: ../../source/mp/observability.rst:309 +msgid "``lmcache_mp.l2_prefetch_failed_keys``" +msgstr "" + +#: ../../source/mp/observability.rst:311 +msgid "Number of keys that failed to load from L2." +msgstr "" + +#: ../../source/mp/observability.rst:312 +msgid "``lmcache_mp.l2_load_completed``" +msgstr "" + +#: ../../source/mp/observability.rst:314 +msgid "Number of per-adapter L2 load tasks completed, labeled by adapter type." +msgstr "" + +#: ../../source/mp/observability.rst:316 +#, python-brace-format +msgid "" +"The ``l2_name``-labeled counters (``l2_store_completed`` and " +"``l2_load_completed``) exist so dashboards can compute per-backend IOPS " +"on demand via " +"``rate(lmcache_mp_l2_store_completed_total{l2_name=\"...\"}[1m])`` (and " +"the equivalent for loads). No separate ``*_iops`` metric is exported; " +"keeping the raw counter lets dashboard users pick their own window." +msgstr "" + +#: ../../source/mp/observability.rst:323 +msgid "Failure & Health Counters" +msgstr "" + +#: ../../source/mp/observability.rst:325 +msgid "" +"Health-monitoring counters emitted on the dedicated ``lmcache_mp.health``" +" OTel meter. Driven by the ``L1FailureMetricsSubscriber`` and " +"``L2FailureMetricsSubscriber``, which are registered automatically when " +"metrics are enabled. All three counters carry ``model_name`` (extracted " +"from each ``ObjectKey``) so operators can slice per-model on the " +"Prometheus ``/metrics`` endpoint." +msgstr "" + +#: ../../source/mp/observability.rst:339 +msgid "``lmcache_mp.l1_allocation_failure``" +msgstr "" + +#: ../../source/mp/observability.rst:341 +#, python-brace-format +msgid "" +"L1 memory allocation failures (OOM) during ``reserve_write``. Tagged by " +"``during`` ∈ {``l1_store``, ``l2_prefetch``} to distinguish user-" +"initiated stores from prefetch-triggered allocations, plus " +"``model_name``." +msgstr "" + +#: ../../source/mp/observability.rst:345 +msgid "``lmcache_mp.l1_read_failure``" +msgstr "" + +#: ../../source/mp/observability.rst:347 +#, python-brace-format +msgid "" +"L1 ``reserve_read`` failures. Tagged by ``during`` ∈ {``l2_store``, " +"``l1_retrieve``}, ``reason`` ∈ {``not_found``, ``write_locked``}, plus " +"``model_name``. **Post-lookup anomaly counter**, not a cache-miss counter" +" — in MP mode ``reserve_read`` is only called after a successful lookup, " +"so any non-zero value indicates a lookup/reserve race or unexpected " +"eviction and should stay near zero in healthy operation." +msgstr "" + +#: ../../source/mp/observability.rst:354 +msgid "``lmcache_mp.l2_prefetch_failure``" +msgstr "" + +#: ../../source/mp/observability.rst:356 +#, python-brace-format +msgid "" +"Keys that L2 reported present at lookup but failed to land in L1. Tagged " +"by ``reason`` ∈ {``l1_oom``, ``not_found``} plus ``model_name``. " +"``l1_oom`` means L1 had no room to receive the prefetched object; " +"``not_found`` means the adapter returned no data despite a positive " +"lookup (e.g. concurrent delete)." +msgstr "" + +#: ../../source/mp/observability.rst:362 +msgid "" +"A ``reason=serde_failure`` value will be added to ``l2_prefetch_failure``" +" as an additive, non-breaking extension once L2 adapters distinguish " +"deserialization errors from missing objects — no dashboard migration " +"needed when that lands." +msgstr "" + +#: ../../source/mp/observability.rst:367 +msgid "" +"For the full design rationale (including which event types drive each " +"counter and why ``lmcache_instance_id`` is deferred), see " +"``docs/design/v1/mp_observability/METRICS.md`` in the source tree." +msgstr "" + +#: ../../source/mp/observability.rst:372 +msgid "Lookup Hit-Rate Metrics" +msgstr "" + +#: ../../source/mp/observability.rst:374 +msgid "" +"Token-level counters whose ratio gives the fraction of tokens requested " +"by a lookup that were served from either L1 or L2. L0 (GPU prefix cache) " +"is intentionally excluded — it is vLLM-owned and not observable from " +"LMCache." +msgstr "" + +#: ../../source/mp/observability.rst:385 +msgid "``lmcache_mp.lookup_requested_tokens``" +msgstr "" + +#: ../../source/mp/observability.rst:386 ../../source/mp/observability.rst:390 +msgid "Counter (attrs: ``model_name``, ``cache_salt``)" +msgstr "" + +#: ../../source/mp/observability.rst:387 +msgid "" +"Total tokens submitted for lookup (denominator of the L1+L2 token-level " +"hit rate). Only chunk-aligned tokens are counted." +msgstr "" + +#: ../../source/mp/observability.rst:389 +msgid "``lmcache_mp.lookup_hit_tokens``" +msgstr "" + +#: ../../source/mp/observability.rst:391 +msgid "" +"Total tokens found in L1 or L2 during lookup (numerator of the L1+L2 " +"token-level hit rate). Counts the contiguous prefix hit only." +msgstr "" + +#: ../../source/mp/observability.rst:394 +msgid "" +"Both counters are driven by the same event (``MP_LOOKUP_PREFETCH_END``), " +"so they always advance together per completed lookup. Early-exit lookups " +"contribute ``0`` to both, and abandoned lookups contribute to neither." +msgstr "" + +#: ../../source/mp/observability.rst:398 +msgid "" +"The ``model_name`` and ``cache_salt`` attributes are captured at lookup " +"time from ``IPCCacheEngineKey`` so dashboards can compute per-model or " +"per-tenant hit rate. ``cache_salt`` can be high-cardinality (one entry " +"per tenant or isolation domain); drop it at scrape time with " +"``metric_relabel_configs`` if storage cost matters." +msgstr "" + +#: ../../source/mp/observability.rst:404 +msgid "**PromQL for hit rate:**" +msgstr "" + +#: ../../source/mp/observability.rst:417 +msgid "L0 (GPU) Block Lifecycle Histograms" +msgstr "" + +#: ../../source/mp/observability.rst:419 +msgid "" +"Sampled (default 1%) GPU KV cache block lifecycle tracking via " +"``L0LifecycleSubscriber``. Eviction is detected at reallocation time " +"(when a block is assigned different tokens). Sampling uses random " +"selection with a ``_skipped`` set (bounded by the finite number of " +"physical GPU blocks)." +msgstr "" + +#: ../../source/mp/observability.rst:425 +#, python-brace-format +msgid "" +"All L0 histograms are emitted with ``instance_id`` and ``model_name`` " +"OTel attributes, enabling per-instance and per-model metric slicing in " +"Prometheus (e.g. " +"``lmcache_mp_l0_block_lifetime_seconds{instance_id=\"12345\",model_name" +"=\"llama-7b\"}``)." +msgstr "" + +#: ../../source/mp/observability.rst:437 +msgid "``lmcache_mp.l0_block_lifetime_seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:439 +msgid "Time from allocation to eviction per sampled GPU block." +msgstr "" + +#: ../../source/mp/observability.rst:440 +msgid "``lmcache_mp.l0_block_idle_before_evict_seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:442 +msgid "Time from last access to eviction per sampled GPU block." +msgstr "" + +#: ../../source/mp/observability.rst:443 +msgid "``lmcache_mp.l0_block_reuse_gap_seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:445 +msgid "Time gaps between consecutive accesses of the same GPU block." +msgstr "" + +#: ../../source/mp/observability.rst:448 +msgid "L0 ↔ L1 Throughput Histograms" +msgstr "" + +#: ../../source/mp/observability.rst:450 +#, python-brace-format +msgid "" +"Per-request throughput of GPU↔CPU copies via " +"``L0L1ThroughputSubscriber``. Every store/retrieve request contributes " +"one sample to the appropriate histogram: ``total_bytes / (end_ts - " +"start_ts)`` in GB/s. Timestamps come from " +"``MP_{STORE,RETRIEVE}_{START,END}`` events published on the GPU cupy " +"stream, so they reflect true GPU-stream copy time — not Python/lock " +"overhead." +msgstr "" + +#: ../../source/mp/observability.rst:458 +#, python-brace-format +msgid "" +"All throughput histograms are emitted with ``engine_id`` (vLLM worker " +"instance id), ``device`` (e.g. ``\"cuda:3\"``), and ``model_name`` OTel " +"attributes, enabling per-worker, per-device, and per-model slicing in " +"Prometheus (e.g. " +"``lmcache_mp_l0_l1_store_throughput_gbs{engine_id=\"0\",device=\"cuda:3\",model_name" +"=\"meta-llama/Llama-3.1-8B\"}``)." +msgstr "" + +#: ../../source/mp/observability.rst:471 +msgid "``lmcache_mp.l0_l1_store_throughput_gbs``" +msgstr "" + +#: ../../source/mp/observability.rst:473 +msgid "GPU→CPU (L0→L1) store throughput in GB/s per request." +msgstr "" + +#: ../../source/mp/observability.rst:474 +msgid "``lmcache_mp.l0_l1_load_throughput_gbs``" +msgstr "" + +#: ../../source/mp/observability.rst:476 +msgid "CPU→GPU (L1→L0) load throughput in GB/s per request." +msgstr "" + +#: ../../source/mp/observability.rst:479 +msgid "L1 ↔ L2 Throughput Histograms" +msgstr "" + +#: ../../source/mp/observability.rst:481 +msgid "" +"Per-task throughput of L1↔L2 transfers via ``L2ThroughputSubscriber``. " +"The store path correlates ``L2_STORE_SUBMITTED`` → ``L2_STORE_COMPLETED``" +" by ``(adapter_index, task_id)``. The load path correlates the per-" +"adapter ``L2_LOAD_TASK_SUBMITTED`` → ``L2_LOAD_TASK_COMPLETED`` events by" +" ``(request_id, adapter_index)``; the request-level " +"``L2_PREFETCH_LOAD_*`` events used by the key-count counters aggregate " +"across adapters and cannot be attributed to a specific ``l2_name``." +msgstr "" + +#: ../../source/mp/observability.rst:490 +msgid "" +"Timestamps span **submit → complete**, so the duration includes adapter " +"queue, network, and disk I/O — the value is *bytes / end-to-end latency*," +" not raw transfer rate. Use these histograms to compare adapter types and" +" catch regressions; use the L0↔L1 histograms when you need pure copy-time" +" throughput." +msgstr "" + +#: ../../source/mp/observability.rst:496 +#, python-brace-format +msgid "" +"All L1↔L2 throughput histograms carry a single ``l2_name`` OTel attribute" +" — the registered adapter type (e.g. ``\"fs\"``, ``\"nixl_store\"``, " +"``\"mooncake_store\"``) — enabling per-backend slicing in Prometheus " +"(e.g. ``lmcache_mp_l2_store_throughput_gbs{l2_name=\"nixl_store\"}``)." +msgstr "" + +#: ../../source/mp/observability.rst:508 +msgid "``lmcache_mp.l2_store_throughput_gbs``" +msgstr "" + +#: ../../source/mp/observability.rst:510 +msgid "L1→L2 store throughput in GB/s per task." +msgstr "" + +#: ../../source/mp/observability.rst:511 +msgid "``lmcache_mp.l2_load_throughput_gbs``" +msgstr "" + +#: ../../source/mp/observability.rst:513 +msgid "L2→L1 load throughput in GB/s per (request, adapter) pair." +msgstr "" + +#: ../../source/mp/observability.rst:516 +msgid "Engine Counters" +msgstr "" + +#: ../../source/mp/observability.rst:518 +msgid "" +"Worker-scoped counters tied to what the MP server delivers back to each " +"vLLM worker via ``retrieve()``. Labeled by ``worker_id`` (the vLLM " +"worker instance id) — distinct from any scheduler-scoped id that may " +"appear on other metrics." +msgstr "" + +#: ../../source/mp/observability.rst:530 +msgid "``lmcache_mp.num_chunks_loaded``" +msgstr "" + +#: ../../source/mp/observability.rst:531 +msgid "Counter (attrs: ``worker_id``, ``model_name``, ``cache_salt``)" +msgstr "" + +#: ../../source/mp/observability.rst:532 +msgid "" +"Total number of LMCache chunks loaded into the engine, summed over all " +"``retrieve()`` completions. Sliceable per worker, per model, and per " +"tenant / isolation domain (``cache_salt``). ``cache_salt`` may be high-" +"cardinality; drop it at scrape time with ``metric_relabel_configs`` if " +"storage cost matters." +msgstr "" + +#: ../../source/mp/observability.rst:539 +msgid "Observable Gauges" +msgstr "" + +#: ../../source/mp/observability.rst:541 +msgid "" +"Point-in-time state snapshots registered via ``register_gauge`` (pull-" +"based OTel observable gauges)." +msgstr "" + +#: ../../source/mp/observability.rst:544 +msgid "" +"The three in-flight metrics carry two attributes that distinguish " +"adapters even when more than one is registered with the same backend type" +" — same shape as ``lmcache_mp.l2_store_completed``:" +msgstr "" + +#: ../../source/mp/observability.rst:548 +msgid "" +"``l2_name`` — the registered adapter type (e.g. ``\"fs\"``, " +"``\"nixl_store\"``, ``\"mooncake_store\"``)." +msgstr "" + +#: ../../source/mp/observability.rst:550 +msgid "``adapter_index`` — position in the controller's adapter list." +msgstr "" + +#: ../../source/mp/observability.rst:552 +msgid "Adapters with no in-flight work emit no datapoint for that scrape." +msgstr "" + +#: ../../source/mp/observability.rst:561 +msgid "``lmcache_mp.active_prefetch_jobs``" +msgstr "" + +#: ../../source/mp/observability.rst:562 ../../source/mp/observability.rst:566 +#: ../../source/mp/observability.rst:571 ../../source/mp/observability.rst:620 +#: ../../source/mp/observability.rst:624 +msgid "ObservableGauge" +msgstr "" + +#: ../../source/mp/observability.rst:563 +msgid "" +"Number of prefetch jobs currently in-flight. A sustained high value may " +"indicate slow L2 backends or polling delays." +msgstr "" + +#: ../../source/mp/observability.rst:565 +msgid "``lmcache_mp.l1_memory_usage_bytes``" +msgstr "" + +#: ../../source/mp/observability.rst:567 +msgid "" +"Bytes currently held in L1. Rising without plateauing typically " +"indicates a leak; saturating at the configured ``--l1-size-gb`` indicates" +" working set exceeds capacity." +msgstr "" + +#: ../../source/mp/observability.rst:570 +msgid "``lmcache_mp.l1_usage_ratio``" +msgstr "" + +#: ../../source/mp/observability.rst:572 +msgid "" +"L1 used/total ratio (``0.0``–``1.0``), sampled at scrape time from " +"``L1Manager.get_memory_usage()``. Returns ``0.0`` when the gauge target " +"is not yet wired up or ``total_bytes`` is zero, so the callback never " +"raises during a scrape. Compare against the eviction watermark (default " +"``0.8``) to read whether the eviction loop is below or above its trigger " +"threshold." +msgstr "" + +#: ../../source/mp/observability.rst:578 +msgid "``lmcache_mp.num_inflight_l2_stores``" +msgstr "" + +#: ../../source/mp/observability.rst:579 ../../source/mp/observability.rst:584 +#: ../../source/mp/observability.rst:589 +msgid "ObservableGauge (attrs: ``l2_name``, ``adapter_index``)" +msgstr "" + +#: ../../source/mp/observability.rst:580 +msgid "" +"L2 store tasks currently executing, per adapter. Sustained non-zero " +"values indicate the adapter cannot keep up with the L1 → L2 write rate." +msgstr "" + +#: ../../source/mp/observability.rst:583 +msgid "``lmcache_mp.num_inflight_l2_loads``" +msgstr "" + +#: ../../source/mp/observability.rst:585 +msgid "" +"L2 → L1 prefetch load tasks currently executing, per adapter. Pair with " +"``num_inflight_l2_stores`` to see whether read or write traffic dominates" +" a given backend." +msgstr "" + +#: ../../source/mp/observability.rst:588 +msgid "``lmcache_mp.inflight_load_memory_usage_bytes``" +msgstr "" + +#: ../../source/mp/observability.rst:590 +msgid "" +"L1 bytes reserved by in-flight L2 → L1 prefetch loads, per adapter. " +"Rising in-flight bytes alongside rising ``l1_memory_usage_bytes`` is a " +"signal that prefetch reservations are crowding out cacheable data. Per-" +"adapter byte attribution follows each request's ``load_plan`` bitmap, so " +"summing across adapters never double-counts." +msgstr "" + +#: ../../source/mp/observability.rst:598 +msgid "EventBus Self-Monitoring" +msgstr "" + +#: ../../source/mp/observability.rst:600 +msgid "" +"Health metrics for the EventBus itself, registered by " +"``EventBusSelfMetricsSubscriber`` on the ``lmcache.event_bus`` OTel " +"meter. These metrics observe bus state directly via the ``EventBus`` " +"accessors and report on every OTel scrape — they are not driven by " +"events, so dropping or failing subscribers cannot silence them." +msgstr "" + +#: ../../source/mp/observability.rst:606 +msgid "" +"Use them to answer: is the EventBus keeping up with publishers, is " +"anything being dropped, and are any subscriber callbacks raising? A non-" +"zero ``dropped_events_total`` or a sustained non-zero " +"``drain_lag_seconds`` indicates the bus is at ``--event-bus-queue-size`` " +"and tail-dropping; raise that flag or investigate slow subscribers." +msgstr "" + +#: ../../source/mp/observability.rst:619 +msgid "``lmcache_mp.event_bus.queue_depth``" +msgstr "" + +#: ../../source/mp/observability.rst:621 +msgid "Events currently queued in the EventBus (``len(_queue)`` at scrape time)." +msgstr "" + +#: ../../source/mp/observability.rst:623 +msgid "``lmcache_mp.event_bus.drain_lag_seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:625 +msgid "" +"Seconds since the oldest queued event was published; ``0.0`` when empty." +" Rising values mean the drain thread is falling behind." +msgstr "" + +#: ../../source/mp/observability.rst:628 +msgid "``lmcache_mp.event_bus.dropped_events_total``" +msgstr "" + +#: ../../source/mp/observability.rst:629 +msgid "ObservableCounter" +msgstr "" + +#: ../../source/mp/observability.rst:630 +msgid "" +"Cumulative events dropped because the EventBus queue was at ``--event-" +"bus-queue-size``." +msgstr "" + +#: ../../source/mp/observability.rst:632 +msgid "``lmcache_mp.event_bus.subscriber_exceptions``" +msgstr "" + +#: ../../source/mp/observability.rst:633 +msgid "ObservableCounter (attr: ``subscriber_name``)" +msgstr "" + +#: ../../source/mp/observability.rst:634 +msgid "" +"Cumulative exceptions raised by subscriber callbacks during EventBus " +"dispatch, tagged by ``subscriber_name`` (the failing callback's owning " +"class for bound methods, or ``__qualname__`` for free functions)." +msgstr "" + +#: ../../source/mp/observability.rst:639 +msgid "" +"For the full design rationale and the in-process accessors that back each" +" metric see ``docs/design/v1/mp_observability/METRICS.md`` and " +"``docs/design/v1/mp_observability/event-bus.md`` in the source tree." +msgstr "" + +#: ../../source/mp/observability.rst:644 +msgid "Prometheus Scrape Configuration" +msgstr "" + +#: ../../source/mp/observability.rst:646 +msgid "Add the LMCache server as a Prometheus scrape target:" +msgstr "" + +#: ../../source/mp/observability.rst:656 +msgid "Logging" +msgstr "" + +#: ../../source/mp/observability.rst:658 +msgid "" +"Logging subscribers emit debug-level messages for store, retrieve, " +"lookup, L1, and StorageManager events via Python's standard ``logging`` " +"module." +msgstr "" + +#: ../../source/mp/observability.rst:661 +msgid "" +"When OpenTelemetry is installed, ``init_logger`` automatically attaches " +"an OTel ``LoggingHandler`` so that log records are forwarded to any " +"configured OTel ``LoggerProvider``. The handler respects the " +"``LMCACHE_LOG_LEVEL`` environment variable." +msgstr "" + +#: ../../source/mp/observability.rst:670 +msgid "Key log messages:" +msgstr "" + +#: ../../source/mp/observability.rst:676 +msgid "Level" +msgstr "" + +#: ../../source/mp/observability.rst:677 +msgid "Message" +msgstr "" + +#: ../../source/mp/observability.rst:678 ../../source/mp/observability.rst:680 +#: ../../source/mp/observability.rst:682 +msgid "INFO" +msgstr "" + +#: ../../source/mp/observability.rst:679 +msgid "``Stored N tokens in X seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:681 +msgid "``Retrieved N tokens in X seconds``" +msgstr "" + +#: ../../source/mp/observability.rst:683 +msgid "``Prefetch request completed (L1+L2): N/M prefix hits``" +msgstr "" + +#: ../../source/mp/observability.rst:684 ../../source/mp/observability.rst:686 +msgid "DEBUG" +msgstr "" + +#: ../../source/mp/observability.rst:685 +msgid "``MP store start: session=... device=...``" +msgstr "" + +#: ../../source/mp/observability.rst:687 +msgid "``MP retrieve end: session=... retrieved_count=...``" +msgstr "" + +#: ../../source/mp/observability.rst:690 +msgid "Tracing" +msgstr "" + +#: ../../source/mp/observability.rst:694 +msgid "" +"``--enable-tracing`` **requires** ``--otlp-endpoint`` to be set. The " +"server will refuse to start if tracing is enabled without an OTLP " +"endpoint, since there is no local fallback for trace export." +msgstr "" + +#: ../../source/mp/observability.rst:698 +msgid "" +"When tracing is enabled (``--enable-tracing --otlp-endpoint ``), the" +" tracing subscriber creates OTel spans from START/END event pairs:" +msgstr "" + +#: ../../source/mp/observability.rst:701 +msgid "``mp.store`` — from ``MP_STORE_START`` to ``MP_STORE_END``" +msgstr "" + +#: ../../source/mp/observability.rst:702 +msgid "``mp.retrieve`` — from ``MP_RETRIEVE_START`` to ``MP_RETRIEVE_END``" +msgstr "" + +#: ../../source/mp/observability.rst:703 +msgid "" +"``mp.lookup_prefetch`` — from ``MP_LOOKUP_PREFETCH_START`` to " +"``MP_LOOKUP_PREFETCH_END``" +msgstr "" + +#: ../../source/mp/observability.rst:705 +msgid "" +"Each span carries event metadata as span attributes (e.g. ``device``, " +"``stored_count``, ``found_count``)." +msgstr "" + +#: ../../source/mp/observability.rst:708 +msgid "" +"View traces in any OTel-compatible backend such as **Jaeger** or " +"**Grafana Tempo**." +msgstr "" + +#: ../../source/mp/observability.rst:724 +msgid "Per-Request Hit-Rate Attributes" +msgstr "" + +#: ../../source/mp/observability.rst:726 +msgid "" +"Each session is wrapped in a per-request root span — ``request`` for the " +"standard MP path and ``cb.request`` for the CacheBlend path — that nests " +"all child spans (``mp.store``, ``mp.retrieve``, ``mp.lookup_prefetch``) " +"beneath it. When the lookup phase ends, the root span is annotated with " +"three OTel attributes that summarise the request-level cache hit rate:" +msgstr "" + +#: ../../source/mp/observability.rst:737 +msgid "OTel type" +msgstr "" + +#: ../../source/mp/observability.rst:739 +msgid "``hit_tokens``" +msgstr "" + +#: ../../source/mp/observability.rst:740 ../../source/mp/observability.rst:743 +msgid "``int``" +msgstr "" + +#: ../../source/mp/observability.rst:741 +msgid "Tokens served from L1+L2 (numerator)." +msgstr "" + +#: ../../source/mp/observability.rst:742 +msgid "``requested_tokens``" +msgstr "" + +#: ../../source/mp/observability.rst:744 +msgid "Chunk-aligned tokens submitted for lookup (denominator)." +msgstr "" + +#: ../../source/mp/observability.rst:745 +msgid "``hit_rate``" +msgstr "" + +#: ../../source/mp/observability.rst:746 +msgid "``float``" +msgstr "" + +#: ../../source/mp/observability.rst:747 +msgid "" +"``hit_tokens / requested_tokens``; ``0.0`` when the denominator is zero." +" Stored as a precomputed float because trace UIs (Tempo, Jaeger) cannot " +"derive it from two integer attributes at query time." +msgstr "" + +#: ../../source/mp/observability.rst:751 +msgid "" +"The attributes are written when ``MP_LOOKUP_PREFETCH_END`` (standard MP " +"path) or ``CB_LOOKUP_END`` (CacheBlend path) is processed — while the " +"root span is still open. **Store-only requests** that never call " +"``lookup_prefetch_start()`` emit no end event for the lookup phase, so " +"their root span will not carry these attributes." +msgstr "" + +#: ../../source/mp/observability.rst:757 +msgid "Example TraceQL queries (Grafana Tempo):" +msgstr "" + +#: ../../source/mp/observability.rst:770 +msgid "" +"For the full event-to-span mapping and the registry pattern that links " +"child spans back to the root see ``docs/design/observability/request-" +"event-span.md`` in the source tree." +msgstr "" + +#: ../../source/mp/observability.rst:777 +msgid "Trace Recording" +msgstr "" + +#: ../../source/mp/observability.rst:781 +msgid "" +"Trace recording is **distinct from** ``--enable-tracing`` (OTel spans). " +"Trace recording captures every ``StorageManager`` public-API call to a " +"binary file so the same workload can be **replayed** later for testing, " +"regression hunting, and benchmarking — without needing vLLM and " +"(eventually) without a GPU. ``--enable-tracing`` exports live OTel spans " +"to an OTLP endpoint for online observability. The two features are " +"independent and can be used together." +msgstr "" + +#: ../../source/mp/observability.rst:789 +#, python-brace-format +msgid "" +"When ``--trace-level storage`` is set, LMCache records every call to " +"``StorageManager.{reserve_write, finish_write, submit_prefetch_task, " +"read_prefetched_results, finish_read_prefetched}`` to a binary file for " +"later replay." +msgstr "" + +#: ../../source/mp/observability.rst:794 +msgid "" +"Recording is **off by default** and adds near-zero overhead when off (a " +"single boolean check per ``StorageManager`` call). When on, recording " +"happens on the EventBus drain thread, off the request path." +msgstr "" + +#: ../../source/mp/observability.rst:799 +msgid "Capturing a trace" +msgstr "" + +#: ../../source/mp/observability.rst:801 +msgid "With an explicit output path:" +msgstr "" + +#: ../../source/mp/observability.rst:809 +msgid "With an implicit timestamped output path under ``$TMPDIR``:" +msgstr "" + +#: ../../source/mp/observability.rst:820 +msgid "" +"The trace file is closed cleanly on shutdown (SIGTERM is handled by the " +"EventBus stop path)." +msgstr "" + +#: ../../source/mp/observability.rst:824 +msgid "Replay" +msgstr "" + +#: ../../source/mp/observability.rst:826 +msgid "" +"Replaying a recorded trace, plus the full set of CLI flags for driving, " +"monitoring, and exporting replay results, is covered in its own page: " +":doc:`tracing_and_debugging`." +msgstr "" + +#: ../../source/mp/observability.rst:831 +msgid "What is captured (and what is not)" +msgstr "" + +#: ../../source/mp/observability.rst:833 +msgid "**Captured:**" +msgstr "" + +#: ../../source/mp/observability.rst:835 +msgid "The fully-qualified name of every decorated ``StorageManager`` call." +msgstr "" + +#: ../../source/mp/observability.rst:836 +msgid "" +"Each call's input arguments (e.g. ``keys``, ``layout_desc``, ``mode``, " +"``extra_count``, ``external_request_id``)." +msgstr "" + +#: ../../source/mp/observability.rst:838 +msgid "Wall-clock and monotonic timestamps of each call." +msgstr "" + +#: ../../source/mp/observability.rst:839 +msgid "" +"A header carrying a trace schema version, start times, and a SHA-256 " +"digest of the active ``StorageManagerConfig`` so replay can detect " +"mismatched configurations." +msgstr "" + +#: ../../source/mp/observability.rst:843 +msgid "**Not captured:**" +msgstr "" + +#: ../../source/mp/observability.rst:845 +msgid "" +"KV tensor bytes. Replay exercises bookkeeping and controller logic; " +"payloads at replay time are zeros." +msgstr "" + +#: ../../source/mp/observability.rst:847 +msgid "" +"Calls inside the ``MPCacheEngine``, the message queue, or any GPU-copy " +"code. These layers are **out of scope** for the storage trace level." +msgstr "" + +#: ../../source/mp/observability.rst:852 +msgid "File format" +msgstr "" + +#: ../../source/mp/observability.rst:854 +msgid "A length-prefixed `msgpack `_ stream:" +msgstr "" + +#: ../../source/mp/observability.rst:863 +msgid "" +"The ``Header`` carries a magic prefix (``LMCT``), a format version, the " +"trace level (``storage`` today), a trace schema version, start " +"timestamps, and the StorageManagerConfig digest. Each ``Record`` carries " +"a relative timestamp, a wall-clock timestamp, the fully-qualified call " +"site (``qualname``), and an argument dict." +msgstr "" + +#: ../../source/mp/observability.rst:869 +msgid "" +"The format is deliberately extensible: future trace **levels** (``mq``, " +"``gpu``) will share this layout and use the ``level`` header field to " +"discriminate. Additional captured ops add new ``qualname`` strings " +"without bumping the format version." +msgstr "" + +#: ../../source/mp/observability.rst:874 +msgid "" +"For the full design rationale see " +"``docs/design/v1/mp_observability/trace.md`` in the source tree." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/operator.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/operator.po new file mode 100644 index 00000000000..c3206ca058e --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/operator.po @@ -0,0 +1,978 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/operator.rst:2 +msgid "Kubernetes Operator" +msgstr "" + +#: ../../source/mp/operator.rst:4 +msgid "" +"The LMCache Kubernetes operator automates the deployment and lifecycle " +"management of LMCache multiprocess servers. Instead of hand-writing " +"DaemonSets, Services, and ConfigMaps (as described in the manual " +":doc:`deployment` guide), you declare a single ``LMCacheEngine`` custom " +"resource and the operator reconciles all underlying Kubernetes objects." +msgstr "" + +#: ../../source/mp/operator.rst:15 +msgid "Why Use the Operator" +msgstr "" + +#: ../../source/mp/operator.rst:17 +msgid "" +"The manual DaemonSet approach works, but it has sharp edges the operator " +"eliminates:" +msgstr "" + +#: ../../source/mp/operator.rst:20 +msgid "" +"**Auto-injected pod settings** -- The operator always sets ``hostIPC: " +"true`` and ``--host 0.0.0.0``. Forgetting ``hostIPC`` in a hand-written " +"manifest causes silent CUDA IPC failures " +"(``cudaErrorMapBufferObjectFailed``) that are hard to debug." +msgstr "" + +#: ../../source/mp/operator.rst:24 +msgid "" +"**Node-local service discovery** -- The operator creates a ClusterIP " +"Service with ``internalTrafficPolicy=Local`` and a connection ConfigMap " +"that vLLM pods simply mount. No ``hostNetwork``, no Downward API, no " +"shell variable substitution." +msgstr "" + +#: ../../source/mp/operator.rst:28 +msgid "" +"**Auto-computed resource sizing** -- Memory requests and limits are " +"derived from ``l1.sizeGB``, avoiding OOM kills (under-provisioned) or " +"wasted node capacity (over-provisioned)." +msgstr "" + +#: ../../source/mp/operator.rst:31 +msgid "" +"**Declarative Prometheus integration** -- Set " +"``prometheus.serviceMonitor.enabled: true`` and the operator creates a " +"``ServiceMonitor`` CR that the Prometheus Operator discovers " +"automatically." +msgstr "" + +#: ../../source/mp/operator.rst:34 +msgid "" +"**CRD validation** -- OpenAPI schema validation catches misconfigurations" +" (e.g., ``l1.sizeGB <= 0``, invalid port range) at ``kubectl apply`` " +"time, before any pods are created." +msgstr "" + +#: ../../source/mp/operator.rst:39 +msgid "Prerequisites" +msgstr "" + +#: ../../source/mp/operator.rst:41 +msgid "Kubernetes 1.20+" +msgstr "" + +#: ../../source/mp/operator.rst:42 +msgid "``kubectl`` configured to access your cluster" +msgstr "" + +#: ../../source/mp/operator.rst:43 +msgid "" +"(Optional) `Prometheus Operator `_ for ServiceMonitor support" +msgstr "" + +#: ../../source/mp/operator.rst:47 +msgid "Installing the Operator" +msgstr "" + +#: ../../source/mp/operator.rst:49 +msgid "**Option A: One-line install from release (recommended)**" +msgstr "" + +#: ../../source/mp/operator.rst:59 +msgid "**Option B: Build from source**" +msgstr "" + +#: ../../source/mp/operator.rst:69 +msgid "Deploying an LMCacheEngine" +msgstr "" + +#: ../../source/mp/operator.rst:71 +msgid "A minimal CR deploys a DaemonSet with 60 GB L1 cache on every GPU node:" +msgstr "" + +#: ../../source/mp/operator.rst:87 +msgid "The operator automatically:" +msgstr "" + +#: ../../source/mp/operator.rst:89 +msgid "Creates a DaemonSet running one LMCache server pod per matched node" +msgstr "" + +#: ../../source/mp/operator.rst:90 +msgid "Sets ``hostIPC: true`` and passes ``--host 0.0.0.0`` to the server" +msgstr "" + +#: ../../source/mp/operator.rst:91 +msgid "Creates a node-local ClusterIP Service for vLLM discovery" +msgstr "" + +#: ../../source/mp/operator.rst:92 +msgid "" +"Creates a connection ConfigMap (``my-cache-connection``) with the ``kv-" +"transfer-config`` JSON that vLLM needs" +msgstr "" + +#: ../../source/mp/operator.rst:94 +msgid "Auto-computes resource requests/limits from the L1 cache size" +msgstr "" + +#: ../../source/mp/operator.rst:95 +msgid "Defaults ``nodeSelector`` to ``nvidia.com/gpu.present: \"true\"``" +msgstr "" + +#: ../../source/mp/operator.rst:98 +msgid "" +"The operator defaults the container image to ``lmcache/vllm-" +"openai:latest``. Override with ``spec.image.repository`` and " +"``spec.image.tag`` to pin a specific version." +msgstr "" + +#: ../../source/mp/operator.rst:103 +msgid "Connecting vLLM" +msgstr "" + +#: ../../source/mp/operator.rst:105 +msgid "" +"The operator creates a ConfigMap named ``-connection`` " +"containing the ``kv-transfer-config`` JSON. Mount it in your vLLM " +"Deployment:" +msgstr "" + +#: ../../source/mp/operator.rst:156 +msgid "Key requirements for vLLM pods:" +msgstr "" + +#: ../../source/mp/operator.rst:158 +msgid "" +"**hostIPC: true** -- CUDA IPC (``cudaIpcOpenMemHandle``) needs a shared " +"IPC namespace between vLLM and LMCache." +msgstr "" + +#: ../../source/mp/operator.rst:160 +msgid "" +"**PYTHONHASHSEED=0** -- Ensures deterministic token hashing so vLLM and " +"LMCache produce consistent cache keys." +msgstr "" + +#: ../../source/mp/operator.rst:162 +msgid "" +"**ConfigMap mount** -- The ``$(cat ...)`` pattern reads the connection " +"JSON inline. The ConfigMap name is always ``-connection``." +msgstr "" + +#: ../../source/mp/operator.rst:164 +msgid "" +"**No hostNetwork needed** -- The operator's node-local Service handles " +"routing via ``internalTrafficPolicy=Local``." +msgstr "" + +#: ../../source/mp/operator.rst:168 +msgid "Verifying the Deployment" +msgstr "" + +#: ../../source/mp/operator.rst:175 +msgid "Expected output:" +msgstr "" + +#: ../../source/mp/operator.rst:194 +msgid "CRD Spec Reference" +msgstr "" + +#: ../../source/mp/operator.rst:197 +msgid "Image" +msgstr "" + +#: ../../source/mp/operator.rst:203 ../../source/mp/operator.rst:226 +#: ../../source/mp/operator.rst:249 ../../source/mp/operator.rst:263 +#: ../../source/mp/operator.rst:283 ../../source/mp/operator.rst:309 +#: ../../source/mp/operator.rst:324 ../../source/mp/operator.rst:347 +#: ../../source/mp/operator.rst:478 +msgid "Field" +msgstr "" + +#: ../../source/mp/operator.rst:204 ../../source/mp/operator.rst:227 +#: ../../source/mp/operator.rst:250 ../../source/mp/operator.rst:264 +#: ../../source/mp/operator.rst:284 ../../source/mp/operator.rst:310 +#: ../../source/mp/operator.rst:325 ../../source/mp/operator.rst:348 +msgid "Default" +msgstr "" + +#: ../../source/mp/operator.rst:205 ../../source/mp/operator.rst:228 +#: ../../source/mp/operator.rst:251 ../../source/mp/operator.rst:265 +#: ../../source/mp/operator.rst:285 ../../source/mp/operator.rst:311 +#: ../../source/mp/operator.rst:326 ../../source/mp/operator.rst:349 +msgid "Description" +msgstr "" + +#: ../../source/mp/operator.rst:206 +msgid "``image.repository``" +msgstr "" + +#: ../../source/mp/operator.rst:207 +msgid "``lmcache/vllm-openai``" +msgstr "" + +#: ../../source/mp/operator.rst:208 +msgid "Container image repository." +msgstr "" + +#: ../../source/mp/operator.rst:209 +msgid "``image.tag``" +msgstr "" + +#: ../../source/mp/operator.rst:210 +msgid "``latest``" +msgstr "" + +#: ../../source/mp/operator.rst:211 +msgid "Container image tag." +msgstr "" + +#: ../../source/mp/operator.rst:212 +msgid "``image.pullPolicy``" +msgstr "" + +#: ../../source/mp/operator.rst:213 +msgid "``IfNotPresent``" +msgstr "" + +#: ../../source/mp/operator.rst:214 +msgid "``Always``, ``Never``, or ``IfNotPresent``." +msgstr "" + +#: ../../source/mp/operator.rst:215 +msgid "``imagePullSecrets``" +msgstr "" + +#: ../../source/mp/operator.rst:216 ../../source/mp/operator.rst:299 +#: ../../source/mp/operator.rst:313 ../../source/mp/operator.rst:331 +#: ../../source/mp/operator.rst:334 ../../source/mp/operator.rst:337 +#: ../../source/mp/operator.rst:354 ../../source/mp/operator.rst:357 +#: ../../source/mp/operator.rst:360 ../../source/mp/operator.rst:363 +#: ../../source/mp/operator.rst:366 ../../source/mp/operator.rst:369 +#: ../../source/mp/operator.rst:372 ../../source/mp/operator.rst:375 +msgid "--" +msgstr "" + +#: ../../source/mp/operator.rst:217 +msgid "Image pull secret references." +msgstr "" + +#: ../../source/mp/operator.rst:220 +msgid "Server" +msgstr "" + +#: ../../source/mp/operator.rst:229 ../../source/mp/operator.rst:488 +msgid "``server.port``" +msgstr "" + +#: ../../source/mp/operator.rst:230 +msgid "``5555``" +msgstr "" + +#: ../../source/mp/operator.rst:231 +msgid "ZMQ listening port (1024--65535)." +msgstr "" + +#: ../../source/mp/operator.rst:232 +msgid "``server.chunkSize``" +msgstr "" + +#: ../../source/mp/operator.rst:233 +msgid "``256``" +msgstr "" + +#: ../../source/mp/operator.rst:234 +msgid "Token chunk size." +msgstr "" + +#: ../../source/mp/operator.rst:235 +msgid "``server.maxWorkers``" +msgstr "" + +#: ../../source/mp/operator.rst:236 +msgid "``1``" +msgstr "" + +#: ../../source/mp/operator.rst:237 +msgid "Worker threads for ZMQ requests." +msgstr "" + +#: ../../source/mp/operator.rst:238 +msgid "``server.hashAlgorithm``" +msgstr "" + +#: ../../source/mp/operator.rst:239 +msgid "``blake3``" +msgstr "" + +#: ../../source/mp/operator.rst:240 +msgid "``builtin``, ``sha256_cbor``, or ``blake3``." +msgstr "" + +#: ../../source/mp/operator.rst:243 +msgid "L1 Cache" +msgstr "" + +#: ../../source/mp/operator.rst:252 ../../source/mp/operator.rst:480 +msgid "``l1.sizeGB``" +msgstr "" + +#: ../../source/mp/operator.rst:253 +msgid "*required*" +msgstr "" + +#: ../../source/mp/operator.rst:254 +msgid "L1 cache size in GB. Must be > 0." +msgstr "" + +#: ../../source/mp/operator.rst:257 +msgid "Eviction" +msgstr "" + +#: ../../source/mp/operator.rst:266 ../../source/mp/operator.rst:482 +msgid "``eviction.policy``" +msgstr "" + +#: ../../source/mp/operator.rst:267 +msgid "``LRU``" +msgstr "" + +#: ../../source/mp/operator.rst:268 +msgid "Only ``LRU`` is supported." +msgstr "" + +#: ../../source/mp/operator.rst:269 ../../source/mp/operator.rst:484 +msgid "``eviction.triggerWatermark``" +msgstr "" + +#: ../../source/mp/operator.rst:270 +msgid "``0.8``" +msgstr "" + +#: ../../source/mp/operator.rst:271 +msgid "Usage ratio (0.0--1.0] to trigger eviction." +msgstr "" + +#: ../../source/mp/operator.rst:272 ../../source/mp/operator.rst:486 +msgid "``eviction.evictionRatio``" +msgstr "" + +#: ../../source/mp/operator.rst:273 +msgid "``0.2``" +msgstr "" + +#: ../../source/mp/operator.rst:274 +msgid "Fraction to evict (0.0--1.0]." +msgstr "" + +#: ../../source/mp/operator.rst:277 ../../source/mp/operator.rst:619 +msgid "Prometheus" +msgstr "" + +#: ../../source/mp/operator.rst:286 +msgid "``prometheus.enabled``" +msgstr "" + +#: ../../source/mp/operator.rst:287 +msgid "``true``" +msgstr "" + +#: ../../source/mp/operator.rst:288 +msgid "Expose Prometheus metrics." +msgstr "" + +#: ../../source/mp/operator.rst:289 +msgid "``prometheus.port``" +msgstr "" + +#: ../../source/mp/operator.rst:290 +msgid "``9090``" +msgstr "" + +#: ../../source/mp/operator.rst:291 +msgid "``/metrics`` endpoint port." +msgstr "" + +#: ../../source/mp/operator.rst:292 +msgid "``prometheus.serviceMonitor.enabled``" +msgstr "" + +#: ../../source/mp/operator.rst:293 +msgid "``false``" +msgstr "" + +#: ../../source/mp/operator.rst:294 +msgid "Create a ServiceMonitor CR." +msgstr "" + +#: ../../source/mp/operator.rst:295 +msgid "``prometheus.serviceMonitor.interval``" +msgstr "" + +#: ../../source/mp/operator.rst:296 +msgid "``30s``" +msgstr "" + +#: ../../source/mp/operator.rst:297 +msgid "Scrape interval." +msgstr "" + +#: ../../source/mp/operator.rst:298 +msgid "``prometheus.serviceMonitor.labels``" +msgstr "" + +#: ../../source/mp/operator.rst:300 +msgid "Extra labels on the ServiceMonitor." +msgstr "" + +#: ../../source/mp/operator.rst:303 +msgid "L2 Storage" +msgstr "" + +#: ../../source/mp/operator.rst:312 +msgid "``l2Backends``" +msgstr "" + +#: ../../source/mp/operator.rst:314 +msgid "List of L2 backends (``type`` + ``config``). See :doc:`l2_storage`." +msgstr "" + +#: ../../source/mp/operator.rst:318 +msgid "Scheduling" +msgstr "" + +#: ../../source/mp/operator.rst:327 +msgid "``nodeSelector``" +msgstr "" + +#: ../../source/mp/operator.rst:328 +msgid "GPU nodes" +msgstr "" + +#: ../../source/mp/operator.rst:329 +msgid "Defaults to ``nvidia.com/gpu.present: \"true\"``." +msgstr "" + +#: ../../source/mp/operator.rst:330 +msgid "``affinity``" +msgstr "" + +#: ../../source/mp/operator.rst:332 +msgid "Pod affinity rules." +msgstr "" + +#: ../../source/mp/operator.rst:333 +msgid "``tolerations``" +msgstr "" + +#: ../../source/mp/operator.rst:335 +msgid "Pod tolerations." +msgstr "" + +#: ../../source/mp/operator.rst:336 +msgid "``priorityClassName``" +msgstr "" + +#: ../../source/mp/operator.rst:338 +msgid "Priority class for pods." +msgstr "" + +#: ../../source/mp/operator.rst:341 +msgid "Overrides & Extras" +msgstr "" + +#: ../../source/mp/operator.rst:350 +msgid "``logLevel``" +msgstr "" + +#: ../../source/mp/operator.rst:351 +msgid "``INFO``" +msgstr "" + +#: ../../source/mp/operator.rst:352 +msgid "``DEBUG``, ``INFO``, ``WARNING``, ``ERROR``." +msgstr "" + +#: ../../source/mp/operator.rst:353 +msgid "``resourceOverrides``" +msgstr "" + +#: ../../source/mp/operator.rst:355 +msgid "Override auto-computed resources." +msgstr "" + +#: ../../source/mp/operator.rst:356 +msgid "``env``" +msgstr "" + +#: ../../source/mp/operator.rst:358 +msgid "Extra environment variables." +msgstr "" + +#: ../../source/mp/operator.rst:359 +msgid "``volumes``" +msgstr "" + +#: ../../source/mp/operator.rst:361 +msgid "Extra volumes." +msgstr "" + +#: ../../source/mp/operator.rst:362 +msgid "``volumeMounts``" +msgstr "" + +#: ../../source/mp/operator.rst:364 +msgid "Extra volume mounts." +msgstr "" + +#: ../../source/mp/operator.rst:365 +msgid "``podAnnotations``" +msgstr "" + +#: ../../source/mp/operator.rst:367 +msgid "Extra pod annotations." +msgstr "" + +#: ../../source/mp/operator.rst:368 +msgid "``podLabels``" +msgstr "" + +#: ../../source/mp/operator.rst:370 +msgid "Extra pod labels." +msgstr "" + +#: ../../source/mp/operator.rst:371 +msgid "``serviceAccountName``" +msgstr "" + +#: ../../source/mp/operator.rst:373 +msgid "ServiceAccount for pods." +msgstr "" + +#: ../../source/mp/operator.rst:374 +msgid "``extraArgs``" +msgstr "" + +#: ../../source/mp/operator.rst:376 +msgid "Extra CLI flags (appended last, can override)." +msgstr "" + +#: ../../source/mp/operator.rst:379 +msgid "Auto-Computed Resources" +msgstr "" + +#: ../../source/mp/operator.rst:381 +msgid "" +"When ``spec.resourceOverrides`` is not set, the operator derives " +"resources from ``l1.sizeGB``:" +msgstr "" + +#: ../../source/mp/operator.rst:384 +msgid "**CPU request**: ``4`` cores" +msgstr "" + +#: ../../source/mp/operator.rst:385 +msgid "**Memory request**: ``ceil(l1.sizeGB + 5)`` Gi" +msgstr "" + +#: ../../source/mp/operator.rst:386 +msgid "**Memory limit**: ``ceil(memoryRequest * 1.5)`` Gi" +msgstr "" + +#: ../../source/mp/operator.rst:388 +msgid "For example, ``l1.sizeGB: 60`` produces a 65 Gi request and 98 Gi limit." +msgstr "" + +#: ../../source/mp/operator.rst:391 +msgid "Auto-Injected Pod Settings" +msgstr "" + +#: ../../source/mp/operator.rst:393 +msgid "" +"The operator always injects these into the pod spec (they are not " +"configurable via the CRD):" +msgstr "" + +#: ../../source/mp/operator.rst:396 +msgid "**hostIPC: true** -- Required for CUDA IPC between LMCache and vLLM." +msgstr "" + +#: ../../source/mp/operator.rst:397 +msgid "" +"**--host 0.0.0.0** -- Binds the server to all interfaces so the node-" +"local Service can route to it." +msgstr "" + +#: ../../source/mp/operator.rst:399 +msgid "" +"**NVIDIA_VISIBLE_DEVICES=all** -- Ensures GPU access for IPC-based memory" +" transfers." +msgstr "" + +#: ../../source/mp/operator.rst:401 +msgid "" +"**TCP socket probes** -- Startup (5s initial, 30 failures), liveness " +"(10s), and readiness (5s) probes on the server port." +msgstr "" + +#: ../../source/mp/operator.rst:405 +msgid "" +"The operator does **not** mount an emptyDir at ``/dev/shm``. With " +"``hostIPC: true``, the container sees the host's ``/dev/shm`` directly. " +"Mounting an emptyDir would shadow it with a private tmpfs and break CUDA " +"IPC." +msgstr "" + +#: ../../source/mp/operator.rst:410 +msgid "Resources Created" +msgstr "" + +#: ../../source/mp/operator.rst:412 +msgid "For an ``LMCacheEngine`` named ``my-cache``:" +msgstr "" + +#: ../../source/mp/operator.rst:418 +msgid "Resource" +msgstr "" + +#: ../../source/mp/operator.rst:419 +msgid "Name" +msgstr "" + +#: ../../source/mp/operator.rst:420 +msgid "Purpose" +msgstr "" + +#: ../../source/mp/operator.rst:421 +msgid "DaemonSet" +msgstr "" + +#: ../../source/mp/operator.rst:422 ../../source/mp/operator.rst:425 +#: ../../source/mp/operator.rst:434 +msgid "``my-cache``" +msgstr "" + +#: ../../source/mp/operator.rst:423 +msgid "Runs LMCache server pods." +msgstr "" + +#: ../../source/mp/operator.rst:424 +msgid "Service (ClusterIP)" +msgstr "" + +#: ../../source/mp/operator.rst:426 +msgid "Node-local discovery (``internalTrafficPolicy=Local``)." +msgstr "" + +#: ../../source/mp/operator.rst:427 +msgid "Service (headless)" +msgstr "" + +#: ../../source/mp/operator.rst:428 +msgid "``my-cache-metrics``" +msgstr "" + +#: ../../source/mp/operator.rst:429 +msgid "Prometheus scrape target." +msgstr "" + +#: ../../source/mp/operator.rst:430 +msgid "ConfigMap" +msgstr "" + +#: ../../source/mp/operator.rst:431 +msgid "``my-cache-connection``" +msgstr "" + +#: ../../source/mp/operator.rst:432 +msgid "``kv-transfer-config`` JSON for vLLM." +msgstr "" + +#: ../../source/mp/operator.rst:433 +msgid "ServiceMonitor" +msgstr "" + +#: ../../source/mp/operator.rst:435 +msgid "Prometheus Operator integration (when enabled)." +msgstr "" + +#: ../../source/mp/operator.rst:437 +msgid "The connection ConfigMap contains:" +msgstr "" + +#: ../../source/mp/operator.rst:451 +msgid "Status & Conditions" +msgstr "" + +#: ../../source/mp/operator.rst:457 +msgid "The status section includes:" +msgstr "" + +#: ../../source/mp/operator.rst:459 +msgid "**phase**: ``Pending``, ``Running``, ``Degraded``, or ``Failed``." +msgstr "" + +#: ../../source/mp/operator.rst:460 +msgid "**readyInstances** / **desiredInstances**: Instance counts." +msgstr "" + +#: ../../source/mp/operator.rst:461 +msgid "" +"**endpoints**: Per-node connection info (node name, host IP, pod name, " +"port, readiness)." +msgstr "" + +#: ../../source/mp/operator.rst:463 +msgid "**conditions**:" +msgstr "" + +#: ../../source/mp/operator.rst:465 +msgid "``Available`` -- At least one instance is ready." +msgstr "" + +#: ../../source/mp/operator.rst:466 +msgid "``AllInstancesReady`` -- All desired instances are ready." +msgstr "" + +#: ../../source/mp/operator.rst:467 +msgid "``ConfigValid`` -- Spec validation passed." +msgstr "" + +#: ../../source/mp/operator.rst:470 +msgid "Validation Rules" +msgstr "" + +#: ../../source/mp/operator.rst:472 +msgid "The operator validates the CR spec at apply time:" +msgstr "" + +#: ../../source/mp/operator.rst:479 +msgid "Rule" +msgstr "" + +#: ../../source/mp/operator.rst:481 +msgid "Required, must be > 0." +msgstr "" + +#: ../../source/mp/operator.rst:483 +msgid "Must be ``LRU`` (if set)." +msgstr "" + +#: ../../source/mp/operator.rst:485 ../../source/mp/operator.rst:487 +msgid "Must be in (0.0, 1.0]." +msgstr "" + +#: ../../source/mp/operator.rst:489 +msgid "Must be in [1024, 65535]." +msgstr "" + +#: ../../source/mp/operator.rst:492 +msgid "Examples" +msgstr "" + +#: ../../source/mp/operator.rst:495 +msgid "Target Only GPU Nodes" +msgstr "" + +#: ../../source/mp/operator.rst:497 +msgid "" +"Use ``nodeSelector`` to run LMCache only on GPU nodes. New GPU nodes " +"automatically get an LMCache pod:" +msgstr "" + +#: ../../source/mp/operator.rst:513 +msgid "" +"The operator defaults ``nodeSelector`` to ``nvidia.com/gpu.present: " +"\"true\"`` when not specified, so a minimal CR already targets GPU nodes." +msgstr "" + +#: ../../source/mp/operator.rst:517 +msgid "Custom Server Port" +msgstr "" + +#: ../../source/mp/operator.rst:519 +msgid "If the default port (5555) conflicts with other services:" +msgstr "" + +#: ../../source/mp/operator.rst:533 +msgid "" +"The connection ConfigMap updates automatically -- vLLM pods pick up the " +"new port on restart." +msgstr "" + +#: ../../source/mp/operator.rst:537 +msgid "Production with Prometheus Monitoring" +msgstr "" + +#: ../../source/mp/operator.rst:573 +msgid "See :doc:`observability` for metric names and Grafana configuration." +msgstr "" + +#: ../../source/mp/operator.rst:576 +msgid "Override Auto-Computed Resources" +msgstr "" + +#: ../../source/mp/operator.rst:595 +msgid "Operator vs Manual Deployment" +msgstr "" + +#: ../../source/mp/operator.rst:601 +msgid "Concern" +msgstr "" + +#: ../../source/mp/operator.rst:602 +msgid "Manual DaemonSet" +msgstr "" + +#: ../../source/mp/operator.rst:603 +msgid "LMCacheEngine Operator" +msgstr "" + +#: ../../source/mp/operator.rst:604 +msgid "hostIPC" +msgstr "" + +#: ../../source/mp/operator.rst:605 ../../source/mp/operator.rst:608 +msgid "Must set manually" +msgstr "" + +#: ../../source/mp/operator.rst:606 ../../source/mp/operator.rst:609 +msgid "Auto-injected" +msgstr "" + +#: ../../source/mp/operator.rst:607 +msgid "``--host 0.0.0.0``" +msgstr "" + +#: ../../source/mp/operator.rst:610 +msgid "Service discovery" +msgstr "" + +#: ../../source/mp/operator.rst:611 +msgid "``hostNetwork`` + ``status.hostIP``" +msgstr "" + +#: ../../source/mp/operator.rst:612 +msgid "Node-local ClusterIP Service + ConfigMap" +msgstr "" + +#: ../../source/mp/operator.rst:613 +msgid "vLLM config" +msgstr "" + +#: ../../source/mp/operator.rst:614 +msgid "Copy JSON into Deployment" +msgstr "" + +#: ../../source/mp/operator.rst:615 +msgid "Mount ``-connection`` ConfigMap" +msgstr "" + +#: ../../source/mp/operator.rst:616 +msgid "Resource sizing" +msgstr "" + +#: ../../source/mp/operator.rst:617 +msgid "Manual calculation" +msgstr "" + +#: ../../source/mp/operator.rst:618 +msgid "Auto-computed from ``l1.sizeGB``" +msgstr "" + +#: ../../source/mp/operator.rst:620 +msgid "Manual ServiceMonitor" +msgstr "" + +#: ../../source/mp/operator.rst:621 +msgid "``serviceMonitor.enabled: true``" +msgstr "" + +#: ../../source/mp/operator.rst:622 +msgid "Validation" +msgstr "" + +#: ../../source/mp/operator.rst:623 +msgid "Runtime errors only" +msgstr "" + +#: ../../source/mp/operator.rst:624 +msgid "``kubectl apply`` rejects invalid specs" +msgstr "" + +#: ../../source/mp/operator.rst:625 +msgid "New GPU nodes" +msgstr "" + +#: ../../source/mp/operator.rst:626 +msgid "DaemonSet handles it" +msgstr "" + +#: ../../source/mp/operator.rst:627 +msgid "DaemonSet handles it (same)" +msgstr "" + +#: ../../source/mp/operator.rst:630 +msgid "Security Considerations" +msgstr "" + +#: ../../source/mp/operator.rst:632 +msgid "" +"**hostIPC** exposes the host's IPC namespace (System V IPC, POSIX message" +" queues) to the container. Any process in the container can interact " +"with IPC resources from other processes on the same host." +msgstr "" + +#: ../../source/mp/operator.rst:636 +msgid "Deploy only in trusted environments." +msgstr "" + +#: ../../source/mp/operator.rst:637 +msgid "" +"Clusters using Pod Security Standards must allow the ``privileged`` " +"profile for the LMCache namespace -- the ``baseline`` and ``restricted`` " +"profiles reject ``hostIPC``." +msgstr "" + +#: ../../source/mp/operator.rst:642 +msgid "Development" +msgstr "" + +#: ../../source/mp/operator.rst:654 +msgid "Pushing a custom operator image:" +msgstr "" + +#: ../../source/mp/operator.rst:665 +msgid "If your cluster needs pull credentials:" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/quickstart.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/quickstart.po new file mode 100644 index 00000000000..c78eea59937 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/quickstart.po @@ -0,0 +1,201 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/quickstart.rst:2 +msgid "Quick Start" +msgstr "" + +#: ../../source/mp/quickstart.rst:4 +msgid "" +"This page walks through the fastest ways to get LMCache multiprocess mode" +" running -- locally, in Docker, and with the HTTP server variant." +msgstr "" + +#: ../../source/mp/quickstart.rst:8 +msgid "Local Quick Start" +msgstr "" + +#: ../../source/mp/quickstart.rst:10 +msgid "**Step 1: Start the LMCache server**" +msgstr "" + +#: ../../source/mp/quickstart.rst:17 +msgid "Expected log output:" +msgstr "" + +#: ../../source/mp/quickstart.rst:24 +msgid "" +"The default ZMQ port is **5555** (use ``--port`` to change it). The HTTP " +"frontend listens on **8080** by default (use ``--http-port`` to change " +"it)." +msgstr "" + +#: ../../source/mp/quickstart.rst:28 +msgid "**Step 2: Start vLLM with the LMCache connector**" +msgstr "" + +#: ../../source/mp/quickstart.rst:30 +msgid "In a new terminal:" +msgstr "" + +#: ../../source/mp/quickstart.rst:39 +msgid "" +"This connects to the default LMCache port (5555) on localhost. If you " +"changed the server port with ``--port``, pass it on the vLLM side via " +"``kv_connector_extra_config``:" +msgstr "" + +#: ../../source/mp/quickstart.rst:49 +msgid "To connect to a remote host, also set ``lmcache.mp.host``:" +msgstr "" + +#: ../../source/mp/quickstart.rst:56 +msgid "You should see on the **vLLM** side:" +msgstr "" + +#: ../../source/mp/quickstart.rst:62 +msgid "And on the **LMCache** side:" +msgstr "" + +#: ../../source/mp/quickstart.rst:68 +msgid "**Step 3: Send a request**" +msgstr "" + +#: ../../source/mp/quickstart.rst:80 +msgid "First request -- tokens are **stored**:" +msgstr "" + +#: ../../source/mp/quickstart.rst:86 +msgid "Second identical request -- tokens are **retrieved** from cache:" +msgstr "" + +#: ../../source/mp/quickstart.rst:93 +msgid "Docker Quick Start" +msgstr "" + +#: ../../source/mp/quickstart.rst:95 +msgid "**Step 1: Start the LMCache container**" +msgstr "" + +#: ../../source/mp/quickstart.rst:107 +msgid "" +"``--network host`` lets the vLLM container reach the LMCache server on " +"localhost. ``--ipc host`` is required for CUDA IPC shared memory." +msgstr "" + +#: ../../source/mp/quickstart.rst:110 +msgid "**Step 2: Start the vLLM container**" +msgstr "" + +#: ../../source/mp/quickstart.rst:123 +msgid "" +"Use the nightly images (``lmcache/standalone:nightly`` and ``lmcache" +"/vllm-openai:latest-nightly``) as the MP-mode interfaces are actively " +"evolving." +msgstr "" + +#: ../../source/mp/quickstart.rst:127 +msgid "**Step 3: Send requests** the same way as in the local quick start." +msgstr "" + +#: ../../source/mp/quickstart.rst:130 +msgid "HTTP Server Quick Start" +msgstr "" + +#: ../../source/mp/quickstart.rst:132 +msgid "" +"The HTTP server wraps the ZMQ server with a FastAPI frontend, adding HTTP" +" management endpoints for health checking and cache administration." +msgstr "" + +#: ../../source/mp/quickstart.rst:140 +msgid "" +"The HTTP server listens on ``0.0.0.0:8080`` by default (configurable with" +" ``--http-host`` and ``--http-port``)." +msgstr "" + +#: ../../source/mp/quickstart.rst:143 +msgid "**Endpoints:**" +msgstr "" + +#: ../../source/mp/quickstart.rst:149 +msgid "Method" +msgstr "" + +#: ../../source/mp/quickstart.rst:150 +msgid "Path" +msgstr "" + +#: ../../source/mp/quickstart.rst:151 +msgid "Description" +msgstr "" + +#: ../../source/mp/quickstart.rst:152 ../../source/mp/quickstart.rst:161 +msgid "GET" +msgstr "" + +#: ../../source/mp/quickstart.rst:153 +msgid "``/healthcheck``" +msgstr "" + +#: ../../source/mp/quickstart.rst:154 +#, python-brace-format +msgid "" +"Returns ``{\"status\": \"healthy\"}`` when the engine is initialized and " +"memory checks pass. Suitable for Kubernetes liveness/readiness probes." +msgstr "" + +#: ../../source/mp/quickstart.rst:156 +msgid "POST" +msgstr "" + +#: ../../source/mp/quickstart.rst:157 +msgid "``/clear-cache``" +msgstr "" + +#: ../../source/mp/quickstart.rst:158 +#, python-brace-format +msgid "" +"Force-clears all KV cache data stored in L1 (CPU) memory, including " +"objects with active read/write locks. Returns ``{\"status\": \"ok\"}`` on" +" success." +msgstr "" + +#: ../../source/mp/quickstart.rst:162 +msgid "``/status``" +msgstr "" + +#: ../../source/mp/quickstart.rst:163 +msgid "" +"Returns detailed internal state of all MP components including L1 cache, " +"L2 adapters, controllers, registered GPUs, and active sessions." +msgstr "" + +#: ../../source/mp/quickstart.rst:166 +msgid "Examples:" +msgstr "" + +#: ../../source/mp/quickstart.rst:181 +msgid "" +"The ZMQ server runs on the same default port (5555) and accepts vLLM " +"connections exactly as in the local quick start." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/serde.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/serde.po new file mode 100644 index 00000000000..2229e1e79ad --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/serde.po @@ -0,0 +1,159 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/serde.rst:2 +msgid "L2 Serde (Serialization / Deserialization)" +msgstr "" + +#: ../../source/mp/serde.rst:4 +msgid "" +"LMCache supports a **per-adapter serde** that transforms KV cache data on" +" its way to and from an L2 adapter. Typical uses: quantization (shrink " +"storage footprint), compression, encryption." +msgstr "" + +#: ../../source/mp/serde.rst:14 +msgid "When to use serde" +msgstr "" + +#: ../../source/mp/serde.rst:16 +msgid "" +"**Save L2 storage or bandwidth.** fp8 quantization halves byte volume vs." +" bf16 with minor accuracy loss — a good fit for disk / remote adapters." +msgstr "" + +#: ../../source/mp/serde.rst:19 +msgid "" +"**Encrypt at rest.** Wrap the raw bytes with authenticated encryption " +"before they land on disk." +msgstr "" + +#: ../../source/mp/serde.rst:21 +msgid "" +"**Custom compression.** Anything lossless (lz4/zstd) or lossy (CacheGen-" +"style) can be plugged in via the ``Serializer`` / ``Deserializer`` ABCs." +msgstr "" + +#: ../../source/mp/serde.rst:25 +msgid "" +"Serde is **opt-in per adapter**: one ``--l2-adapter`` may use fp8 while " +"another stores raw bytes. When omitted, the adapter behaves exactly as if" +" serde did not exist (no extra allocations, no extra threads)." +msgstr "" + +#: ../../source/mp/serde.rst:31 +msgid "Configuring serde on an L2 adapter" +msgstr "" + +#: ../../source/mp/serde.rst:33 +msgid "" +"Add a ``\"serde\"`` sub-dict to any ``--l2-adapter`` JSON spec. The " +"``type`` field selects a registered serde; remaining keys are forwarded " +"to the serde factory." +msgstr "" + +#: ../../source/mp/serde.rst:48 +msgid "Built-in serde types" +msgstr "" + +#: ../../source/mp/serde.rst:52 +msgid "``type``" +msgstr "" + +#: ../../source/mp/serde.rst:53 +msgid "Description" +msgstr "" + +#: ../../source/mp/serde.rst:54 +msgid "Config fields" +msgstr "" + +#: ../../source/mp/serde.rst:55 +msgid "``fp8``" +msgstr "" + +#: ../../source/mp/serde.rst:56 +msgid "" +"Quantize each element to 8-bit float; dequantize on load. Lossy but " +"highly compressible." +msgstr "" + +#: ../../source/mp/serde.rst:58 +msgid "" +"``fp8_dtype`` (default ``float8_e4m3fn``; also accepts ``float8_e5m2``), " +"``max_workers`` (thread pool size, default 1)" +msgstr "" + +#: ../../source/mp/serde.rst:64 +msgid "Writing a custom serde" +msgstr "" + +#: ../../source/mp/serde.rst:66 +msgid "" +"Implement the two sync ABCs (``Serializer``, ``Deserializer``) with your " +"transform logic, then register a factory keyed on a name you pick:" +msgstr "" + +#: ../../source/mp/serde.rst:98 +msgid "Reference it from your adapter config:" +msgstr "" + +#: ../../source/mp/serde.rst:106 +msgid "Notes" +msgstr "" + +#: ../../source/mp/serde.rst:108 +msgid "" +"**Buffer size.** ``estimate_serialized_size(layout)`` must return an " +"upper bound on the actual serialized output — include any safety margin " +"directly in the estimate (e.g., the built-in fp8 serializer returns ``1.5" +" * num_elements``)." +msgstr "" + +#: ../../source/mp/serde.rst:112 +msgid "" +"**Failure handling.** If any step fails (serialize, store, load, or " +"deserialize), the whole submitted batch is reported as failed — partial " +"success within one batch is not surfaced. Failed keys are cleaned up " +"automatically." +msgstr "" + +#: ../../source/mp/serde.rst:116 +msgid "" +"**Thread pool.** ``AsyncSerdeProcessor(max_workers=N)`` controls the pool" +" size. Transforms that release the GIL (e.g., torch ops) benefit from ``N" +" > 1``; pure-Python transforms do not." +msgstr "" + +#: ../../source/mp/serde.rst:122 +msgid "Example" +msgstr "" + +#: ../../source/mp/serde.rst:124 +msgid "" +"An end-to-end script that starts an lmcache server with fp8 on a disk " +"adapter, runs vLLM, clears L1, and re-runs the same request to trigger " +"the L2 prefetch + fp8 deserialize path lives at " +":file:`examples/serde/fp8/`. A pytest-based filesystem round-trip test " +"(no vLLM required) is at " +":file:`tests/v1/distributed/serde/test_serde_fs_e2e.py`." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/tracing_and_debugging.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/tracing_and_debugging.po new file mode 100644 index 00000000000..397d01e3e86 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/tracing_and_debugging.po @@ -0,0 +1,469 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/mp/tracing_and_debugging.rst:2 +msgid "Tracing and Debugging" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:4 +msgid "" +"LMCache MP mode can record every ``StorageManager`` public-API call to a " +"binary **trace file** and reissue those calls later against a fresh " +"server via ``lmcache trace replay``. The feature is designed for:" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:8 +msgid "" +"**Regression hunting** — capture a production workload, then replay it " +"against a build under investigation to reproduce a bug offline." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:10 +msgid "" +"**Performance characterization** — measure L1/L2 latency distributions " +"under a realistic storage-level access pattern, without needing vLLM or a" +" GPU." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:13 +msgid "" +"**Configuration tuning** — replay the same trace against different L1 " +"sizes, eviction policies, and L2 adapters to compare their behavior on " +"identical input." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:19 +msgid "" +"Trace recording is **independent** from ``--enable-tracing`` (OTel " +"spans). OTel tracing exports *live* spans to an OTLP endpoint for online " +"observability; trace recording persists a replayable binary file for " +"offline analysis. Both can be enabled simultaneously." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:28 +msgid "Recording a Trace" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:30 +msgid "" +"Recording is **off by default**. Enable it by adding ``--trace-level " +"storage`` to ``lmcache server``:" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:48 +msgid "" +"Drive traffic through the server as usual (vLLM requests, benchmark " +"scripts, etc.). The trace file is closed cleanly on ``SIGTERM`` via the " +"EventBus stop path — no ``--stop-tracing`` command needed." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:52 +msgid "**What is captured:**" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:54 +msgid "" +"The fully-qualified name of every decorated ``StorageManager`` call (e.g." +" ``StorageManager.reserve_write``, " +"``StorageManager.submit_prefetch_task``)." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:57 +msgid "" +"Each call's input arguments (``keys``, ``layout_desc``, ``mode``, " +"``extra_count``, ``external_request_id``, …)." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:59 +msgid "Wall-clock and monotonic timestamps per call." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:60 +msgid "" +"A header with the file format version, trace schema version, start " +"timestamps, and a SHA-256 digest of the active ``StorageManagerConfig`` " +"so replay can flag mismatched configurations." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:65 +msgid "**What is not captured:**" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:67 +msgid "" +"**KV tensor bytes.** Replay exercises bookkeeping and controller logic; " +"payloads at replay time are zeros. The trace file stays bounded even for " +"long runs." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:70 +msgid "" +"Calls inside ``MPCacheEngine``, the message queue, or GPU-copy code. " +"Those layers are out of scope for the ``storage`` trace level." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:73 +msgid "**Overhead:**" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:75 +msgid "Off: a single boolean check per ``StorageManager`` call. Effectively free." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:76 +msgid "" +"On: encoding and file I/O happen on the EventBus drain thread, off the " +"request path. In practice this has no visible impact on request latency." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:81 +msgid "Inspecting a Trace" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:83 +msgid "Before replaying, ``lmcache trace info`` prints a one-screen summary:" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:106 +msgid "" +"Use this to sanity-check that the trace you intend to replay covers the " +"expected operation mix and duration." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:110 +msgid "Replaying a Trace" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:112 +msgid "" +"``lmcache trace replay FILE`` reissues every recorded call against a " +"**fresh** ``StorageManager`` built from CLI flags you supply. The replay-" +"side config is **chosen by you**, not copied from the recording. This is " +"the feature's main value — you can compare different L1/L2 setups on " +"identical input." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:118 +msgid "Minimal invocation:" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:125 +msgid "" +"``--l1-size-gb`` and ``--eviction-policy`` are required, just like on " +"``lmcache server``. Any storage-manager flag accepted by the server also " +"works here (``--l2-adapter``, ``--l1-use-lazy``, ``--l2-store-policy``, " +"…); run ``lmcache trace replay --help`` for the full list." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:131 +msgid "" +"**Pacing.** The driver always honors the recorded inter-call timings by " +"sleeping to align each dispatch with its recorded ``t_mono`` offset. " +"There is **no** as-fast-as-possible mode: ``StorageManager`` reads and " +"writes are asynchronous and carry cross-call dependencies (for example, a" +" retrieve may depend on an earlier L2 load completing), so collapsing the" +" recorded gaps races the internal queues and causes non-deterministic " +"retrieve misses. If the replay host is slower than the recording host, " +"the loop simply lags the recorded schedule." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:140 +msgid "" +"**Output.** Every replay prints a terminal metrics table and writes a " +"per-qualname CSV by default:" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:160 +msgid "Additional per-record output is controlled by:" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:166 +#: ../../source/mp/tracing_and_debugging.rst:215 +msgid "Flag" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:167 +msgid "Purpose" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:168 +msgid "``--output-dir DIR``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:169 +msgid "Directory for aggregated summary files. Default: current dir." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:170 +msgid "``--no-csv``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:171 +msgid "Skip the ``trace_replay_ops.csv`` export." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:172 +msgid "``--json``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:173 +msgid "" +"Also write ``trace_replay_summary.json`` (per-qualname count / mean / p50" +" / p90 / p99 / min / max, plus total duration)." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:176 +msgid "``--verbose``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:177 +msgid "" +"Print one ``[N/total] OK|FAIL (Xms)`` line per record to " +"stdout in addition to the INFO log." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:179 +msgid "``--jsonl-out PATH``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:180 +#, python-brace-format +msgid "" +"Write one JSON object per replayed record to ``PATH`` (``{qualname, " +"latency_ms, failed}``) for post-hoc analysis." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:183 +msgid "``-q`` / ``--quiet``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:184 +msgid "" +"Suppress the terminal metrics table. The aggregated files are still " +"written." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:187 +msgid "Even without ``--verbose``, the driver logs each dispatch at INFO:" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:195 +msgid "" +"Progress numbers come from a cheap pre-scan of the trace file, so you " +"always see ``[N/total]`` rather than just a running counter." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:199 +msgid "Monitoring During Replay" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:201 +msgid "" +"The replay driver initializes the full observability EventBus **before** " +"constructing the replay-side ``StorageManager``. Internal events (L1/L2 " +"lifecycle, eviction ticks, store/retrieve publishes, etc.) therefore flow" +" through a live bus during replay and the standard subscribers — logging," +" metrics, OTel tracing — can attach to them." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:208 +msgid "" +"The same observability CLI flags that the server accepts are available on" +" ``lmcache trace replay``:" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:216 +msgid "Effect" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:217 +msgid "``--disable-observability``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:218 +msgid "Turn the EventBus off entirely. No subscribers fire." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:219 +msgid "``--disable-metrics``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:220 +msgid "" +"Skip OTel metrics init and metrics subscribers. Useful to avoid binding " +"the Prometheus port when you only want logs." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:222 +msgid "``--disable-logging``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:223 +msgid "Skip logging subscribers." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:224 +msgid "``--enable-tracing``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:225 +msgid "Enable OTel span subscribers. Requires ``--otlp-endpoint``." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:226 +msgid "``--otlp-endpoint URL``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:227 +msgid "" +"Export metrics/traces to an OTLP gRPC collector (e.g. " +"``http://localhost:4317``). When unset, metrics fall back to the in-" +"process Prometheus pull endpoint." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:230 +msgid "``--prometheus-port PORT``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:231 +msgid "" +"Port for the Prometheus ``/metrics`` endpoint in pull mode. Default " +"``9090``." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:233 +msgid "``--metrics-sample-rate FLOAT``" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:234 +msgid "Sampling rate for lifecycle histograms. Counters always count all events." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:237 +msgid "Typical monitoring setups:" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:239 +msgid "**Raw log trail (SM/L1/L2 events to stdout):**" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:247 +msgid "**Prometheus metrics in pull mode:**" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:256 +msgid "**OTel metrics + traces to a collector:**" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:267 +msgid "" +"The ``--trace-level`` and ``--trace-output`` flags are **recording-only**" +" and are not accepted by ``lmcache trace replay``. A replay never writes " +"a new trace file." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:272 +msgid "Notes, Hints, and Caveats" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:274 +msgid "" +"**Retrieve misses are expected when the replay environment differs.** At " +"replay start, the CLI prints a visible warning banner:" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:283 +msgid "" +"Because KV payloads are not captured and the replay-side config and host " +"speed may differ from recording, retrieve calls that hit at record time " +"can miss at replay time — for instance, an async L2 load that had " +"finished by the time the recorded retrieve was issued may still be in " +"flight when the replayed retrieve fires. Treat retrieve-miss counts as a " +"signal about the replay environment, **not** as a defect in the trace." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:291 +msgid "" +"**Config-digest mismatch is informational, not fatal.** The replay always" +" runs whether the digests match or not. A mismatch simply tells you the " +"replay-side ``StorageManagerConfig`` differs from what was recorded — " +"often exactly what you intended (comparing two configs on the same " +"trace)." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:297 +msgid "" +"**Prometheus port binding.** The server's ``--prometheus-port`` defaults " +"to ``9090``. Running ``lmcache trace replay`` concurrently with the " +"server — or running two replays at once — on the same port will fail. " +"Either pass a different ``--prometheus-port`` or ``--disable-metrics`` on" +" the secondary runs." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:303 +msgid "" +"**Trace-recording overhead.** Recording happens on the EventBus drain " +"thread, not the request-handling threads. The gate is a single boolean " +"check when disabled (default), so production builds with recording off " +"pay no measurable cost." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:308 +msgid "" +"**Trace files are not encrypted.** Arguments such as ``ObjectKey`` chunk " +"hashes are written in plaintext. Treat trace files with the same care as " +"cache hash logs." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:312 +msgid "" +"**Forward compatibility.** The header carries a format version and a " +"trace schema version. Readers reject files with unknown versions rather " +"than silently producing garbage. Captured API surface changes (new " +"arguments on a traced method, new codec tags) bump the schema version; " +"framing changes bump the format version." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:318 +msgid "" +"**Extensibility.** The format is designed to accommodate future trace " +"**levels** (``mq``, ``gpu``). Adding a new traced method in an existing " +"level requires only decorating it on the recording side and registering a" +" handler on the replay side — no format changes." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:324 +msgid "See Also" +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:326 +msgid "" +":ref:`trace-recording` — the short ``Trace Recording`` section in the " +"Observability page focuses on the recording-side flags." +msgstr "" + +#: ../../source/mp/tracing_and_debugging.rst:328 +msgid "" +"``docs/design/v1/mp_observability/trace.md`` in the source tree — full " +"design doc: architecture, replay dispatcher, context-manager pairing, " +"stats collector, and test matrix." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/encoder_cache.po b/docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/encoder_cache.po new file mode 100644 index 00000000000..08406d1c978 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/encoder_cache.po @@ -0,0 +1,282 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/non_kv_cache/encoder_cache.rst:2 +msgid "Encoder caching" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:4 +msgid "" +"The **Encoder Cache (EC)** stores the output of a multimodal model's " +"encoder stage, keyed by vLLM's per-input ``mm_hash``. When two requests " +"share a multimodal input — the same image, video, or audio clip — the " +"second request loads the encoder output from the cache and the encoder " +"does not run." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:10 +msgid "" +"This applies to any modality vLLM exposes through its encoder-cache " +"extension point: vision encoders (CLIP / ViT-style towers used for images" +" and sampled video frames), audio encoders (Whisper-style towers used for" +" raw waveforms), and combined-modality encoders such as Qwen2.5-Omni. The" +" connector is modality-agnostic — it caches a tensor of shape " +"``[num_tokens, hidden_dim]`` keyed by ``mm_hash``, without knowing which " +"encoder produced it." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:18 +msgid "" +"vLLM exposes the encoder-cache extension point via ``ECConnectorBase`` " +"(vLLM v1 only). LMCache provides an ``LMCacheECConnector`` shim on the " +"vLLM side and an ``ECCacheEngine`` on the LMCache side; together they " +"back the encoder cache with any of LMCache's storage backends (local CPU," +" local disk, remote, NIXL)." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:25 +msgid "Enabling it" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:27 +msgid "Pass ``--ec-transfer-config`` to ``vllm serve``:" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:38 +msgid "" +"``ec_role`` choices: ``ec_producer`` (saves only), ``ec_consumer`` (reads" +" only), ``ec_both`` (single-instance default)." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:41 +msgid "" +"Set ``LMCACHE_CONFIG_FILE`` to point at a YAML with at least one storage " +"backend configured for EC:" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:52 +msgid "" +"To size EC storage independently from the (separate) KV cache, prefix " +"overrides with ``ec_`` in YAML or ``LMCACHE_EC_`` in the environment " +"(e.g. ``ec_max_local_disk_size: 64`` or " +"``LMCACHE_EC_MAX_LOCAL_DISK_SIZE=64``). EC and KV always run with " +"**separate** ``StorageManager`` instances so one cannot evict the other." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:59 +msgid "" +"If you don't set ``local_disk`` (or its EC override) the engine still " +"starts, but EC entries live in CPU memory only and do not survive process" +" restart. Set ``local_disk`` (or ``ec_local_disk``) to a real path if you" +" want cache persistence — there is no implicit on-disk default location." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:66 +msgid "Verifying it's working" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:68 +msgid "Three independent signals:" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:70 +msgid "" +"**vLLM metric.** ``loggers.py`` reports ``MM cache hit rate: X%`` after " +"warm requests." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:72 +msgid "" +"**LMCache log line.** Cold (first-time) requests emit ``LMCache INFO: EC " +"put: stored N bytes for mm_hash=H``. Warm requests emit no ``EC put``." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:75 +msgid "" +"**On-disk file.** Under ``local_disk`` an entry of the form " +"``@1@0@@.pt`` appears after the first request " +"and is reused thereafter. The ``@1@0@`` prefix reflects sentinel " +"``world_size=1, worker_id=0`` in the EC cache key, so all tensor-parallel" +" ranks share one entry." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:82 +msgid "Design notes (user-visible)" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:84 +msgid "" +"**Cache key uses sentinel TP shape.** Encoder outputs are replicated " +"across TP ranks, so the EC key uses ``world_size=1, worker_id=0`` " +"regardless of the deployment's actual TP. Concurrent puts from N ranks " +"land on the same key with identical contents." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:89 +msgid "" +"**Dtype decoupled from KV quant.** The dtype field of the EC cache key is" +" the encoder output dtype (``vllm_config.model_config.dtype``), not " +"``metadata.kv_dtype``. Changing KV quantization does not invalidate EC " +"entries." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:93 +msgid "" +"**Separate StorageManager from KV.** KV and EC have very different access" +" patterns (KV chunked / layerwise / high-volume; EC single-tensor / " +"request-scoped). Sharing one allocator pool would let one cache evict the" +" other in unpredictable ways. Per-cache sizing knobs (``ec_max_local_*``)" +" are explicit instead." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:98 +msgid "" +"**Connector role pinned to \"worker\".** vLLM's ``ECConnectorBase`` is " +"dual-role (scheduler and worker). The LMCache connector calls " +"``create_lmcache_metadata(..., role=\"worker\")`` regardless, because the" +" scheduler-side ``has_cache_item`` needs a fully constructed " +"``StorageManager`` and LMCache currently aborts disk-backend setup when " +"``metadata.role == \"scheduler\"``." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:105 +msgid "" +"The full internal design (class layering, code paths, follow-up work) " +"lives at :file:`docs/design/v1/encoder-cache.md` in the source tree." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:109 +msgid "Benchmark" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:111 +msgid "" +"Live measurement on a single H100 80GB with ``Qwen/Qwen2.5-VL-7B-" +"Instruct`` (bf16) and Big Buck Bunny (10:34, 720p, ≈ 60 MB MP4). Same " +"chat-completion request sent 1 cold + N warm times against the same vLLM " +"server." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:116 +msgid "" +"Two configurations, varying only ``num_frames`` (how many frames vLLM " +"samples from the video):" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:123 +msgid "num_frames" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:124 +msgid "EC entry" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:125 +msgid "Cold TTFT (s)" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:126 +msgid "Warm TTFT mean (s)" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:127 +msgid "Saved" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:128 +msgid "Speedup" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:129 +msgid "32 (vLLM default)" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:130 +msgid "34.3 MB" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:131 +msgid "3.923" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:132 +msgid "3.125" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:133 +msgid "798 ms" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:134 +msgid "**1.26×**" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:135 +msgid "128" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:136 +msgid "130.8 MB" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:137 +msgid "5.895" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:138 +msgid "3.375" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:139 +msgid "2.52 s" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:140 +msgid "**1.75×**" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:142 +msgid "" +"Speedup grows with ``num_frames`` because the encoder workload scales " +"linearly with frame count while the rest of prefill (LM forward over the " +"resulting multimodal tokens + the short text prompt) scales sublinearly. " +"The same principle applies to other modalities: the win is largest when " +"the encoder is the dominant share of prefill (long videos at high frame " +"counts, long audio clips, large images at high resolution) and smallest " +"when text prefill dominates." +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:151 +msgid "Reproducing" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:153 +msgid "Server (heavier-encoder configuration):" +msgstr "" + +#: ../../source/non_kv_cache/encoder_cache.rst:169 +msgid "" +"Client: any streaming OpenAI-compatible client that re-sends the same " +"multimodal payload. The benchmark measures TTFT (time to first token) " +"because the encoder runs during prefill — any encoder savings show up " +"there. Decode tokens-per-second is unaffected by EC." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/docker_deployment.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/docker_deployment.po new file mode 100644 index 00000000000..1ad304238e4 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/docker_deployment.po @@ -0,0 +1,79 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/docker_deployment.rst:4 +msgid "Docker deployment" +msgstr "" + +#: ../../source/production/docker_deployment.rst:6 +msgid "**Prerequisites:** Docker Engine 27.0+" +msgstr "" + +#: ../../source/production/docker_deployment.rst:8 +msgid "See :ref:`installation_guide` for pulling images." +msgstr "" + +#: ../../source/production/docker_deployment.rst:11 +msgid "Running the container" +msgstr "" + +#: ../../source/production/docker_deployment.rst:27 +msgid "" +"See the `docker run example " +"`_ for more details." +msgstr "" + +#: ../../source/production/docker_deployment.rst:30 +msgid "ROCm (AMD)" +msgstr "" + +#: ../../source/production/docker_deployment.rst:32 +msgid "" +"The `AMD Infinity hub `__ for " +"vLLM offers a prebuilt, optimized image for the AMD Instinct™ MI300X. See" +" `LLM inference performance validation on AMD Instinct MI300X " +"`__ for full " +"instructions." +msgstr "" + +#: ../../source/production/docker_deployment.rst:37 +msgid "" +"Validated environment: ``rocm/vllm-" +"dev:nightly_0624_rc2_0624_rc2_20250620``, MI300X, vLLM V1." +msgstr "" + +#: ../../source/production/docker_deployment.rst:56 +msgid "XPU (Intel)" +msgstr "" + +#: ../../source/production/docker_deployment.rst:58 +msgid "" +"The `Intel vLLM XPU hub`__ offers a " +"prebuilt, optimized docker image designed for validating inference " +"performance on the Intel GPU accelerators like ARC770, B60/B70, and " +"future products." +msgstr "" + +#: ../../source/production/docker_deployment.rst:60 +msgid "Validated environment: ``intel/vllm:0.17.0-xpu``, Intel B60 GPU, vLLM V1." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/kubernetes_deployment.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/kubernetes_deployment.po new file mode 100644 index 00000000000..b65fd6efbbd --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/kubernetes_deployment.po @@ -0,0 +1,58 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/kubernetes_deployment.rst:2 +msgid "Kubernetes Deployment" +msgstr "" + +#: ../../source/production/kubernetes_deployment.rst:4 +msgid "" +"For Kubernetes deployment of vLLM with LMCache integration, we recommend " +"using the `vLLM Production Stack `_ project. This is a specialized production-ready " +"implementation for K8S-native cluster-wide deployment for vllm & lmcache." +msgstr "" + +#: ../../source/production/kubernetes_deployment.rst:6 +msgid "" +"For a quick start guide, please refer to the official `documentation " +"`_" +msgstr "" + +#: ../../source/production/kubernetes_deployment.rst:8 +msgid "" +"and replace the Helm values file with (`values-05-cpu-offloading.yaml " +"`_):" +msgstr "" + +#: ../../source/production/kubernetes_deployment.rst:35 +msgid "OR" +msgstr "" + +#: ../../source/production/kubernetes_deployment.rst:37 +msgid "" +"refer to a detailed `step-by-step tutorial `_ on" +" how to offload KV cache with LMCache in the production stack." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/kv_cache_events.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/kv_cache_events.po new file mode 100644 index 00000000000..134b07112c4 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/kv_cache_events.po @@ -0,0 +1,186 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/kv_cache_events.rst:4 +msgid "KV Cache Events" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:6 +msgid "" +"KV cache events are actions or lifecycle events that occur when managing " +"the KV cache during inference. These events can be used for KV-cache-" +"aware routing." +msgstr "" + +#: ../../source/production/kv_cache_events.rst:8 +msgid "LMCache supports KV cache events as follows:" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:10 +msgid "Generates storage KV cache events" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:11 +msgid "" +"The events format is defined as per the `BlockStored class " +"`_ in vLLM" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:12 +msgid "" +"LMCache passes the events to SGLang or vLLM to publish them using their " +"messaging system" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:15 +msgid "Prerequisites" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:17 +msgid "The following prerequisites are required:" +msgstr "" + +#: ../../source/production/kv_cache_events.rst +msgid "vLLM" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:23 +msgid "vLLM v0.13.0+" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:24 +msgid "LMCache v0.3.11+" +msgstr "" + +#: ../../source/production/kv_cache_events.rst +msgid "SGLang" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:28 +msgid "SGLang vx.y.z+" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:29 +msgid "LMCache vx.y.z+" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:32 +msgid "How to Generate KV Cache events" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:38 +#: ../../source/production/kv_cache_events.rst:89 +msgid "" +"Before starting to generate KV events, you need to be aware of the " +"following:" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:40 +#: ../../source/production/kv_cache_events.rst:91 +msgid "" +"You need to enable ``enable_kv_events`` for LMCache as events are not " +"generated by default." +msgstr "" + +#: ../../source/production/kv_cache_events.rst:41 +msgid "" +"If running more than 1 worker in vLLM, you need to use a non-default " +"hashing algorithm (set ``pre_caching_hash_algorithm`` in LMCache) so that" +" hashes generated per worker are the same. If not then you will have " +"duplicate events for the same operation as events are generated per " +"worker." +msgstr "" + +#: ../../source/production/kv_cache_events.rst:42 +msgid "" +"LMCache sends the events to vLLM for publishing. To enable events to be " +"published, you need to set the vLLM configuration setting ``--kv-events-" +"config``. See `vLLM KV Events configuration " +"`_ for more " +"details." +msgstr "" + +#: ../../source/production/kv_cache_events.rst:44 +#: ../../source/production/kv_cache_events.rst:94 +msgid "" +"The steps that follow give an example of how KV events can be generated, " +"published and consumed:" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:46 +msgid "Start vLLM with LMCache and model ``Qwen/Qwen3-0.6B`` as follows:" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:54 +#: ../../source/production/kv_cache_events.rst:107 +msgid "Example of the LMCache configuration is as follows:" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:63 +msgid "" +"To be able to process the events that are published by vLLM, you need a " +"client that subscribes to the publisher message channel and can consume " +"the events. vLLM provides such a client example `KV Events Subscriber " +"`_." +" Run this python script in a separate terminal." +msgstr "" + +#: ../../source/production/kv_cache_events.rst:65 +#: ../../source/production/kv_cache_events.rst:119 +msgid "Prompt the model:" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:78 +#: ../../source/production/kv_cache_events.rst:133 +msgid "" +"You should receive a message in the client (that you started in step 2.) " +"window, similar to the following:" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:85 +#: ../../source/production/kv_cache_events.rst:155 +msgid "This is the event generated after the cache store operation." +msgstr "" + +#: ../../source/production/kv_cache_events.rst:92 +msgid "" +"LMCache sends the events to SGLang for publishing. To enable events to be" +" published, you need to set the SGLang configuration setting ``--kv-" +"events-config``." +msgstr "" + +#: ../../source/production/kv_cache_events.rst:96 +msgid "Start SGLang with LMCache and model ``Qwen/Qwen3-0.6B`` as follows:" +msgstr "" + +#: ../../source/production/kv_cache_events.rst:117 +msgid "" +"To be able to process the events that are published by SGLang, you need a" +" client that subscribes to the publisher message channel and can consume " +"the events. vLLM provides such a client example `KV Events Subscriber " +"`_." +" To use this client for SGLang, you need to remove the properties " +"``medium`` and ``lora_name`` from the ``BlockStored`` class definition " +"and ``medium`` from ``BlockRemoved`` class definition. Save the changes " +"and run this updated python script in a separate terminal." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/chunk_statistics.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/chunk_statistics.po new file mode 100644 index 00000000000..290bbd57f3c --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/chunk_statistics.po @@ -0,0 +1,580 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/observability/chunk_statistics.rst:4 +msgid "Chunk Statistics" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:6 +msgid "" +"The chunk statistics feature provides insights into KV cache chunk reuse " +"patterns, helping you understand cache efficiency and optimize your " +"deployment." +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:9 +msgid "Overview" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:11 +msgid "" +"Chunk statistics tracks and analyzes KV cache chunks to provide metrics " +"on:" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:13 +msgid "" +"**Total chunks processed**: The total number of chunks that have been " +"processed" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:14 +msgid "**Unique chunks**: The number of distinct chunks encountered" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:15 +msgid "**Duplicate chunks**: The number of repeated chunks" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:16 +msgid "" +"**Reuse rate**: The ratio of duplicate chunks to total chunks, indicating" +" cache efficiency" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:18 +msgid "This information is valuable for:" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:20 +msgid "Understanding cache hit patterns" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:21 +msgid "Optimizing cache size and eviction policies" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:22 +msgid "Analyzing workload characteristics" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:23 +msgid "Capacity planning for production deployments" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:26 +msgid "Recording Strategies" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:28 +msgid "" +"LMCache supports multiple recording strategies, each optimized for " +"different use cases:" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:31 +msgid "Memory Bloom Filter Strategy" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:33 +msgid "" +"A memory-efficient strategy using Bloom filters for probabilistic " +"duplicate detection." +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:35 +#: ../../source/production/observability/chunk_statistics.rst:64 +msgid "**Advantages:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:37 +msgid "Low memory footprint" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:38 +msgid "Fast lookup operations" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:39 +msgid "Suitable for large-scale deployments" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:41 +#: ../../source/production/observability/chunk_statistics.rst:70 +msgid "**Configuration:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:51 +#: ../../source/production/observability/chunk_statistics.rst:81 +msgid "**Environment Variables:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:60 +msgid "File Hash Strategy" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:62 +msgid "" +"A file-based strategy that writes chunk hashes to disk for exact tracking" +" and offline analysis." +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:66 +msgid "Exact duplicate detection (no false positives)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:67 +msgid "Persistent storage for offline analysis" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:68 +msgid "Automatic file rotation and cleanup" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:90 +msgid "Quick Start Guide" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:93 +msgid "Step 1: Enable Chunk Statistics" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:95 +msgid "Configure your LMCache instance with chunk statistics enabled:" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:97 +msgid "**Using YAML Configuration:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:118 +msgid "**Using vLLM with LMCache:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:135 +msgid "Step 2: Access Statistics" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:137 +msgid "Retrieve statistics through the internal API server:" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:144 +msgid "**Example Response:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:187 +msgid "Configuration Options" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:190 +msgid "Basic Configuration" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:192 +msgid "Basic Configuration Options" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:196 +#: ../../source/production/observability/chunk_statistics.rst:224 +#: ../../source/production/observability/chunk_statistics.rst:243 +msgid "Configuration Key" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:197 +#: ../../source/production/observability/chunk_statistics.rst:225 +#: ../../source/production/observability/chunk_statistics.rst:244 +msgid "Default Value" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:198 +#: ../../source/production/observability/chunk_statistics.rst:226 +#: ../../source/production/observability/chunk_statistics.rst:245 +msgid "Description" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:199 +msgid "``enable_chunk_statistics``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:200 +#: ../../source/production/observability/chunk_statistics.rst:206 +msgid "``false``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:201 +msgid "Enable chunk statistics tracking" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:202 +msgid "``chunk_statistics_strategy``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:203 +msgid "``memory_bloom_filter``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:204 +msgid "Recording strategy: ``memory_bloom_filter`` or ``file_hash``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:205 +msgid "``chunk_statistics_auto_start_statistics``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:207 +msgid "Automatically start statistics collection on initialization" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:208 +msgid "``chunk_statistics_auto_exit_timeout_hours``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:209 +msgid "``0.0``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:210 +msgid "Auto-stop after specified hours (0 = disabled)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:211 +msgid "``chunk_statistics_auto_exit_target_unique_chunks``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:212 +msgid "``0``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:213 +msgid "Auto-stop after reaching target unique chunks (0 = disabled)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:216 +msgid "Memory Bloom Filter Options" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:218 +#: ../../source/production/observability/chunk_statistics.rst:237 +msgid "Configure these options in the ``extra_config`` section:" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:220 +msgid "Bloom Filter Configuration" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:227 +msgid "``chunk_statistics_mem_bf_expected_chunks``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:228 +msgid "``20000000``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:229 +msgid "Expected number of chunks for capacity planning" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:230 +msgid "``chunk_statistics_mem_bf_false_positive_rate``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:231 +msgid "``0.01``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:232 +msgid "Target false positive rate (1%)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:235 +msgid "File Hash Options" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:239 +msgid "File Hash Configuration" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:246 +msgid "``chunk_statistics_file_output_dir``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:247 +msgid "``/tmp/lmcache_chunk_statistics``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:248 +msgid "Directory for storing chunk hash files" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:249 +msgid "``chunk_statistics_file_rotation_size``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:250 +msgid "``104857600``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:251 +msgid "File size threshold for rotation (bytes, default 100MB)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:252 +msgid "``chunk_statistics_file_max_count``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:253 +msgid "``100``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:254 +msgid "Maximum number of files to keep" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:257 +msgid "Advanced Usage" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:260 +msgid "Programmatic Control" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:262 +msgid "Control statistics collection programmatically through the internal API:" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:282 +msgid "Auto-Stop Configuration" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:284 +msgid "Configure automatic stopping based on time or chunk count:" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:291 +msgid "Prometheus Metrics" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:293 +msgid "" +"When using the internal API server, chunk statistics are exposed as " +"Prometheus metrics:" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:295 +msgid "" +"``lmcache_chunk_statistics_total_chunks``: Total number of chunks " +"processed" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:296 +msgid "``lmcache_chunk_statistics_unique_chunks``: Number of unique chunks" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:297 +msgid "``lmcache_chunk_statistics_reuse_rate``: Cache reuse rate (0.0 to 1.0)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:298 +msgid "" +"``lmcache_chunk_statistics_bloom_filter_size_mb``: Bloom filter memory " +"usage (MB)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:299 +msgid "" +"``lmcache_chunk_statistics_bloom_filter_fill_rate``: Bloom filter fill " +"rate (0.0 to 1.0)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:300 +msgid "``lmcache_chunk_statistics_file_count``: Number of hash files created" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:301 +msgid "``lmcache_chunk_statistics_current_file_size``: Current file size (bytes)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:304 +msgid "Offline Analysis" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:306 +msgid "" +"For the file hash strategy, you can perform detailed offline analysis of " +"collected chunk hash data." +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:309 +msgid "Using the Analysis Script" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:311 +msgid "" +"LMCache provides a comprehensive analysis script at " +"``examples/chunk_statistics/analyze_chunk_hashes.py`` that supports " +"multiple analysis modes." +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:314 +msgid "Best Practices" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:316 +msgid "**Choose the Right Strategy:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:318 +msgid "Use **memory_bloom_filter** for real-time monitoring with minimal overhead" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:319 +msgid "Use **file_hash** when exact tracking is required or for offline analysis" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:321 +msgid "**Tune Bloom Filter Parameters:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:323 +msgid "Set ``expected_chunks`` based on your workload size" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:324 +msgid "Lower ``false_positive_rate`` increases memory usage but improves accuracy" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:326 +msgid "**Monitor Memory Usage:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:328 +msgid "" +"Track ``bloom_filter_size_mb`` metric to ensure it fits in available " +"memory" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:329 +msgid "Adjust ``expected_chunks`` if memory usage is too high" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:331 +msgid "**File Rotation:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:333 +msgid "" +"Configure appropriate ``file_rotation_size`` to balance file size and " +"count" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:334 +msgid "Set ``file_max_count`` to prevent unlimited disk usage" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:336 +msgid "**Production Deployment:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:338 +msgid "Enable auto-stop to prevent indefinite data collection" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:339 +msgid "Use internal API server for centralized metrics collection" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:340 +msgid "Integrate with your monitoring stack (Prometheus, Grafana, etc.)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:343 +msgid "Troubleshooting" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:346 +msgid "Statistics Not Updating" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:348 +msgid "**Issue:** Statistics remain at zero or don't update." +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:350 +#: ../../source/production/observability/chunk_statistics.rst:361 +#: ../../source/production/observability/chunk_statistics.rst:372 +msgid "**Solutions:**" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:352 +msgid "Verify ``enable_chunk_statistics`` is set to ``true``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:353 +msgid "Check that statistics collection is started (auto-start or manual start)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:354 +msgid "Ensure requests are being processed by the LMCache instance" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:357 +msgid "High Memory Usage" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:359 +msgid "**Issue:** Bloom filter consuming too much memory." +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:363 +msgid "Reduce ``chunk_statistics_mem_bf_expected_chunks``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:364 +msgid "" +"Increase ``chunk_statistics_mem_bf_false_positive_rate`` (trade accuracy " +"for memory)" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:365 +msgid "Consider switching to ``file_hash`` strategy" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:368 +msgid "File System Full" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:370 +msgid "**Issue:** Disk space exhausted with file hash strategy." +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:374 +msgid "Reduce ``chunk_statistics_file_max_count``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:375 +msgid "Decrease ``chunk_statistics_file_rotation_size``" +msgstr "" + +#: ../../source/production/observability/chunk_statistics.rst:376 +msgid "Implement external log rotation or archival" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/frontend.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/frontend.po new file mode 100644 index 00000000000..2b12b6590f6 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/frontend.po @@ -0,0 +1,222 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/observability/frontend.rst:4 +msgid "LMCache Frontend" +msgstr "" + +#: ../../source/production/observability/frontend.rst:6 +msgid "" +"LMCache Frontend is a monitoring and proxy service for LMCache clusters, " +"providing a web interface for cluster management and HTTP request " +"proxying to cluster nodes." +msgstr "" + +#: ../../source/production/observability/frontend.rst:10 +msgid "LMCache Frontend Dashboard" +msgstr "" + +#: ../../source/production/observability/frontend.rst:16 +msgid "Features" +msgstr "" + +#: ../../source/production/observability/frontend.rst:18 +msgid "**Cluster Monitoring**: Web-based dashboard for visualizing cluster status" +msgstr "" + +#: ../../source/production/observability/frontend.rst:19 +msgid "" +"**Request Proxying**: HTTP proxy service to forward requests to any " +"cluster node" +msgstr "" + +#: ../../source/production/observability/frontend.rst:20 +msgid "" +"**Flexible Configuration**: Support for both IP:port and Unix domain " +"sockets" +msgstr "" + +#: ../../source/production/observability/frontend.rst:21 +msgid "**Plugin System**: Integration with LMCache plugin framework" +msgstr "" + +#: ../../source/production/observability/frontend.rst:25 +msgid "Installation" +msgstr "" + +#: ../../source/production/observability/frontend.rst:27 +msgid "Install from PyPI:" +msgstr "" + +#: ../../source/production/observability/frontend.rst:33 +msgid "Or install from source:" +msgstr "" + +#: ../../source/production/observability/frontend.rst:43 +msgid "Quick Start" +msgstr "" + +#: ../../source/production/observability/frontend.rst:46 +msgid "Starting the Service" +msgstr "" + +#: ../../source/production/observability/frontend.rst:54 +msgid "Command Line Options" +msgstr "" + +#: ../../source/production/observability/frontend.rst:60 +msgid "Option" +msgstr "" + +#: ../../source/production/observability/frontend.rst:61 +msgid "Description" +msgstr "" + +#: ../../source/production/observability/frontend.rst:62 +msgid "``--port``" +msgstr "" + +#: ../../source/production/observability/frontend.rst:63 +msgid "Service port (default: 8000)" +msgstr "" + +#: ../../source/production/observability/frontend.rst:64 +msgid "``--host``" +msgstr "" + +#: ../../source/production/observability/frontend.rst:65 +msgid "Bind host address (default: 0.0.0.0)" +msgstr "" + +#: ../../source/production/observability/frontend.rst:66 +msgid "``--config``" +msgstr "" + +#: ../../source/production/observability/frontend.rst:67 +msgid "Path to configuration file" +msgstr "" + +#: ../../source/production/observability/frontend.rst:68 +msgid "``--nodes``" +msgstr "" + +#: ../../source/production/observability/frontend.rst:69 +msgid "Direct node configuration (JSON string)" +msgstr "" + +#: ../../source/production/observability/frontend.rst:72 +msgid "" +"After starting the service, access the dashboard at " +"``http://localhost:8080/``." +msgstr "" + +#: ../../source/production/observability/frontend.rst:76 +msgid "Configuration" +msgstr "" + +#: ../../source/production/observability/frontend.rst:79 +msgid "Node Configuration" +msgstr "" + +#: ../../source/production/observability/frontend.rst:81 +msgid "Create a ``config.json`` file with node definitions:" +msgstr "" + +#: ../../source/production/observability/frontend.rst:98 +msgid "" +"The ``port`` field can be configured as either an integer port number or " +"a string path for Unix domain sockets." +msgstr "" + +#: ../../source/production/observability/frontend.rst:103 +msgid "LMCache Plugin Integration" +msgstr "" + +#: ../../source/production/observability/frontend.rst:105 +msgid "" +"To start the frontend via the LMCache plugin framework, add the following" +" to your ``lmcache.yaml``:" +msgstr "" + +#: ../../source/production/observability/frontend.rst:119 +msgid "Proxying Requests" +msgstr "" + +#: ../../source/production/observability/frontend.rst:121 +msgid "Proxy requests using the format:" +msgstr "" + +#: ../../source/production/observability/frontend.rst:127 +msgid "Examples:" +msgstr "" + +#: ../../source/production/observability/frontend.rst:140 +msgid "Contributing" +msgstr "" + +#: ../../source/production/observability/frontend.rst:142 +msgid "" +"LMCache Frontend is an open-source project and we welcome contributions " +"from the community! Whether you want to fix bugs, add new features, " +"improve documentation, or share ideas, your contributions are " +"appreciated." +msgstr "" + +#: ../../source/production/observability/frontend.rst:146 +msgid "Ways to contribute:" +msgstr "" + +#: ../../source/production/observability/frontend.rst:148 +msgid "" +"**Report Issues**: Found a bug or have a feature request? Open an issue " +"on GitHub." +msgstr "" + +#: ../../source/production/observability/frontend.rst:150 +msgid "" +"**Submit Pull Requests**: Fork the repository, make your changes, and " +"submit a PR." +msgstr "" + +#: ../../source/production/observability/frontend.rst:152 +msgid "**Improve Documentation**: Help us make the docs clearer and more helpful." +msgstr "" + +#: ../../source/production/observability/frontend.rst:153 +msgid "" +"**Share Feedback**: Let us know how you're using LMCache Frontend and " +"what could be improved." +msgstr "" + +#: ../../source/production/observability/frontend.rst:156 +msgid "Join us in making LMCache Frontend better for everyone!" +msgstr "" + +#: ../../source/production/observability/frontend.rst:160 +msgid "More Information" +msgstr "" + +#: ../../source/production/observability/frontend.rst:162 +msgid "" +"For more details, visit the `LMCache Frontend GitHub repository " +"`_." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/health_monitor.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/health_monitor.po new file mode 100644 index 00000000000..500d4a3ed7c --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/health_monitor.po @@ -0,0 +1,495 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/observability/health_monitor.rst:4 +msgid "Health Monitor" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:6 +msgid "" +"LMCache includes a comprehensive health monitoring framework that " +"continuously monitors the health of the cache engine and its components. " +"This feature is essential for production deployments to detect and " +"respond to failures in remote storage backends." +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:9 +msgid "Overview" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:11 +msgid "The Health Monitor provides:" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:13 +msgid "" +"**Automatic health checks**: Periodically monitors the health of all " +"registered components" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:14 +msgid "" +"**Extensible framework**: Easily add custom health checks for new " +"components" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:15 +msgid "" +"**Remote backend monitoring**: Built-in support for monitoring remote " +"storage backends via ping" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:16 +msgid "" +"**Degraded mode support**: Automatically blocks operations when the " +"system is unhealthy" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:17 +msgid "" +"**Prometheus metrics integration**: Health status exposed via metrics " +"endpoint" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:20 +msgid "Architecture" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:22 +msgid "The health monitoring system consists of three main components:" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:24 +msgid "**HealthCheck (Abstract Base Class)**" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:26 +msgid "" +"Base class for individual health checks. Each health check represents one" +" aspect of system health." +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:28 +msgid "**HealthMonitor**" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:30 +msgid "" +"The central monitor that orchestrates all health checks. It runs in a " +"background thread and periodically executes all registered health checks." +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:32 +msgid "**RemoteBackendHealthCheck**" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:34 +msgid "" +"Built-in health check for remote storage backends. It pings the remote " +"connector to verify connectivity." +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:37 +msgid "Auto-Discovery" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:39 +msgid "" +"The Health Monitor uses an auto-discovery mechanism to find and " +"instantiate health checks:" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:41 +msgid "" +"At startup, the monitor scans the ``lmcache.v1.health_monitor.checks`` " +"package" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:42 +msgid "All classes that inherit from ``HealthCheck`` are discovered" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:43 +msgid "Each check's ``create_from_engine()`` method is called to create instances" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:44 +msgid "The instances are registered with the monitor" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:46 +msgid "" +"This design allows you to add new health checks by simply creating a new " +"module in the checks package." +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:49 +msgid "Configuration" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:51 +msgid "" +"Health monitor configuration is done through the ``extra_config`` section" +" of your LMCache configuration:" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:53 +msgid "Health Monitor Configuration Options" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:57 +msgid "Configuration Key" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:58 +msgid "Default Value" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:59 +#: ../../source/production/observability/health_monitor.rst:148 +#: ../../source/production/observability/health_monitor.rst:175 +msgid "Description" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:60 +msgid "``ping_interval``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:61 +msgid "``30.0``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:62 +msgid "Interval (in seconds) between health check cycles" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:63 +msgid "``ping_timeout``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:64 +msgid "``5.0``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:65 +msgid "Timeout (in seconds) for each ping operation" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:66 +msgid "``get_blocking_failed_threshold``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:67 +msgid "``10``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:68 +msgid "Max number of get_blocking failed count in check interval" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:69 +msgid "``waiting_time_for_recovery``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:70 +msgid "``300.0``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:71 +msgid "Waiting time (in seconds) for recovery if get_blocking failed" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:75 +msgid "How It Works" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:78 +msgid "Runtime Behavior" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:80 +msgid "The health monitor runs in a background thread:" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:82 +msgid "Every ``ping_interval`` seconds, all health checks are executed" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:83 +msgid "If any check fails, the system is marked as unhealthy" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:84 +msgid "When unhealthy, store/retrieve operations are blocked with a warning log" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:85 +msgid "" +"Once all checks pass again, the system is marked as healthy and " +"operations resume" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:88 +msgid "Initialization Failure Handling" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:90 +msgid "When initialization or post-initialization fails irrecoverably:" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:92 +msgid "The system is marked with ``_init_failed = True``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:93 +msgid "``is_healthy()`` method returns ``False`` permanently" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:94 +msgid "" +"Health monitoring thread will not start (if initialization fails before " +"it starts)" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:95 +msgid "The system operates in degraded mode (recompute-only)" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:97 +msgid "" +"This ensures that irrecoverable initialization errors don't cause " +"cascading failures and the system can gracefully fall back to " +"recomputation." +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:101 +msgid "Graceful Degradation" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:103 +msgid "When the health monitor detects an unhealthy state:" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:105 +msgid "**Store operations**: Skipped with a warning message" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:106 +msgid "**Retrieve operations**: Return empty results with a warning message" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:107 +msgid "**Lookup operations**: Return 0 (no cache hits) with a warning message" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:109 +msgid "This prevents cascading failures when remote backends are unavailable." +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:112 +msgid "Built-in Health Checks" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:115 +msgid "RemoteBackendHealthCheck" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:117 +msgid "" +"This check monitors the connectivity to remote storage backends (e.g., " +"Redis, Valkey)." +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:119 +msgid "**What it checks:**" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:121 +msgid "Pings the remote connector to verify it's reachable" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:122 +msgid "Measures ping latency" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:123 +msgid "Reports error codes for failures" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:125 +msgid "**When it's active:**" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:127 +msgid "Only when a remote backend is configured (``remote_url`` is set)" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:128 +msgid "Only if the connector supports the ``ping()`` operation" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:130 +msgid "**Metrics reported:**" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:132 +msgid "``lmcache:remote_ping_latency``: Latest ping latency (milliseconds)" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:133 +msgid "``lmcache:remote_ping_error_code``: Latest error code (0 = success)" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:134 +msgid "``lmcache:remote_ping_errors``: Total number of ping errors" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:135 +msgid "``lmcache:remote_ping_successes``: Total number of successful pings" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:138 +msgid "Prometheus Metrics" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:140 +msgid "The health monitor exposes metrics through the Prometheus endpoint:" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:142 +msgid "Health Monitor Metrics" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:146 +msgid "Metric Name" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:147 +msgid "Type" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:149 +msgid "``lmcache:is_healthy``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:150 +#: ../../source/production/observability/health_monitor.rst:153 +#: ../../source/production/observability/health_monitor.rst:156 +msgid "Gauge" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:151 +msgid "Overall system health status (1 = healthy, 0 = unhealthy)" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:152 +msgid "``lmcache:remote_ping_latency``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:154 +msgid "Latest ping latency to remote backends (milliseconds)" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:155 +msgid "``lmcache:remote_ping_error_code``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:157 +msgid "Latest ping error code (0 = success, -1 = timeout, -2 = generic error)" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:158 +msgid "``lmcache:remote_ping_errors``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:159 +#: ../../source/production/observability/health_monitor.rst:162 +msgid "Counter" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:160 +msgid "Total number of ping errors to remote backends" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:161 +msgid "``lmcache:remote_ping_successes``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:163 +msgid "Total number of successful pings to remote backends" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:166 +msgid "Error Codes" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:168 +msgid "The health check system uses the following error codes:" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:170 +msgid "Health Check Error Codes" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:174 +msgid "Code" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:176 +msgid "``0``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:177 +msgid "Success - the health check passed" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:178 +msgid "``-1``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:179 +msgid "Timeout - the ping operation exceeded the configured timeout" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:180 +msgid "``-2``" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:181 +msgid "Generic error - an unexpected error occurred during the health check" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:184 +msgid "Extending the Health Monitor" +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:186 +msgid "" +"You can add custom health checks by creating a new module in the " +"``lmcache/v1/health_monitor/checks/`` directory." +msgstr "" + +#: ../../source/production/observability/health_monitor.rst:188 +msgid "" +"The custom check will be automatically discovered and registered when " +"LMCache starts." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/index.po new file mode 100644 index 00000000000..118b87314fa --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/index.po @@ -0,0 +1,25 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/observability/index.rst:4 +msgid "Observability" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/internal_api_server.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/internal_api_server.po new file mode 100644 index 00000000000..ed984b28a8d --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/internal_api_server.po @@ -0,0 +1,138 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/observability/internal_api_server.rst:4 +msgid "Internal API Server Metrics" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:6 +msgid "" +"Another approach to retrieve LMCache metrics is to use the internal API " +"server." +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:9 +msgid "Overview" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:11 +msgid "" +"The internal API server exposes Prometheus-compatible metrics endpoints " +"in your LMCache deployment." +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:14 +msgid "Quick Start Guide" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:17 +msgid "Step 1: Enable Internal API Server" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:19 +msgid "Configure your vLLM instance to enable the internal API server:" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:28 +msgid "Step 2: Access Metrics Endpoint" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:30 +msgid "Retrieve metrics from the worker's endpoint:" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:37 +msgid "Port Configuration" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:39 +msgid "" +"The following environment variables are used implicitly with their " +"default values:" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:41 +msgid "Default Port Configuration" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:45 +msgid "Environment Variable" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:46 +msgid "Default Value" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:47 +msgid "Description" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:48 +msgid "``LMCACHE_INTERNAL_API_SERVER_HOST``" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:49 +msgid "``0.0.0.0``" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:50 +msgid "Host address for the internal API server to bind to." +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:51 +msgid "``LMCACHE_INTERNAL_API_SERVER_PORT_START``" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:52 +msgid "``6999``" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:53 +msgid "Starting port number, e.g.:" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:55 +msgid "Scheduler: port_start + 0 (6999)" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:56 +msgid "Worker 0: port_start + 1 (7000)" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:57 +msgid "Worker 1: port_start + 2 (7001)" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:60 +msgid "Therefore, the metrics endpoint curl command above uses port 7000." +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:63 +msgid "Advanced Usage" +msgstr "" + +#: ../../source/production/observability/internal_api_server.rst:65 +msgid "" +"For comprehensive testing and configuration options, refer to " +":ref:`testing_internal_api_server` for detailed examples and best " +"practices." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/metrics.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/metrics.po new file mode 100644 index 00000000000..a54606dacb1 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/metrics.po @@ -0,0 +1,899 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/observability/metrics.rst:4 +msgid "Metrics Reference" +msgstr "" + +#: ../../source/production/observability/metrics.rst:6 +msgid "" +"LMCache provides comprehensive metrics via Prometheus to help you monitor" +" performance, cache efficiency, and system health. These metrics are " +"exposed via the vLLM ``/metrics`` endpoint when LMCache is integrated " +"with vLLM, or via the LMCache internal API server." +msgstr "" + +#: ../../source/production/observability/metrics.rst:9 +msgid "Available Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:11 +msgid "" +"The following tables list all available LMCache metrics organized by " +"category." +msgstr "" + +#: ../../source/production/observability/metrics.rst:14 +#: ../../source/production/observability/metrics.rst:16 +msgid "Core Request Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:20 +#: ../../source/production/observability/metrics.rst:40 +#: ../../source/production/observability/metrics.rst:72 +#: ../../source/production/observability/metrics.rst:95 +#: ../../source/production/observability/metrics.rst:127 +#: ../../source/production/observability/metrics.rst:162 +#: ../../source/production/observability/metrics.rst:185 +#: ../../source/production/observability/metrics.rst:229 +#: ../../source/production/observability/metrics.rst:255 +#: ../../source/production/observability/metrics.rst:278 +#: ../../source/production/observability/metrics.rst:301 +#: ../../source/production/observability/metrics.rst:333 +#: ../../source/production/observability/metrics.rst:371 +msgid "Metric Name" +msgstr "" + +#: ../../source/production/observability/metrics.rst:21 +#: ../../source/production/observability/metrics.rst:41 +#: ../../source/production/observability/metrics.rst:73 +#: ../../source/production/observability/metrics.rst:96 +#: ../../source/production/observability/metrics.rst:128 +#: ../../source/production/observability/metrics.rst:163 +#: ../../source/production/observability/metrics.rst:186 +#: ../../source/production/observability/metrics.rst:230 +#: ../../source/production/observability/metrics.rst:256 +#: ../../source/production/observability/metrics.rst:279 +#: ../../source/production/observability/metrics.rst:302 +#: ../../source/production/observability/metrics.rst:334 +#: ../../source/production/observability/metrics.rst:372 +msgid "Type" +msgstr "" + +#: ../../source/production/observability/metrics.rst:22 +#: ../../source/production/observability/metrics.rst:42 +#: ../../source/production/observability/metrics.rst:74 +#: ../../source/production/observability/metrics.rst:97 +#: ../../source/production/observability/metrics.rst:129 +#: ../../source/production/observability/metrics.rst:164 +#: ../../source/production/observability/metrics.rst:187 +#: ../../source/production/observability/metrics.rst:231 +#: ../../source/production/observability/metrics.rst:257 +#: ../../source/production/observability/metrics.rst:280 +#: ../../source/production/observability/metrics.rst:303 +#: ../../source/production/observability/metrics.rst:335 +#: ../../source/production/observability/metrics.rst:373 +msgid "Description" +msgstr "" + +#: ../../source/production/observability/metrics.rst:23 +msgid "``lmcache:num_retrieve_requests``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:24 +#: ../../source/production/observability/metrics.rst:27 +#: ../../source/production/observability/metrics.rst:30 +#: ../../source/production/observability/metrics.rst:44 +#: ../../source/production/observability/metrics.rst:47 +#: ../../source/production/observability/metrics.rst:50 +#: ../../source/production/observability/metrics.rst:53 +#: ../../source/production/observability/metrics.rst:56 +#: ../../source/production/observability/metrics.rst:59 +#: ../../source/production/observability/metrics.rst:62 +#: ../../source/production/observability/metrics.rst:85 +#: ../../source/production/observability/metrics.rst:114 +#: ../../source/production/observability/metrics.rst:117 +#: ../../source/production/observability/metrics.rst:189 +#: ../../source/production/observability/metrics.rst:192 +#: ../../source/production/observability/metrics.rst:195 +#: ../../source/production/observability/metrics.rst:198 +#: ../../source/production/observability/metrics.rst:213 +#: ../../source/production/observability/metrics.rst:216 +#: ../../source/production/observability/metrics.rst:233 +#: ../../source/production/observability/metrics.rst:236 +#: ../../source/production/observability/metrics.rst:239 +#: ../../source/production/observability/metrics.rst:265 +#: ../../source/production/observability/metrics.rst:282 +#: ../../source/production/observability/metrics.rst:285 +msgid "Counter" +msgstr "" + +#: ../../source/production/observability/metrics.rst:25 +msgid "Total number of retrieve requests sent to LMCache." +msgstr "" + +#: ../../source/production/observability/metrics.rst:26 +msgid "``lmcache:num_store_requests``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:28 +msgid "Total number of store requests sent to LMCache." +msgstr "" + +#: ../../source/production/observability/metrics.rst:29 +msgid "``lmcache:num_lookup_requests``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:31 +msgid "Total number of lookup requests sent to LMCache." +msgstr "" + +#: ../../source/production/observability/metrics.rst:34 +#: ../../source/production/observability/metrics.rst:36 +msgid "Token Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:43 +msgid "``lmcache:num_requested_tokens``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:45 +msgid "Total number of tokens requested for retrieval." +msgstr "" + +#: ../../source/production/observability/metrics.rst:46 +msgid "``lmcache:num_hit_tokens``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:48 +msgid "Total number of tokens hit in LMCache during retrieval." +msgstr "" + +#: ../../source/production/observability/metrics.rst:49 +msgid "``lmcache:num_stored_tokens``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:51 +msgid "Total number of tokens stored in LMCache." +msgstr "" + +#: ../../source/production/observability/metrics.rst:52 +msgid "``lmcache:num_lookup_tokens``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:54 +msgid "Total number of tokens requested in lookup operations." +msgstr "" + +#: ../../source/production/observability/metrics.rst:55 +msgid "``lmcache:num_lookup_hits``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:57 +msgid "Total number of tokens hit in lookup operations." +msgstr "" + +#: ../../source/production/observability/metrics.rst:58 +msgid "``lmcache:num_vllm_hit_tokens``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:60 +msgid "Number of hit tokens in vLLM." +msgstr "" + +#: ../../source/production/observability/metrics.rst:61 +msgid "``lmcache:num_prompt_tokens``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:63 +msgid "Number of prompt tokens in LMCache." +msgstr "" + +#: ../../source/production/observability/metrics.rst:66 +#: ../../source/production/observability/metrics.rst:68 +msgid "Hit Rate Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:75 +msgid "``lmcache:retrieve_hit_rate``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:76 +#: ../../source/production/observability/metrics.rst:79 +#: ../../source/production/observability/metrics.rst:166 +#: ../../source/production/observability/metrics.rst:169 +#: ../../source/production/observability/metrics.rst:172 +#: ../../source/production/observability/metrics.rst:210 +#: ../../source/production/observability/metrics.rst:219 +#: ../../source/production/observability/metrics.rst:242 +#: ../../source/production/observability/metrics.rst:245 +#: ../../source/production/observability/metrics.rst:259 +#: ../../source/production/observability/metrics.rst:262 +#: ../../source/production/observability/metrics.rst:268 +#: ../../source/production/observability/metrics.rst:305 +#: ../../source/production/observability/metrics.rst:308 +#: ../../source/production/observability/metrics.rst:311 +#: ../../source/production/observability/metrics.rst:314 +#: ../../source/production/observability/metrics.rst:317 +#: ../../source/production/observability/metrics.rst:320 +#: ../../source/production/observability/metrics.rst:323 +#: ../../source/production/observability/metrics.rst:337 +#: ../../source/production/observability/metrics.rst:340 +#: ../../source/production/observability/metrics.rst:343 +#: ../../source/production/observability/metrics.rst:346 +#: ../../source/production/observability/metrics.rst:349 +#: ../../source/production/observability/metrics.rst:352 +#: ../../source/production/observability/metrics.rst:355 +#: ../../source/production/observability/metrics.rst:358 +#: ../../source/production/observability/metrics.rst:361 +#: ../../source/production/observability/metrics.rst:375 +#: ../../source/production/observability/metrics.rst:378 +#: ../../source/production/observability/metrics.rst:381 +#: ../../source/production/observability/metrics.rst:384 +#: ../../source/production/observability/metrics.rst:387 +#: ../../source/production/observability/metrics.rst:390 +#: ../../source/production/observability/metrics.rst:393 +msgid "Gauge" +msgstr "" + +#: ../../source/production/observability/metrics.rst:77 +msgid "The hit rate for retrieve requests since last log." +msgstr "" + +#: ../../source/production/observability/metrics.rst:78 +msgid "``lmcache:lookup_hit_rate``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:80 +msgid "The hit rate for lookup requests since last log." +msgstr "" + +#: ../../source/production/observability/metrics.rst:81 +msgid "``lmcache:request_cache_hit_rate``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:82 +#: ../../source/production/observability/metrics.rst:99 +#: ../../source/production/observability/metrics.rst:102 +#: ../../source/production/observability/metrics.rst:105 +#: ../../source/production/observability/metrics.rst:108 +#: ../../source/production/observability/metrics.rst:111 +#: ../../source/production/observability/metrics.rst:131 +#: ../../source/production/observability/metrics.rst:134 +#: ../../source/production/observability/metrics.rst:137 +#: ../../source/production/observability/metrics.rst:140 +#: ../../source/production/observability/metrics.rst:143 +#: ../../source/production/observability/metrics.rst:146 +#: ../../source/production/observability/metrics.rst:149 +#: ../../source/production/observability/metrics.rst:152 +#: ../../source/production/observability/metrics.rst:175 +#: ../../source/production/observability/metrics.rst:201 +#: ../../source/production/observability/metrics.rst:204 +#: ../../source/production/observability/metrics.rst:207 +#: ../../source/production/observability/metrics.rst:288 +#: ../../source/production/observability/metrics.rst:291 +msgid "Histogram" +msgstr "" + +#: ../../source/production/observability/metrics.rst:83 +msgid "Distribution of hit rates per request." +msgstr "" + +#: ../../source/production/observability/metrics.rst:84 +msgid "``lmcache:lookup_0_hit_requests``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:86 +msgid "Total number of lookup requests with zero hits." +msgstr "" + +#: ../../source/production/observability/metrics.rst:89 +#: ../../source/production/observability/metrics.rst:91 +msgid "Performance & Latency Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:98 +msgid "``lmcache:time_to_retrieve``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:100 +msgid "Time taken to retrieve from the cache (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:101 +msgid "``lmcache:time_to_store``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:103 +msgid "Time taken to store to the cache (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:104 +msgid "``lmcache:time_to_lookup``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:106 +msgid "Time taken to perform a lookup in the cache (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:107 +msgid "``lmcache:retrieve_speed``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:109 +msgid "Retrieval speed (tokens per second)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:110 +msgid "``lmcache:store_speed``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:112 +msgid "Storage speed (tokens per second)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:113 +msgid "``lmcache:num_slow_retrieval_by_time``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:115 +msgid "Total number of slow retrievals exceeding the time threshold." +msgstr "" + +#: ../../source/production/observability/metrics.rst:116 +msgid "``lmcache:num_slow_retrieval_by_speed``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:118 +msgid "Total number of slow retrievals below the speed threshold." +msgstr "" + +#: ../../source/production/observability/metrics.rst:121 +msgid "Detailed Profiling Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:123 +msgid "Profiling Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:130 +msgid "``lmcache:retrieve_process_tokens_time``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:132 +msgid "Time to process tokens in retrieve (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:133 +msgid "``lmcache:retrieve_broadcast_time``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:135 +msgid "Time to broadcast memory objects in retrieve (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:136 +msgid "``lmcache:retrieve_to_gpu_time``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:138 +msgid "Time to move data to GPU in retrieve (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:139 +msgid "``lmcache:store_process_tokens_time``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:141 +msgid "Time to process tokens in store (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:142 +msgid "``lmcache:store_from_gpu_time``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:144 +msgid "Time to move data from GPU in store (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:145 +msgid "``lmcache:store_put_time``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:147 +msgid "Time to put data to storage in store (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:148 +msgid "``lmcache:remote_backend_batched_get_blocking_time``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:150 +msgid "Time spent waiting for data from remote backend (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:151 +msgid "``lmcache:instrumented_connector_batched_get_time``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:153 +msgid "Time spent in the connector layer (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:156 +msgid "Cache Usage & Lifecycle Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:158 +msgid "Cache Usage Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:165 +msgid "``lmcache:local_cache_usage``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:167 +msgid "Local cache usage in bytes." +msgstr "" + +#: ../../source/production/observability/metrics.rst:168 +msgid "``lmcache:remote_cache_usage``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:170 +msgid "Remote cache usage in bytes." +msgstr "" + +#: ../../source/production/observability/metrics.rst:171 +msgid "``lmcache:local_storage_usage``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:173 +msgid "Local storage usage in bytes." +msgstr "" + +#: ../../source/production/observability/metrics.rst:174 +msgid "``lmcache:request_cache_lifespan``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:176 +msgid "Distribution of request cache lifespan in minutes." +msgstr "" + +#: ../../source/production/observability/metrics.rst:179 +msgid "Remote Backend & Network Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:181 +msgid "Remote Backend Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:188 +msgid "``lmcache:num_remote_read_requests``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:190 +msgid "Total number of read requests to remote backends." +msgstr "" + +#: ../../source/production/observability/metrics.rst:191 +msgid "``lmcache:num_remote_read_bytes``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:193 +msgid "Total number of bytes read from remote backends." +msgstr "" + +#: ../../source/production/observability/metrics.rst:194 +msgid "``lmcache:num_remote_write_requests``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:196 +msgid "Total number of write requests to remote backends." +msgstr "" + +#: ../../source/production/observability/metrics.rst:197 +msgid "``lmcache:num_remote_write_bytes``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:199 +msgid "Total number of bytes written to remote backends." +msgstr "" + +#: ../../source/production/observability/metrics.rst:200 +msgid "``lmcache:remote_time_to_get``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:202 +msgid "Time taken to get data from remote backends (ms)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:203 +msgid "``lmcache:remote_time_to_put``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:205 +msgid "Time taken to put data to remote backends (ms)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:206 +msgid "``lmcache:remote_time_to_get_sync``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:208 +msgid "Time taken to get data from remote backends synchronously (ms)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:209 +msgid "``lmcache:remote_ping_latency``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:211 +msgid "Latest ping latency to remote backends (ms)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:212 +msgid "``lmcache:remote_ping_errors``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:214 +msgid "Total number of ping errors to remote backends." +msgstr "" + +#: ../../source/production/observability/metrics.rst:215 +msgid "``lmcache:remote_ping_successes``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:217 +msgid "Total number of successful pings to remote backends." +msgstr "" + +#: ../../source/production/observability/metrics.rst:218 +msgid "``lmcache:remote_ping_error_code``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:220 +msgid "Latest ping error code to remote backends." +msgstr "" + +#: ../../source/production/observability/metrics.rst:223 +#: ../../source/production/observability/metrics.rst:225 +msgid "Local CPU Backend Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:232 +msgid "``lmcache:local_cpu_evict_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:234 +msgid "Total number of evictions in local CPU backend." +msgstr "" + +#: ../../source/production/observability/metrics.rst:235 +msgid "``lmcache:local_cpu_evict_keys_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:237 +msgid "Total number of evicted keys in local CPU backend." +msgstr "" + +#: ../../source/production/observability/metrics.rst:238 +msgid "``lmcache:local_cpu_evict_failed_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:240 +msgid "Total number of failed evictions in local CPU backend." +msgstr "" + +#: ../../source/production/observability/metrics.rst:241 +msgid "``lmcache:local_cpu_hot_cache_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:243 +msgid "Current number of items in the hot cache." +msgstr "" + +#: ../../source/production/observability/metrics.rst:244 +msgid "``lmcache:local_cpu_keys_in_request_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:246 +msgid "Current number of keys being processed in requests." +msgstr "" + +#: ../../source/production/observability/metrics.rst:249 +#: ../../source/production/observability/metrics.rst:251 +msgid "Memory Management Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:258 +msgid "``lmcache:active_memory_objs_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:260 +msgid "The number of currently active memory objects." +msgstr "" + +#: ../../source/production/observability/metrics.rst:261 +msgid "``lmcache:pinned_memory_objs_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:263 +msgid "The number of currently pinned memory objects." +msgstr "" + +#: ../../source/production/observability/metrics.rst:264 +msgid "``lmcache:forced_unpin_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:266 +msgid "Total number of forced unpins due to timeout." +msgstr "" + +#: ../../source/production/observability/metrics.rst:267 +msgid "``lmcache:pin_monitor_pinned_objects_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:269 +msgid "The number of pinned objects tracked by the PinMonitor." +msgstr "" + +#: ../../source/production/observability/metrics.rst:272 +#: ../../source/production/observability/metrics.rst:274 +msgid "P2P Transfer Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:281 +msgid "``lmcache:num_p2p_requests``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:283 +msgid "Total number of P2P transfer requests." +msgstr "" + +#: ../../source/production/observability/metrics.rst:284 +msgid "``lmcache:num_p2p_transferred_tokens``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:286 +msgid "Total number of tokens transferred via P2P." +msgstr "" + +#: ../../source/production/observability/metrics.rst:287 +msgid "``lmcache:p2p_time_to_transfer``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:289 +msgid "Time taken for P2P transfers (seconds)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:290 +msgid "``lmcache:p2p_transfer_speed``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:292 +msgid "P2P transfer speed (tokens per second)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:295 +msgid "Health & Internal System Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:297 +msgid "Health & Internal Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:304 +msgid "``lmcache:lmcache_is_healthy``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:306 +msgid "Overall health status of LMCache (1 = healthy, 0 = unhealthy)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:307 +msgid "``lmcache:interval_get_blocking_failed_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:309 +msgid "Number of failed blocking get operations in the current interval." +msgstr "" + +#: ../../source/production/observability/metrics.rst:310 +msgid "``lmcache:kv_msg_queue_size``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:312 +msgid "Size of the KV message queue in the BatchedMessageSender." +msgstr "" + +#: ../../source/production/observability/metrics.rst:313 +msgid "``lmcache:remote_put_task_num``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:315 +msgid "Number of pending remote put tasks." +msgstr "" + +#: ../../source/production/observability/metrics.rst:316 +msgid "``lmcache:storage_events_ongoing_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:318 +msgid "Number of storage events currently in progress." +msgstr "" + +#: ../../source/production/observability/metrics.rst:319 +msgid "``lmcache:storage_events_done_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:321 +msgid "Number of storage events completed successfully." +msgstr "" + +#: ../../source/production/observability/metrics.rst:322 +msgid "``lmcache:storage_events_not_found_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:324 +msgid "Number of storage events where the requested data was not found." +msgstr "" + +#: ../../source/production/observability/metrics.rst:327 +#: ../../source/production/observability/metrics.rst:329 +msgid "Chunk Statistics Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:336 +msgid "``lmcache:chunk_statistics_enabled``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:338 +msgid "" +"Whether chunk statistics collection is enabled (1 = enabled, 0 = " +"disabled)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:339 +msgid "``lmcache:chunk_statistics_total_requests``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:341 +msgid "Total number of requests processed by chunk statistics." +msgstr "" + +#: ../../source/production/observability/metrics.rst:342 +msgid "``lmcache:chunk_statistics_total_chunks``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:344 +msgid "Total number of chunks processed." +msgstr "" + +#: ../../source/production/observability/metrics.rst:345 +msgid "``lmcache:chunk_statistics_unique_chunks``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:347 +msgid "Estimated number of unique chunks encountered." +msgstr "" + +#: ../../source/production/observability/metrics.rst:348 +msgid "``lmcache:chunk_statistics_reuse_rate``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:350 +msgid "Chunk reuse rate (0.0 to 1.0)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:351 +msgid "``lmcache:chunk_statistics_bloom_filter_size_mb``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:353 +msgid "Memory usage of the Bloom filter in megabytes." +msgstr "" + +#: ../../source/production/observability/metrics.rst:354 +msgid "``lmcache:chunk_statistics_bloom_filter_fill_rate``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:356 +msgid "Fill rate of the Bloom filter (0.0 to 1.0)." +msgstr "" + +#: ../../source/production/observability/metrics.rst:357 +msgid "``lmcache:chunk_statistics_file_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:359 +msgid "Number of files created when using the ``file_hash`` strategy." +msgstr "" + +#: ../../source/production/observability/metrics.rst:360 +msgid "``lmcache:chunk_statistics_current_file_size``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:362 +msgid "Current size of the active statistics file in bytes." +msgstr "" + +#: ../../source/production/observability/metrics.rst:365 +#: ../../source/production/observability/metrics.rst:367 +msgid "Connector Metrics" +msgstr "" + +#: ../../source/production/observability/metrics.rst:374 +msgid "``lmcache:scheduler_unfinished_requests_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:376 +msgid "Current count of unfinished requests in the scheduler." +msgstr "" + +#: ../../source/production/observability/metrics.rst:377 +msgid "``lmcache:connector_load_specs_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:379 +msgid "Number of load specifications currently in the connector." +msgstr "" + +#: ../../source/production/observability/metrics.rst:380 +msgid "``lmcache:connector_request_trackers_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:382 +msgid "Number of active request trackers in the connector." +msgstr "" + +#: ../../source/production/observability/metrics.rst:383 +msgid "``lmcache:connector_kv_caches_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:385 +msgid "Number of KV caches currently managed by the connector." +msgstr "" + +#: ../../source/production/observability/metrics.rst:386 +msgid "``lmcache:connector_layerwise_retrievers_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:388 +msgid "Number of layer-wise retrievers active in the connector." +msgstr "" + +#: ../../source/production/observability/metrics.rst:389 +msgid "``lmcache:connector_invalid_block_ids_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:391 +msgid "Number of invalid block IDs encountered by the connector." +msgstr "" + +#: ../../source/production/observability/metrics.rst:392 +msgid "``lmcache:connector_requests_priority_count``" +msgstr "" + +#: ../../source/production/observability/metrics.rst:394 +msgid "Number of requests prioritized by the connector." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/periodic_thread_api.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/periodic_thread_api.po new file mode 100644 index 00000000000..6032dcf7a8e --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/periodic_thread_api.po @@ -0,0 +1,232 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/observability/periodic_thread_api.rst:4 +msgid "Periodic Thread Monitoring API" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:6 +msgid "" +"This document describes the Periodic Thread Monitoring API feature " +"introduced in LMCache." +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:10 +msgid "Overview" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:12 +msgid "" +"The Periodic Thread Monitoring API provides HTTP endpoints to monitor and" +" inspect the status of all periodic background threads running in the " +"LMCache system. This is useful for debugging, health checking, and " +"operational monitoring." +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:18 +msgid "API Endpoints" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:20 +msgid "Three API endpoints are available under the internal API server:" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:23 +msgid "GET /periodic-threads" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:25 +msgid "Returns information about all registered periodic threads." +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:27 +msgid "**Query Parameters:**" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:33 +msgid "Parameter" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:34 +msgid "Default" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:35 +#: ../../source/production/observability/periodic_thread_api.rst:121 +msgid "Description" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:36 +msgid "``level``" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:37 +msgid "(none)" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:38 +msgid "Filter by thread level (``critical``, ``high``, ``medium``, ``low``)" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:39 +msgid "``running_only``" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:40 +#: ../../source/production/observability/periodic_thread_api.rst:43 +msgid "``false``" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:41 +msgid "Only show running threads" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:42 +msgid "``active_only``" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:44 +msgid "Only show active threads" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:46 +#: ../../source/production/observability/periodic_thread_api.rst:79 +#: ../../source/production/observability/periodic_thread_api.rst:101 +msgid "**Response Example:**" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:75 +#, python-brace-format +msgid "GET /periodic-threads/{thread_name}" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:77 +msgid "Returns detailed information about a specific periodic thread." +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:96 +msgid "GET /periodic-threads-health" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:98 +msgid "" +"Quick health check for periodic threads. Returns whether all critical and" +" high-level threads are active." +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:112 +msgid "Thread Levels" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:114 +msgid "Periodic threads are categorized by importance level:" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:120 +msgid "Level" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:122 +msgid "``critical``" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:123 +msgid "Essential for system operation" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:124 +msgid "``high``" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:125 +msgid "Important for performance" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:126 +msgid "``medium``" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:127 +msgid "Standard background tasks" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:128 +msgid "``low``" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:129 +msgid "Optional/auxiliary tasks" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:131 +msgid "" +"The health check endpoint specifically monitors ``critical`` and ``high``" +" level threads to determine system health." +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:135 +msgid "IrrecoverableException Handling" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:137 +msgid "" +"Periodic threads now properly handle ``IrrecoverableException``. When " +"such an exception is raised during thread execution:" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:140 +msgid "The exception is logged with full traceback" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:141 +msgid "The thread run is marked as failed" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:142 +msgid "The thread **stops its execution loop** instead of continuing" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:144 +msgid "" +"This prevents threads from endlessly retrying operations that cannot " +"succeed." +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:147 +msgid "Usage Examples" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:150 +msgid "Check Overall Thread Health" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:157 +msgid "List All Running Threads" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:164 +msgid "Get Critical Threads Only" +msgstr "" + +#: ../../source/production/observability/periodic_thread_api.rst:171 +msgid "Check Specific Thread Status" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/vllm_endpoint.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/vllm_endpoint.po new file mode 100644 index 00000000000..745f29f5f3c --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/vllm_endpoint.po @@ -0,0 +1,458 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/observability/vllm_endpoint.rst:4 +msgid "Metrics by vLLM API" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:6 +msgid "" +"LMCache provides detailed metrics via a Prometheus endpoint, allowing for" +" in-depth monitoring of cache performance and behavior. This section " +"outlines how to enable and configure observability from embedded vLLM " +"``/metrics`` API endpoint." +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:11 +msgid "Quick Start Guide" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:14 +msgid "1) On vLLM/LMCache side" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:16 +msgid "" +"In v1, vLLM and LMCache run in separate processes, so you have to use " +"multi‑process Prometheus." +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:18 +msgid "" +"The ``PROMETHEUS_MULTIPROC_DIR`` environment variable must be the same in" +" both processes, as a IPC directory." +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:26 +msgid "" +"Once the HTTP server is running, you can access the LMCache metrics at " +"the ``/metrics`` endpoint." +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:35 +msgid "" +"And you will also find some ``.db`` files in the " +"``$PROMETHEUS_MULTIPROC_DIR`` directory." +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:39 +msgid "2) Prometheus Configuration" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:41 +msgid "" +"To scrape the LMCache metrics with a Prometheus server, add the following" +" job to your ``prometheus.yml`` configuration, or equivalent " +"configuration to scrape the metrics endpoint:" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:53 +msgid "Available Metrics" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:55 +msgid "" +"LMCache exposes a variety of metrics to monitor its performance. The " +"following table lists all available metrics organized by category:" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:57 +msgid "LMCache Metrics" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:61 +msgid "Metric Name" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:62 +msgid "Type" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:63 +msgid "Description" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:64 +msgid "**Core Request Metrics**" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:67 +msgid "``lmcache:num_retrieve_requests``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:68 +#: ../../source/production/observability/vllm_endpoint.rst:71 +#: ../../source/production/observability/vllm_endpoint.rst:74 +#: ../../source/production/observability/vllm_endpoint.rst:77 +#: ../../source/production/observability/vllm_endpoint.rst:80 +#: ../../source/production/observability/vllm_endpoint.rst:83 +#: ../../source/production/observability/vllm_endpoint.rst:86 +#: ../../source/production/observability/vllm_endpoint.rst:89 +#: ../../source/production/observability/vllm_endpoint.rst:131 +#: ../../source/production/observability/vllm_endpoint.rst:134 +#: ../../source/production/observability/vllm_endpoint.rst:137 +#: ../../source/production/observability/vllm_endpoint.rst:140 +#: ../../source/production/observability/vllm_endpoint.rst:158 +#: ../../source/production/observability/vllm_endpoint.rst:161 +#: ../../source/production/observability/vllm_endpoint.rst:170 +#: ../../source/production/observability/vllm_endpoint.rst:173 +#: ../../source/production/observability/vllm_endpoint.rst:176 +msgid "Counter" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:69 +msgid "Total number of retrieve requests" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:70 +msgid "``lmcache:num_store_requests``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:72 +msgid "Total number of store requests" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:73 +msgid "``lmcache:num_lookup_requests``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:75 +msgid "Total number of lookup requests" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:76 +msgid "``lmcache:num_requested_tokens``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:78 +msgid "Total number of tokens requested for retrieval" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:79 +msgid "``lmcache:num_hit_tokens``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:81 +msgid "Total number of cache hit tokens from retrieval" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:82 +msgid "``lmcache:num_lookup_tokens``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:84 +msgid "Total number of tokens requested in lookup operations" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:85 +msgid "``lmcache:num_lookup_hits``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:87 +msgid "Total number of tokens hit in lookup operations" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:88 +msgid "``lmcache:num_vllm_hit_tokens``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:90 +msgid "Number of hit tokens in vLLM" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:91 +msgid "**Hit Rate Metrics**" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:94 +msgid "``lmcache:retrieve_hit_rate``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:95 +#: ../../source/production/observability/vllm_endpoint.rst:98 +#: ../../source/production/observability/vllm_endpoint.rst:104 +#: ../../source/production/observability/vllm_endpoint.rst:107 +#: ../../source/production/observability/vllm_endpoint.rst:110 +#: ../../source/production/observability/vllm_endpoint.rst:155 +#: ../../source/production/observability/vllm_endpoint.rst:164 +#: ../../source/production/observability/vllm_endpoint.rst:179 +#: ../../source/production/observability/vllm_endpoint.rst:182 +#: ../../source/production/observability/vllm_endpoint.rst:188 +#: ../../source/production/observability/vllm_endpoint.rst:191 +msgid "Gauge" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:96 +msgid "The hit rate for retrieve requests" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:97 +msgid "``lmcache:lookup_hit_rate``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:99 +msgid "The hit rate for lookup requests" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:100 +msgid "**Cache Usage Metrics**" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:103 +msgid "``lmcache:local_cache_usage``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:105 +msgid "Local cache usage in bytes" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:106 +msgid "``lmcache:remote_cache_usage``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:108 +msgid "Remote cache usage in bytes" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:109 +msgid "``lmcache:local_storage_usage``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:111 +msgid "Local storage usage in bytes" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:112 +msgid "**Performance Metrics**" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:115 +msgid "``lmcache:time_to_retrieve``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:116 +#: ../../source/production/observability/vllm_endpoint.rst:119 +#: ../../source/production/observability/vllm_endpoint.rst:122 +#: ../../source/production/observability/vllm_endpoint.rst:125 +#: ../../source/production/observability/vllm_endpoint.rst:143 +#: ../../source/production/observability/vllm_endpoint.rst:146 +#: ../../source/production/observability/vllm_endpoint.rst:149 +msgid "Histogram" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:117 +msgid "Time taken to retrieve from the cache (seconds)" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:118 +msgid "``lmcache:time_to_store``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:120 +msgid "Time taken to store to the cache (seconds)" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:121 +msgid "``lmcache:retrieve_speed``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:123 +msgid "Retrieval speed (tokens per second)" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:124 +msgid "``lmcache:store_speed``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:126 +msgid "Storage speed (tokens per second)" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:127 +msgid "**Remote Backend Metrics**" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:130 +msgid "``lmcache:num_remote_read_requests``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:132 +msgid "Total number of read requests to remote backends" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:133 +msgid "``lmcache:num_remote_read_bytes``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:135 +msgid "Total number of bytes read from remote backends" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:136 +msgid "``lmcache:num_remote_write_requests``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:138 +msgid "Total number of write requests to remote backends" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:139 +msgid "``lmcache:num_remote_write_bytes``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:141 +msgid "Total number of bytes written to remote backends" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:142 +msgid "``lmcache:remote_time_to_get``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:144 +msgid "Time taken to get data from remote backends (milliseconds)" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:145 +msgid "``lmcache:remote_time_to_put``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:147 +msgid "Time taken to put data to remote backends (milliseconds)" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:148 +msgid "``lmcache:remote_time_to_get_sync``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:150 +msgid "Time taken to get data from remote backends synchronously (milliseconds)" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:151 +msgid "**Network Monitoring Metrics**" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:154 +msgid "``lmcache:remote_ping_latency``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:156 +msgid "Latest ping latency to remote backends (milliseconds)" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:157 +msgid "``lmcache:remote_ping_errors``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:159 +msgid "Number of ping errors to remote backends" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:160 +msgid "``lmcache:remote_ping_successes``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:162 +msgid "Number of ping successes to remote backends" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:163 +msgid "``lmcache:remote_ping_error_code``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:165 +msgid "Latest ping error code to remote backends" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:166 +msgid "**Local CPU Backend Metrics**" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:169 +msgid "``lmcache:local_cpu_evict_count``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:171 +msgid "Total number of evictions in local CPU backend" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:172 +msgid "``lmcache:local_cpu_evict_keys_count``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:174 +msgid "Total number of evicted keys in local CPU backend" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:175 +msgid "``lmcache:local_cpu_evict_failed_count``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:177 +msgid "Total number of failed evictions in local CPU backend" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:178 +msgid "``lmcache:local_cpu_hot_cache_count``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:180 +msgid "The size of the hot cache" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:181 +msgid "``lmcache:local_cpu_keys_in_request_count``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:183 +msgid "The size of the keys in request" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:184 +msgid "**Memory Management Metrics**" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:187 +msgid "``lmcache:active_memory_objs_count``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:189 +msgid "The number of active memory objects" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:190 +msgid "``lmcache:pinned_memory_objs_count``" +msgstr "" + +#: ../../source/production/observability/vllm_endpoint.rst:192 +msgid "The number of pinned memory objects" +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/performance_tuning.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/performance_tuning.po new file mode 100644 index 00000000000..d17648e5ada --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/performance_tuning.po @@ -0,0 +1,129 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/production/performance_tuning.rst:4 +msgid "Performance Tuning" +msgstr "" + +#: ../../source/production/performance_tuning.rst:6 +msgid "" +"This guide covers key LMCache configuration options that can help you " +"optimize performance in production deployments." +msgstr "" + +#: ../../source/production/performance_tuning.rst:10 +msgid "Minimum Retrieve Tokens" +msgstr "" + +#: ../../source/production/performance_tuning.rst:12 +msgid "" +"When LMCache finds a partial KV cache hit, it loads the cached tokens " +"into GPU memory to avoid recomputation. However, if only a small number " +"of tokens are hit, the overhead of loading them from the cache may " +"outweigh the benefit of skipping recomputation." +msgstr "" + +#: ../../source/production/performance_tuning.rst:17 +msgid "" +"The ``min_retrieve_tokens`` setting lets you set a threshold: if the " +"number of tokens that need to be loaded is below this value, LMCache will" +" skip the retrieve and let the inference engine recompute them instead." +msgstr "" + +#: ../../source/production/performance_tuning.rst:24 +msgid "" +"Even when retrieve is skipped, LMCache still records the hit tokens " +"internally so that it does **not** re-store chunks that already exist in " +"the cache." +msgstr "" + +#: ../../source/production/performance_tuning.rst:29 +msgid "When to Use" +msgstr "" + +#: ../../source/production/performance_tuning.rst:31 +msgid "Consider setting ``min_retrieve_tokens`` when:" +msgstr "" + +#: ../../source/production/performance_tuning.rst:33 +msgid "" +"For the backend you are using, the transfer latency is noticeable for " +"small payloads." +msgstr "" + +#: ../../source/production/performance_tuning.rst:35 +msgid "" +"Your workload has many requests with **low cache hit ratios**, where " +"recomputation is faster than cache loading." +msgstr "" + +#: ../../source/production/performance_tuning.rst:37 +msgid "You want to reduce unnecessary I/O for marginal cache hits." +msgstr "" + +#: ../../source/production/performance_tuning.rst:39 +msgid "You can increase or decrease based on your latency observations." +msgstr "" + +#: ../../source/production/performance_tuning.rst:42 +msgid "Configuration" +msgstr "" + +#: ../../source/production/performance_tuning.rst:44 +msgid "**YAML configuration file:**" +msgstr "" + +#: ../../source/production/performance_tuning.rst:52 +msgid "**Environment variable:**" +msgstr "" + +#: ../../source/production/performance_tuning.rst:59 +msgid "Working Example" +msgstr "" + +#: ../../source/production/performance_tuning.rst:61 +msgid "Create an LMCache configuration file ``lmcache_config.yaml``:" +msgstr "" + +#: ../../source/production/performance_tuning.rst:70 +msgid "Start vLLM with LMCache:" +msgstr "" + +#: ../../source/production/performance_tuning.rst:81 +msgid "Send a request to populate the cache:" +msgstr "" + +#: ../../source/production/performance_tuning.rst:94 +msgid "Send a similar request that partially reuses the cached prefix:" +msgstr "" + +#: ../../source/production/performance_tuning.rst:107 +msgid "" +"Check the server logs. If the number of loadable hit tokens is below " +"1024, you will see a log message like:" +msgstr "" + +#: ../../source/production/performance_tuning.rst:115 +msgid "" +"This confirms that the small cache hit was skipped in favor of " +"recomputation, avoiding unnecessary transfer overhead." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/devstral.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/devstral.po new file mode 100644 index 00000000000..5bcaa29aa84 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/devstral.po @@ -0,0 +1,124 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/recipes/devstral.rst:4 +msgid "MistralForCausalLM" +msgstr "" + +#: ../../source/recipes/devstral.rst:7 +msgid "Validated models" +msgstr "" + +#: ../../source/recipes/devstral.rst:9 +msgid "" +"`mistralai/Devstral-2-123B-Instruct-2512 " +"`_" +msgstr "" + +#: ../../source/recipes/devstral.rst +msgid "vLLM" +msgstr "" + +#: ../../source/recipes/devstral.rst:16 +msgid "" +"**Engine documentation:** `Mistral / Devstral in vLLM supported models " +"`_ (architecture ``MistralForCausalLM``)." +msgstr "" + +#: ../../source/recipes/devstral.rst:21 +msgid "**Status:** Validated with LMCache." +msgstr "" + +#: ../../source/recipes/devstral.rst:23 +msgid "Start the LMCache MP server:" +msgstr "" + +#: ../../source/recipes/devstral.rst:31 +msgid "Start vLLM with the LMCache MP connector:" +msgstr "" + +#: ../../source/recipes/devstral.rst:44 +msgid "" +"Adjust ``--tensor-parallel-size`` to match your hardware. For the generic" +" LMCache + vLLM wiring (ports, remote hosts, in-process mode), see " +":doc:`../mp/quickstart`." +msgstr "" + +#: ../../source/recipes/devstral.rst:48 +msgid "" +"If there are any issues with vLLM setup, please refer to the `vLLM " +"Recipes `_ " +"for more details." +msgstr "" + +#: ../../source/recipes/devstral.rst +msgid "SGLang" +msgstr "" + +#: ../../source/recipes/devstral.rst:54 +msgid "**Status:** Not validated with LMCache." +msgstr "" + +#: ../../source/recipes/devstral.rst +msgid "TRT-LLM" +msgstr "" + +#: ../../source/recipes/devstral.rst:58 +msgid "**Status:** Not supported. LMCache TRT-LLM integration is in progress." +msgstr "" + +#: ../../source/recipes/devstral.rst:61 +msgid "CacheBlend support" +msgstr "" + +#: ../../source/recipes/devstral.rst:64 +msgid "Compression support" +msgstr "" + +#: ../../source/recipes/devstral.rst:70 +msgid "Method" +msgstr "" + +#: ../../source/recipes/devstral.rst:71 +msgid "Status" +msgstr "" + +#: ../../source/recipes/devstral.rst:72 +msgid "Notes" +msgstr "" + +#: ../../source/recipes/devstral.rst:73 +msgid ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" +msgstr "" + +#: ../../source/recipes/devstral.rst:74 +msgid "Not validated" +msgstr "" + +#: ../../source/recipes/devstral.rst:78 +msgid "Caveats" +msgstr "" + +#: ../../source/recipes/devstral.rst:80 +msgid "None known." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gemma4.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gemma4.po new file mode 100644 index 00000000000..6a78548a101 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gemma4.po @@ -0,0 +1,122 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/recipes/gemma4.rst:4 +msgid "Gemma4ForConditionalGeneration" +msgstr "" + +#: ../../source/recipes/gemma4.rst:7 +msgid "Validated models" +msgstr "" + +#: ../../source/recipes/gemma4.rst:9 +msgid "`google/gemma-4-31B-it `_" +msgstr "" + +#: ../../source/recipes/gemma4.rst +msgid "vLLM" +msgstr "" + +#: ../../source/recipes/gemma4.rst:16 +msgid "" +"**Engine documentation:** `Gemma 4 in vLLM supported models " +"`_ (architecture ``Gemma4ForConditionalGeneration``)." +msgstr "" + +#: ../../source/recipes/gemma4.rst:21 +msgid "**Status:** Validated with LMCache." +msgstr "" + +#: ../../source/recipes/gemma4.rst:23 +msgid "Start the LMCache MP server:" +msgstr "" + +#: ../../source/recipes/gemma4.rst:31 +msgid "Start vLLM with the LMCache MP connector:" +msgstr "" + +#: ../../source/recipes/gemma4.rst:42 +msgid "" +"Adjust ``--tensor-parallel-size`` to match your hardware. For the generic" +" LMCache + vLLM wiring (ports, remote hosts, in-process mode), see " +":doc:`../mp/quickstart`." +msgstr "" + +#: ../../source/recipes/gemma4.rst:46 +msgid "" +"If there are any issues with vLLM setup, please refer to the `vLLM " +"Recipes `_ " +"for more details." +msgstr "" + +#: ../../source/recipes/gemma4.rst +msgid "SGLang" +msgstr "" + +#: ../../source/recipes/gemma4.rst:52 +msgid "**Status:** Not validated with LMCache." +msgstr "" + +#: ../../source/recipes/gemma4.rst +msgid "TRT-LLM" +msgstr "" + +#: ../../source/recipes/gemma4.rst:56 +msgid "**Status:** Not supported. LMCache TRT-LLM integration is in progress." +msgstr "" + +#: ../../source/recipes/gemma4.rst:59 +msgid "CacheBlend support" +msgstr "" + +#: ../../source/recipes/gemma4.rst:62 +msgid "Compression support" +msgstr "" + +#: ../../source/recipes/gemma4.rst:68 +msgid "Method" +msgstr "" + +#: ../../source/recipes/gemma4.rst:69 +msgid "Status" +msgstr "" + +#: ../../source/recipes/gemma4.rst:70 +msgid "Notes" +msgstr "" + +#: ../../source/recipes/gemma4.rst:71 +msgid ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" +msgstr "" + +#: ../../source/recipes/gemma4.rst:72 +msgid "Not validated" +msgstr "" + +#: ../../source/recipes/gemma4.rst:76 +msgid "Caveats" +msgstr "" + +#: ../../source/recipes/gemma4.rst:78 +msgid "None known." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gpt_oss.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gpt_oss.po new file mode 100644 index 00000000000..7a9a0087a84 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gpt_oss.po @@ -0,0 +1,130 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/recipes/gpt_oss.rst:4 +msgid "GptOssForCausalLM" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:7 +msgid "Validated models" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:9 +msgid "`openai/gpt-oss-120b `_" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:10 +msgid "`openai/gpt-oss-20b `_" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst +msgid "vLLM" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:17 +msgid "" +"**Engine documentation:** `GPT-OSS in vLLM supported models " +"`_ (architecture ``GptOssForCausalLM``)." +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:22 +msgid "**Status:** Validated with LMCache." +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:24 +msgid "Start the LMCache MP server:" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:32 +msgid "**gpt-oss-120b** (2 GPUs):" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:45 +msgid "**gpt-oss-20b** (1 GPU):" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:57 +msgid "" +"Adjust ``--tensor-parallel-size`` to match your hardware. For the generic" +" LMCache + vLLM wiring (ports, remote hosts, in-process mode), see " +":doc:`../mp/quickstart`." +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:61 +msgid "" +"If there are any issues with vLLM setup, please refer to the `vLLM " +"Recipes `_ " +"for more details." +msgstr "" + +#: ../../source/recipes/gpt_oss.rst +msgid "SGLang" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:67 +msgid "**Status:** Not validated with LMCache." +msgstr "" + +#: ../../source/recipes/gpt_oss.rst +msgid "TRT-LLM" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:71 +msgid "**Status:** Not supported. LMCache TRT-LLM integration is in progress." +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:74 +msgid "CacheBlend support" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:77 +msgid "Compression support" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:83 +msgid "Method" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:84 +msgid "Status" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:85 +msgid "Notes" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:86 +msgid ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:87 +msgid "Not validated" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:91 +msgid "Caveats" +msgstr "" + +#: ../../source/recipes/gpt_oss.rst:93 +msgid "None known." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/index.po new file mode 100644 index 00000000000..92880121cef --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/index.po @@ -0,0 +1,211 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/recipes/index.rst:4 +msgid "Recipes" +msgstr "" + +#: ../../source/recipes/index.rst:6 +msgid "" +"This section lists model architectures that have been validated end-to-" +"end with LMCache, with a recipe page per architecture covering only the " +"LMCache-specific configuration that diverges from defaults." +msgstr "" + +#: ../../source/recipes/index.rst:10 +msgid "" +"Engine-side documentation (how to serve the model itself) lives with the " +"serving engine. Recipe pages link out rather than duplicate." +msgstr "" + +#: ../../source/recipes/index.rst:14 +msgid "Recipe page contents" +msgstr "" + +#: ../../source/recipes/index.rst:16 +msgid "Each recipe page is intentionally minimal:" +msgstr "" + +#: ../../source/recipes/index.rst:18 +msgid "**Validated models** -- exact HF repo IDs that have been tested." +msgstr "" + +#: ../../source/recipes/index.rst:19 +msgid "" +"**Engine tabs** -- one tab per serving engine (vLLM, SGLang, TRT-LLM). " +"Each tab links to the engine's own documentation for the model and shows " +"the exact ``lmcache server`` and engine launch commands. Tabs for engines" +" that are not yet validated state so explicitly." +msgstr "" + +#: ../../source/recipes/index.rst:23 +msgid "**CacheBlend support** -- validation status (may be empty)." +msgstr "" + +#: ../../source/recipes/index.rst:24 +msgid "" +"**Compression support** -- table of compression methods (CacheGen, etc.) " +"with per-method validation status. Extensible: new methods get a row." +msgstr "" + +#: ../../source/recipes/index.rst:26 +msgid "**Caveats** -- known limitations, if any." +msgstr "" + +#: ../../source/recipes/index.rst:28 +msgid "" +"For the generic LMCache + engine wiring (ports, remote hosts, in-process " +"mode, sending a first request), see :doc:`../getting_started/quickstart` " +"and :doc:`../mp/quickstart`. Recipes assume those pages as a " +"prerequisite." +msgstr "" + +#: ../../source/recipes/index.rst:33 +msgid "Supported architectures" +msgstr "" + +#: ../../source/recipes/index.rst:39 +msgid "Architecture" +msgstr "" + +#: ../../source/recipes/index.rst:40 +msgid "Example HF model" +msgstr "" + +#: ../../source/recipes/index.rst:41 +msgid "vLLM" +msgstr "" + +#: ../../source/recipes/index.rst:42 +msgid "SGLang" +msgstr "" + +#: ../../source/recipes/index.rst:43 +msgid "TRT-LLM" +msgstr "" + +#: ../../source/recipes/index.rst:44 +msgid "Recipe" +msgstr "" + +#: ../../source/recipes/index.rst:45 +msgid "``MiniMaxM2ForCausalLM``" +msgstr "" + +#: ../../source/recipes/index.rst:46 +msgid "``MiniMaxAI/MiniMax-M2``" +msgstr "" + +#: ../../source/recipes/index.rst:47 ../../source/recipes/index.rst:53 +#: ../../source/recipes/index.rst:59 ../../source/recipes/index.rst:65 +#: ../../source/recipes/index.rst:71 +msgid "✓" +msgstr "" + +#: ../../source/recipes/index.rst:48 ../../source/recipes/index.rst:49 +#: ../../source/recipes/index.rst:54 ../../source/recipes/index.rst:55 +#: ../../source/recipes/index.rst:60 ../../source/recipes/index.rst:61 +#: ../../source/recipes/index.rst:66 ../../source/recipes/index.rst:67 +#: ../../source/recipes/index.rst:72 ../../source/recipes/index.rst:73 +msgid "—" +msgstr "" + +#: ../../source/recipes/index.rst:50 +msgid ":doc:`minimax_m2`" +msgstr "" + +#: ../../source/recipes/index.rst:51 +msgid "``Gemma4ForConditionalGeneration``" +msgstr "" + +#: ../../source/recipes/index.rst:52 +msgid "``google/gemma-4-31B-it``" +msgstr "" + +#: ../../source/recipes/index.rst:56 +msgid ":doc:`gemma4`" +msgstr "" + +#: ../../source/recipes/index.rst:57 +msgid "``MistralForCausalLM``" +msgstr "" + +#: ../../source/recipes/index.rst:58 +msgid "``mistralai/Devstral-2-123B-Instruct-2512``" +msgstr "" + +#: ../../source/recipes/index.rst:62 +msgid ":doc:`devstral`" +msgstr "" + +#: ../../source/recipes/index.rst:63 +msgid "``GptOssForCausalLM``" +msgstr "" + +#: ../../source/recipes/index.rst:64 +msgid "``openai/gpt-oss-120b``" +msgstr "" + +#: ../../source/recipes/index.rst:68 +msgid ":doc:`gpt_oss`" +msgstr "" + +#: ../../source/recipes/index.rst:69 +msgid "``Qwen3MoeForCausalLM``" +msgstr "" + +#: ../../source/recipes/index.rst:70 +msgid "``Qwen/Qwen3-235B-A22B``" +msgstr "" + +#: ../../source/recipes/index.rst:74 +msgid ":doc:`qwen3`" +msgstr "" + +#: ../../source/recipes/index.rst:76 +msgid "Legend: ``✓`` validated, ``—`` not validated." +msgstr "" + +#: ../../source/recipes/index.rst:79 +msgid "Contributing a recipe" +msgstr "" + +#: ../../source/recipes/index.rst:81 +msgid "To add a new architecture:" +msgstr "" + +#: ../../source/recipes/index.rst:83 +msgid "" +"Copy an existing page (e.g. ``minimax_m2.rst``) to " +"``recipes/.rst``." +msgstr "" + +#: ../../source/recipes/index.rst:85 +msgid "" +"Fill in **Validated models**, **Engines**, **LMCache configuration**, and" +" **Caveats**. Keep each section terse -- if a field has nothing to say, " +"say so in one line rather than padding it." +msgstr "" + +#: ../../source/recipes/index.rst:88 +msgid "Add a row to the table above and an entry to the hidden toctree below." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/minimax_m2.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/minimax_m2.po new file mode 100644 index 00000000000..ff066a0b428 --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/minimax_m2.po @@ -0,0 +1,150 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/recipes/minimax_m2.rst:4 +msgid "MiniMaxM2ForCausalLM" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:7 +msgid "Validated models" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:9 +msgid "`MiniMaxAI/MiniMax-M2 `_" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:10 +msgid "`MiniMaxAI/MiniMax-M2.5 `_" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:11 +msgid "`MiniMaxAI/MiniMax-M2.7 `_" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst +msgid "vLLM" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:18 +msgid "" +"**Engine documentation:** `MiniMax-M2 in vLLM supported models " +"`_ (architecture ``MiniMaxM2ForCausalLM``)." +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:23 +msgid "**Status:** Validated with LMCache." +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:25 +msgid "Start the LMCache MP server:" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:33 +msgid "Start vLLM with the LMCache MP connector:" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:35 +msgid "**MiniMax-M2** (8 GPUs):" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:47 +msgid "**MiniMax-M2.5** (4 GPUs):" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:62 +msgid "**MiniMax-M2.7** (4 GPUs):" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:77 +msgid "" +"Adjust ``--tensor-parallel-size`` to match your hardware. For the generic" +" LMCache + vLLM wiring (ports, remote hosts, in-process mode), see " +":doc:`../mp/quickstart`." +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:81 +msgid "" +"If there are any issues with vLLM setup, please refer to the `vLLM " +"Recipes `_ " +"for more details." +msgstr "" + +#: ../../source/recipes/minimax_m2.rst +msgid "SGLang" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:87 +msgid "" +"**Engine documentation:** `MiniMax-M2 SGLang cookbook " +"`_, " +"`MiniMax M2.5/M2.1/M2 usage guide " +"`_." +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:93 +msgid "**Status:** Not validated with LMCache." +msgstr "" + +#: ../../source/recipes/minimax_m2.rst +msgid "TRT-LLM" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:97 +msgid "**Status:** Not supported. LMCache TRT-LLM integration is in progress." +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:100 +msgid "CacheBlend support" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:103 +msgid "Compression support" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:109 +msgid "Method" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:110 +msgid "Status" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:111 +msgid "Notes" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:112 +msgid ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:113 +msgid "Not validated" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:117 +msgid "Caveats" +msgstr "" + +#: ../../source/recipes/minimax_m2.rst:119 +msgid "None known." +msgstr "" + diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/qwen3.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/qwen3.po new file mode 100644 index 00000000000..d6badb4ff7c --- /dev/null +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/qwen3.po @@ -0,0 +1,150 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) 2024, The LMCache Team +# This file is distributed under the same license as the LMCache package. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: LMCache \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: zh_CN\n" +"Language-Team: zh_CN \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +#: ../../source/recipes/qwen3.rst:4 +msgid "Qwen3MoeForCausalLM" +msgstr "" + +#: ../../source/recipes/qwen3.rst:7 +msgid "Validated models" +msgstr "" + +#: ../../source/recipes/qwen3.rst:9 +msgid "`Qwen/Qwen3-235B-A22B `_" +msgstr "" + +#: ../../source/recipes/qwen3.rst:10 +msgid "`Qwen/Qwen3-30B-A3B `_" +msgstr "" + +#: ../../source/recipes/qwen3.rst:11 +msgid "" +"`Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8 " +"`_" +msgstr "" + +#: ../../source/recipes/qwen3.rst:12 +msgid "" +"`Qwen/Qwen3-Coder-30B-A3B-Instruct `_" +msgstr "" + +#: ../../source/recipes/qwen3.rst +msgid "vLLM" +msgstr "" + +#: ../../source/recipes/qwen3.rst:19 +msgid "" +"**Engine documentation:** `Qwen3 MoE in vLLM supported models " +"`_ (architecture ``Qwen3MoeForCausalLM``)." +msgstr "" + +#: ../../source/recipes/qwen3.rst:24 +msgid "**Status:** Validated with LMCache." +msgstr "" + +#: ../../source/recipes/qwen3.rst:26 +msgid "Start the LMCache MP server:" +msgstr "" + +#: ../../source/recipes/qwen3.rst:34 +msgid "**Qwen3-235B-A22B** (4 GPUs, expert parallel):" +msgstr "" + +#: ../../source/recipes/qwen3.rst:49 +msgid "**Qwen3-30B-A3B** (1 GPU):" +msgstr "" + +#: ../../source/recipes/qwen3.rst:62 +msgid "**Qwen3-Coder-480B-A35B-Instruct-FP8** (8 GPUs, expert parallel):" +msgstr "" + +#: ../../source/recipes/qwen3.rst:76 +msgid "**Qwen3-Coder-30B-A3B-Instruct** (1 GPU):" +msgstr "" + +#: ../../source/recipes/qwen3.rst:88 +msgid "" +"Adjust ``--tensor-parallel-size`` to match your hardware. For the generic" +" LMCache + vLLM wiring (ports, remote hosts, in-process mode), see " +":doc:`../mp/quickstart`." +msgstr "" + +#: ../../source/recipes/qwen3.rst:92 +msgid "" +"If there are any issues with vLLM setup, please refer to the `vLLM " +"Recipes `_ " +"for more details." +msgstr "" + +#: ../../source/recipes/qwen3.rst +msgid "SGLang" +msgstr "" + +#: ../../source/recipes/qwen3.rst:98 +msgid "**Status:** Not validated with LMCache." +msgstr "" + +#: ../../source/recipes/qwen3.rst +msgid "TRT-LLM" +msgstr "" + +#: ../../source/recipes/qwen3.rst:102 +msgid "**Status:** Not supported. LMCache TRT-LLM integration is in progress." +msgstr "" + +#: ../../source/recipes/qwen3.rst:105 +msgid "CacheBlend support" +msgstr "" + +#: ../../source/recipes/qwen3.rst:108 +msgid "Compression support" +msgstr "" + +#: ../../source/recipes/qwen3.rst:114 +msgid "Method" +msgstr "" + +#: ../../source/recipes/qwen3.rst:115 +msgid "Status" +msgstr "" + +#: ../../source/recipes/qwen3.rst:116 +msgid "Notes" +msgstr "" + +#: ../../source/recipes/qwen3.rst:117 +msgid ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" +msgstr "" + +#: ../../source/recipes/qwen3.rst:118 +msgid "Not validated" +msgstr "" + +#: ../../source/recipes/qwen3.rst:122 +msgid "Caveats" +msgstr "" + +#: ../../source/recipes/qwen3.rst:124 +msgid "None known." +msgstr "" + From 10ad9e42d39d513e86647519c30d93a162334c66 Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Wed, 20 May 2026 05:23:17 +0800 Subject: [PATCH 34/69] chore: add hlin99 to CODEOWNERS (#3316) Signed-off-by: Tony Lin --- .github/CODEOWNERS | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 21201a99356..b70783ae4ba 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,21 +9,24 @@ /lmcache/v1/memory_management.py @sammshen @deng451e @chunxiaozheng @DongDongJu @ApostaC # Multiprocess -/lmcache/v1/multiprocess/ @ApostaC @OasisGit +/lmcache/v1/multiprocess/ @ApostaC @OasisGit @hlin99 /lmcache/v1/multiprocess/protocols/observability.py @royyhuang @ApostaC /lmcache/v1/multiprocess/http_server.py @royyhuang @ApostaC @OasisGit /lmcache/v1/mp_observability @royyhuang @ApostaC @OasisGit @sammshen # Distributed / L2 /lmcache/v1/distributed/ @ApostaC @maobaolong @chunxiaozheng -/lmcache/v1/distributed/eviction.py @YaoJiayi -/lmcache/v1/distributed/eviction_policy/ @YaoJiayi -/lmcache/v1/distributed/storage_controllers/eviction_controller.py @YaoJiayi +/lmcache/v1/distributed/eviction.py @YaoJiayi @hlin99 +/lmcache/v1/distributed/eviction_policy/ @YaoJiayi @hlin99 +/lmcache/v1/distributed/storage_controllers/eviction_controller.py @YaoJiayi @hlin99 /lmcache/v1/distributed/l2_adapters/mooncake_store_l2_adapter.py @maobaolong /lmcache/v1/distributed/l2_adapters/resp_l2_adapter.py @sammshen # GPU connector -/lmcache/v1/gpu_connector/ @sammshen @ApostaC +/lmcache/v1/gpu_connector/ @sammshen @ApostaC @hlin99 + +# Platform +/lmcache/v1/platform/ @maobaolong @hlin99 # Lookup client /lmcache/v1/lookup_client/ @maobaolong @YaoJiayi @sammshen @@ -41,13 +44,13 @@ /lmcache/v1/storage_backend/connector/fs_adapter.py @maobaolong @chunxiaozheng /lmcache/v1/storage_backend/connector/audit_connector.py @maobaolong @chunxiaozheng /lmcache/v1/storage_backend/connector/audit_adapter.py @maobaolong @chunxiaozheng -/lmcache/v1/storage_backend/connector/redis_connector.py @sammshen -/lmcache/v1/storage_backend/connector/redis_adapter.py @sammshen -/lmcache/v1/storage_backend/connector/lm_connector.py @sammshen -/lmcache/v1/storage_backend/connector/lm_adapter.py @sammshen +/lmcache/v1/storage_backend/connector/redis_connector.py @sammshen @hlin99 +/lmcache/v1/storage_backend/connector/redis_adapter.py @sammshen @hlin99 +/lmcache/v1/storage_backend/connector/lm_connector.py @sammshen @hlin99 +/lmcache/v1/storage_backend/connector/lm_adapter.py @sammshen @hlin99 # Integration -/lmcache/integration/vllm/ @sammshen @maobaolong @YaoJiayi @deng451e +/lmcache/integration/vllm/ @sammshen @maobaolong @YaoJiayi @deng451e @hlin99 /lmcache/integration/sglang/ @OasisGit @DongDongJu @sammshen /lmcache/integration/tensorrt_llm/ @sammshen @@ -56,12 +59,12 @@ # C extensions /csrc/ @ApostaC @YaoJiayi @sammshen -/csrc/storage_backends/ @sammshen +/csrc/storage_backends/ @sammshen @hlin99 # Native connector L2 adapters -/lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py @sammshen -/lmcache/v1/distributed/l2_adapters/native_plugin_l2_adapter.py @sammshen -/lmcache/v1/distributed/l2_adapters/fs_native_l2_adapter.py @sammshen +/lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py @sammshen @hlin99 +/lmcache/v1/distributed/l2_adapters/native_plugin_l2_adapter.py @sammshen @hlin99 +/lmcache/v1/distributed/l2_adapters/fs_native_l2_adapter.py @sammshen @hlin99 # Tests /tests/ @hickeyma @sammshen @ApostaC @deng451e From d74bcfcd6727a3b348a61fee1221f3cca714014e Mon Sep 17 00:00:00 2001 From: kumaneko Date: Wed, 20 May 2026 05:23:38 +0800 Subject: [PATCH 35/69] Add yoo-kumaneko as CODEOWNER for mp_observability and tools (#3324) Signed-off-by: rigginschen --- .github/CODEOWNERS | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b70783ae4ba..e2961fb6712 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,7 +12,7 @@ /lmcache/v1/multiprocess/ @ApostaC @OasisGit @hlin99 /lmcache/v1/multiprocess/protocols/observability.py @royyhuang @ApostaC /lmcache/v1/multiprocess/http_server.py @royyhuang @ApostaC @OasisGit -/lmcache/v1/mp_observability @royyhuang @ApostaC @OasisGit @sammshen +/lmcache/v1/mp_observability/ @yoo-kumaneko @royyhuang @ApostaC @OasisGit @sammshen # Distributed / L2 /lmcache/v1/distributed/ @ApostaC @maobaolong @chunxiaozheng @@ -66,6 +66,9 @@ /lmcache/v1/distributed/l2_adapters/native_plugin_l2_adapter.py @sammshen @hlin99 /lmcache/v1/distributed/l2_adapters/fs_native_l2_adapter.py @sammshen @hlin99 +# Offline analysis tools +/lmcache/tools/ @yoo-kumaneko + # Tests /tests/ @hickeyma @sammshen @ApostaC @deng451e From 272115eecb129b216df12f7ee93a18712dcf3c32 Mon Sep 17 00:00:00 2001 From: Shaoting Date: Tue, 19 May 2026 15:34:16 -0700 Subject: [PATCH 36/69] [MP][Observability] Refactor metric names and units (#3290) * [MP][Observability] Refactor metric names and units Signed-off-by: Shaoting-Feng --- .../scripts/run-long-doc-qa-l2.sh | 24 ++-- .../scripts/run-restart-recovery.sh | 12 +- docs/design/v1/mp_observability/DEBUG.md | 4 +- docs/design/v1/mp_observability/METRICS.md | 120 ++++++---------- docs/source/mp/observability.rst | 131 +++++++----------- .../provisioning/dashboards/lmcache.json | 58 ++++---- lmcache/v1/mp_observability/config.py | 2 - lmcache/v1/mp_observability/event_bus.py | 22 +++ .../mp_observability/subscribers/__init__.py | 2 - .../subscribers/metrics/__init__.py | 2 - .../subscribers/metrics/event_bus.py | 20 +-- .../subscribers/metrics/l0_l1_throughput.py | 8 +- .../subscribers/metrics/l0_lifecycle.py | 12 +- .../subscribers/metrics/l1.py | 21 +-- .../subscribers/metrics/l1_failures.py | 2 + .../subscribers/metrics/l1_lifecycle.py | 16 +-- .../subscribers/metrics/l2.py | 127 +++++++++-------- .../subscribers/metrics/l2_failures.py | 1 + .../subscribers/metrics/l2_throughput.py | 8 +- .../subscribers/metrics/lookup.py | 8 +- .../subscribers/metrics/sm.py | 69 --------- .../subscribers/metrics/sm_lifecycle.py | 4 +- .../metrics/test_event_bus_self_metrics.py | 24 +--- .../metrics/test_l0_l1_throughput.py | 4 +- .../subscribers/metrics/test_l0_lifecycle.py | 26 ++-- .../subscribers/metrics/test_l1_lifecycle.py | 36 ++--- .../subscribers/metrics/test_l2.py | 62 ++++----- .../subscribers/metrics/test_l2_throughput.py | 4 +- .../subscribers/metrics/test_lookup.py | 34 +++-- .../subscribers/metrics/test_sm.py | 105 -------------- .../subscribers/metrics/test_sm_lifecycle.py | 66 ++++----- tests/v1/mp_observability/test_event_bus.py | 28 +++- 32 files changed, 404 insertions(+), 658 deletions(-) delete mode 100644 lmcache/v1/mp_observability/subscribers/metrics/sm.py delete mode 100644 tests/v1/mp_observability/subscribers/metrics/test_sm.py diff --git a/.buildkite/k3_tests/multiprocess/scripts/run-long-doc-qa-l2.sh b/.buildkite/k3_tests/multiprocess/scripts/run-long-doc-qa-l2.sh index d20a64024f4..b61b0f48baa 100755 --- a/.buildkite/k3_tests/multiprocess/scripts/run-long-doc-qa-l2.sh +++ b/.buildkite/k3_tests/multiprocess/scripts/run-long-doc-qa-l2.sh @@ -388,14 +388,14 @@ def get_counter(name): return 0.0 # L1 metrics -l1_write_keys = get_counter('lmcache_mp_l1_write_keys_total') +l1_write_keys = get_counter('lmcache_mp_l1_write_chunks_total') # L2 metrics -store_keys = get_counter('lmcache_mp_l2_store_keys_total') -store_succeeded = get_counter('lmcache_mp_l2_store_succeeded_keys_total') -prefetch_lookups = get_counter('lmcache_mp_l2_prefetch_lookups_total') -prefetch_hits = get_counter('lmcache_mp_l2_prefetch_hit_keys_total') -prefetch_loaded = get_counter('lmcache_mp_l2_prefetch_loaded_keys_total') +store_keys = get_counter('lmcache_mp_l2_store_submitted_objects_chunks_total') +store_succeeded = get_counter('lmcache_mp_l2_store_completed_objects_chunks_total') +prefetch_lookups = get_counter('lmcache_mp_l2_prefetch_lookup_requests_total') +prefetch_hits = get_counter('lmcache_mp_l2_prefetch_hit_chunks_total') +prefetch_loaded = get_counter('lmcache_mp_l2_prefetch_load_completed_chunks_total') print('=' * 60) print('Data Flow Metrics') @@ -518,8 +518,8 @@ def has_label(base_name: str, label: str) -> bool: # (kind, metric_name, optional_label_to_assert_present_or_None) checks = [ # ── Newer counters (with label dimensions) ───────────────────── - ("counter", "lmcache_mp_l2_store_completed_total", "l2_name"), - ("counter", "lmcache_mp_l2_load_completed_total", "l2_name"), + ("counter", "lmcache_mp_l2_store_completed_requests_total", "l2_name"), + ("counter", "lmcache_mp_l2_load_completed_requests_total", "l2_name"), ("counter", "lmcache_mp_lookup_requested_tokens_total", "model_name"), ("counter", "lmcache_mp_lookup_hit_tokens_total", "model_name"), ("counter", "lmcache_mp_num_chunks_loaded_total", "worker_id"), @@ -528,10 +528,10 @@ checks = [ # ``unit="GB/s"`` actually reports as # ``_GB_per_second_count`` / ``..._sum`` / ``..._bucket``. # Match by base name and let the helper tolerate the unit suffix. - ("hist", "lmcache_mp_l0_l1_store_throughput_gbs", None), - ("hist", "lmcache_mp_l0_l1_load_throughput_gbs", None), - ("hist", "lmcache_mp_l2_store_throughput_gbs", "l2_name"), - ("hist", "lmcache_mp_l2_load_throughput_gbs", "l2_name"), + ("hist", "lmcache_mp_l0_l1_store_throughput", None), + ("hist", "lmcache_mp_l0_l1_load_throughput", None), + ("hist", "lmcache_mp_l2_store_throughput", "l2_name"), + ("hist", "lmcache_mp_l2_load_throughput", "l2_name"), ] failed = False diff --git a/.buildkite/k3_tests/multiprocess/scripts/run-restart-recovery.sh b/.buildkite/k3_tests/multiprocess/scripts/run-restart-recovery.sh index 5b94ede8318..95d57d88cde 100755 --- a/.buildkite/k3_tests/multiprocess/scripts/run-restart-recovery.sh +++ b/.buildkite/k3_tests/multiprocess/scripts/run-restart-recovery.sh @@ -5,7 +5,7 @@ # # Flow: # 1. Run a `lmcache bench engine --workload random-prefill` round. -# 2. Snapshot lmcache_mp_l1_write_keys_total via /metrics. +# 2. Snapshot lmcache_mp_l1_write_chunks_total via /metrics. # 3. Kill the LMCache server, relaunch on the same port. # 4. Wait for the new server to be ready and for the worker to # re-register (poll /status until gpu_context_meta is non-empty). @@ -97,7 +97,7 @@ run_bench_round() { } scrape_l1_write_keys() { - # Print the latest lmcache_mp_l1_write_keys_total value (single sample, + # Print the latest lmcache_mp_l1_write_chunks_total value (single sample, # unlabeled) from the LMCache HTTP server's /metrics endpoint. python3 - < **Note:** `SM_READ_PREFETCHED_FINISHED` is published but has no metrics subscriber — -> it is available for logging subscribers only. - ---- - -## StorageManager Write Metrics - -| OTel metric name | Prometheus name | Type | Source event | Calculation | -|---|---|---|---|---| -| `lmcache_mp.sm_write_requests` | `lmcache_mp_sm_write_requests_total` | Counter | `SM_WRITE_RESERVED` | +1 per event | -| `lmcache_mp.sm_write_succeed_keys` | `lmcache_mp_sm_write_succeed_keys_total` | Counter | `SM_WRITE_RESERVED` | `+len(succeeded_keys)` | -| `lmcache_mp.sm_write_failed_keys` | `lmcache_mp_sm_write_failed_keys_total` | Counter | `SM_WRITE_RESERVED` | `+len(failed_keys)` | - -**What it answers:** How often are writes attempted? What fraction fail due to OOM or write conflicts? - -> **Note:** `SM_WRITE_FINISHED` is published but has no metrics subscriber. - ---- - ## L1 Read Metrics | OTel metric name | Prometheus name | Type | Source event | Calculation | |---|---|---|---|---| -| `lmcache_mp.l1_read_keys` | `lmcache_mp_l1_read_keys_total` | Counter | `L1_READ_FINISHED` | `+len(keys)` | +| `lmcache_mp.l1_read` | `lmcache_mp_l1_read_chunks_total` | Counter | `L1_READ_FINISHED` | `+len(keys)` | -**What it answers:** How many keys are being read from L1? +**What it answers:** How many chunks are being read from L1? > **Note:** `L1_READ_RESERVED` is published but has no metrics subscriber — key counts > are recorded only when the read actually completes. @@ -85,10 +57,10 @@ datapoints and are orthogonal to these Resource attributes. | OTel metric name | Prometheus name | Type | Source event | Calculation | |---|---|---|---|---| -| `lmcache_mp.l1_write_keys` | `lmcache_mp_l1_write_keys_total` | Counter | `L1_WRITE_FINISHED` | `+len(keys)` | +| `lmcache_mp.l1_write` | `lmcache_mp_l1_write_chunks_total` | Counter | `L1_WRITE_FINISHED` | `+len(keys)` | | *(same counter)* | *(same)* | Counter | `L1_WRITE_FINISHED_AND_READ_RESERVED` | `+len(keys)` | -**What it answers:** How many keys are being written to L1? +**What it answers:** How many chunks are being written to L1? > **Note:** `L1_WRITE_RESERVED` is published but has no metrics subscriber. > `L1_WRITE_FINISHED_AND_READ_RESERVED` (atomic write-then-read used by prefetch) @@ -100,7 +72,7 @@ datapoints and are orthogonal to these Resource attributes. | OTel metric name | Prometheus name | Type | Source event | Calculation | |---|---|---|---|---| -| `lmcache_mp.l1_evicted_keys` | `lmcache_mp_l1_evicted_keys_total` | Counter | `L1_KEYS_EVICTED` | `+len(keys)` | +| `lmcache_mp.l1_evicted` | `lmcache_mp_l1_evicted_chunks_total` | Counter | `L1_KEYS_EVICTED` | `+len(keys)` | | `lmcache_mp.l1_eviction_loop_ticks` | `lmcache_mp_l1_eviction_loop_ticks_total` | Counter | `L1_EVICTION_LOOP_TICK` | +1 per loop iteration | | `lmcache_mp.l1_eviction_loop_triggered` | `lmcache_mp_l1_eviction_loop_triggered_total` | Counter | `L1_EVICTION_LOOP_TICK` | +1 when `triggered=True` | | `lmcache_mp.l1_usage_ratio` | `lmcache_mp_l1_usage_ratio` | Observable Gauge | (callback on `L1Manager`) | `used / total` at scrape time | @@ -124,8 +96,8 @@ require a cross-cutting API change out of scope for this PR. | OTel metric name | Prometheus name | Type | Source event | Calculation | Tags | |---|---|---|---|---|---| -| `lmcache_mp.l1_allocation_failure` | `lmcache_mp_l1_allocation_failure_total` | Counter | `L1_ALLOCATION_FAILED` | `+count` per `(during, model_name)` bucket | `during` ∈ {`l1_store`, `l2_prefetch`}, `model_name` | -| `lmcache_mp.l1_read_failure` | `lmcache_mp_l1_read_failure_total` | Counter | `L1_READ_FAILED` | `+count` per `(during, reason, model_name)` bucket | `during` ∈ {`l2_store`, `l1_retrieve`}, `reason` ∈ {`not_found`, `write_locked`}, `model_name` | +| `lmcache_mp.l1_allocation_failure` | `lmcache_mp_l1_allocation_failure_chunks_total` | Counter | `L1_ALLOCATION_FAILED` | `+count` per `(during, model_name)` bucket | `during` ∈ {`l1_store`, `l2_prefetch`}, `model_name` | +| `lmcache_mp.l1_read_failure` | `lmcache_mp_l1_read_failure_chunks_total` | Counter | `L1_READ_FAILED` | `+count` per `(during, reason, model_name)` bucket | `during` ∈ {`l2_store`, `l1_retrieve`}, `reason` ∈ {`not_found`, `write_locked`}, `model_name` | **What it answers:** - `l1_allocation_failure` — how often is L1 rejecting writes for lack of memory, split by whether the pressure is user stores or L2 prefetch? @@ -140,12 +112,12 @@ contribute to histograms; counters above always count all events. | OTel metric name | Prometheus name | Type | Source event | Calculation | |---|---|---|---|---| -| `lmcache_mp.l1_chunk_lifetime_seconds` | `lmcache_mp_l1_chunk_lifetime_seconds` | Histogram | `L1_KEYS_EVICTED` | `eviction_time - alloc_time` per sampled chunk | -| `lmcache_mp.l1_chunk_idle_before_evict_seconds` | `lmcache_mp_l1_chunk_idle_before_evict_seconds` | Histogram | `L1_KEYS_EVICTED` | `eviction_time - last_access_time` per sampled chunk | -| `lmcache_mp.l1_chunk_reuse_gap_seconds` | `lmcache_mp_l1_chunk_reuse_gap_seconds` | Histogram | `L1_READ_FINISHED`, `L1_WRITE_FINISHED`, `L1_WRITE_FINISHED_AND_READ_RESERVED` | Time gap between consecutive touches of the same chunk | -| `lmcache_mp.l1_chunk_evict_reuse_gap_seconds` | `lmcache_mp_l1_chunk_evict_reuse_gap_seconds` | Histogram | `L1_KEYS_EVICTED` → `L1_WRITE_FINISHED` | Time from eviction to next reuse (capped at 300 s) | -| `lmcache_mp.real_reuse_gap_seconds` | `lmcache_mp_real_reuse_gap_seconds` | Histogram (tagged `cache_salt`) | `SM_READ_PREFETCHED_FINISHED`, `SM_WRITE_FINISHED` | Time gap between a chunk's last access (read or write) and the next read. Captures **storage cost**. Emitted only on read events. | -| `lmcache_mp.real_reuse_gap_chunks` | `lmcache_mp_real_reuse_gap_chunks` | Histogram (tagged `cache_salt`) | `SM_READ_PREFETCHED_FINISHED`, `SM_WRITE_FINISHED` | Per-`cache_salt` access-counter gap between two reads of the same chunk. Counter bumps on every read and write of every chunk; histogram emitted only on read events for sampled chunks. Captures **storage volume**. | +| `lmcache_mp.l1_chunk_lifetime` | `lmcache_mp_l1_chunk_lifetime_seconds` | Histogram | `L1_KEYS_EVICTED` | `eviction_time - alloc_time` per sampled chunk | +| `lmcache_mp.l1_chunk_idle_before_evict` | `lmcache_mp_l1_chunk_idle_before_evict_seconds` | Histogram | `L1_KEYS_EVICTED` | `eviction_time - last_access_time` per sampled chunk | +| `lmcache_mp.l1_chunk_reuse_gap` | `lmcache_mp_l1_chunk_reuse_gap_seconds` | Histogram | `L1_READ_FINISHED`, `L1_WRITE_FINISHED`, `L1_WRITE_FINISHED_AND_READ_RESERVED` | Time gap between consecutive touches of the same chunk | +| `lmcache_mp.l1_chunk_evict_reuse_gap` | `lmcache_mp_l1_chunk_evict_reuse_gap_seconds` | Histogram | `L1_KEYS_EVICTED` → `L1_WRITE_FINISHED` | Time from eviction to next reuse (capped at 300 s) | +| `lmcache_mp.real_reuse_gap` | `lmcache_mp_real_reuse_gap_seconds` | Histogram (tagged `cache_salt`) | `SM_READ_PREFETCHED_FINISHED`, `SM_WRITE_FINISHED` | Time gap between a chunk's last access (read or write) and the next read. Captures **storage cost**. Emitted only on read events. | +| `lmcache_mp.real_reuse_gap_objects` | `lmcache_mp_real_reuse_gap_objects_chunks` | Histogram (tagged `cache_salt`) | `SM_READ_PREFETCHED_FINISHED`, `SM_WRITE_FINISHED` | Per-`cache_salt` access-counter gap between two reads of the same chunk. Counter bumps on every read and write of every chunk; histogram emitted only on read events for sampled chunks. Captures **storage volume**. | **What it answers:** How long do L1 chunks live? How idle are they before eviction? How quickly are evicted chunks reused? @@ -156,13 +128,12 @@ contribute to histograms; counters above always count all events. | OTel metric name | Prometheus name | Type | Source event | Calculation | |---|---|---|---|---| -| `lmcache_mp.l2_store_tasks` | `lmcache_mp_l2_store_tasks_total` | Counter | `L2_STORE_SUBMITTED` | +1 per event | -| `lmcache_mp.l2_store_keys` | `lmcache_mp_l2_store_keys_total` | Counter | `L2_STORE_SUBMITTED` | `+key_count` | -| `lmcache_mp.l2_store_completed` | `lmcache_mp_l2_store_completed_total` | Counter (attr: `l2_name`) | `L2_STORE_COMPLETED` | +1 per event | -| `lmcache_mp.l2_store_succeeded_keys` | `lmcache_mp_l2_store_succeeded_keys_total` | Counter | `L2_STORE_COMPLETED` | `+succeeded_count` | -| `lmcache_mp.l2_store_failed_keys` | `lmcache_mp_l2_store_failed_keys_total` | Counter | `L2_STORE_COMPLETED` | `+failed_count` | +| `lmcache_mp.l2_store_submitted` | `lmcache_mp_l2_store_submitted_requests_total` | Counter | `L2_STORE_SUBMITTED` | +1 per event | +| `lmcache_mp.l2_store_submitted_objects` | `lmcache_mp_l2_store_submitted_objects_chunks_total` | Counter | `L2_STORE_SUBMITTED` | `+key_count` | +| `lmcache_mp.l2_store_completed` | `lmcache_mp_l2_store_completed_requests_total` | Counter (attr: `l2_name`) | `L2_STORE_COMPLETED` | +1 per event | +| `lmcache_mp.l2_store_completed_objects` | `lmcache_mp_l2_store_completed_objects_chunks_total` | Counter | `L2_STORE_COMPLETED` | `+succeeded_count` | -**What it answers:** How many keys are being pushed to L2? What fraction fail? +**What it answers:** How many chunks are being pushed to L2? What fraction fail? --- @@ -170,22 +141,21 @@ contribute to histograms; counters above always count all events. | OTel metric name | Prometheus name | Type | Source event | Calculation | |---|---|---|---|---| -| `lmcache_mp.l2_prefetch_lookups` | `lmcache_mp_l2_prefetch_lookups_total` | Counter | `L2_PREFETCH_LOOKUP_SUBMITTED` | +1 per event | -| `lmcache_mp.l2_prefetch_lookup_keys` | `lmcache_mp_l2_prefetch_lookup_keys_total` | Counter | `L2_PREFETCH_LOOKUP_SUBMITTED` | `+key_count` | -| `lmcache_mp.l2_prefetch_hit_keys` | `lmcache_mp_l2_prefetch_hit_keys_total` | Counter | `L2_PREFETCH_LOOKUP_COMPLETED` | `+prefix_hit_count` | -| `lmcache_mp.l2_prefetch_load_tasks` | `lmcache_mp_l2_prefetch_load_tasks_total` | Counter | `L2_PREFETCH_LOAD_SUBMITTED` | `+adapter_count` | -| `lmcache_mp.l2_prefetch_load_keys` | `lmcache_mp_l2_prefetch_load_keys_total` | Counter | `L2_PREFETCH_LOAD_SUBMITTED` | `+key_count` | -| `lmcache_mp.l2_prefetch_loaded_keys` | `lmcache_mp_l2_prefetch_loaded_keys_total` | Counter | `L2_PREFETCH_LOAD_COMPLETED` | `+loaded_count` | -| `lmcache_mp.l2_prefetch_failed_keys` | `lmcache_mp_l2_prefetch_failed_keys_total` | Counter | `L2_PREFETCH_LOAD_COMPLETED` | `+failed_count` | -| `lmcache_mp.l2_load_completed` | `lmcache_mp_l2_load_completed_total` | Counter (attr: `l2_name`) | `L2_LOAD_TASK_COMPLETED` | +1 per event | +| `lmcache_mp.l2_prefetch_lookup` | `lmcache_mp_l2_prefetch_lookup_requests_total` | Counter | `L2_PREFETCH_LOOKUP_SUBMITTED` | +1 per event | +| `lmcache_mp.l2_prefetch_lookup_objects` | `lmcache_mp_l2_prefetch_lookup_objects_chunks_total` | Counter | `L2_PREFETCH_LOOKUP_SUBMITTED` | `+key_count` | +| `lmcache_mp.l2_prefetch_hit` | `lmcache_mp_l2_prefetch_hit_chunks_total` | Counter | `L2_PREFETCH_LOOKUP_COMPLETED` | `+prefix_hit_count` | +| `lmcache_mp.l2_prefetch_load_submitted` | `lmcache_mp_l2_prefetch_load_submitted_requests_total` | Counter | `L2_PREFETCH_LOAD_SUBMITTED` | `+adapter_count` (per-adapter task count) | +| `lmcache_mp.l2_prefetch_load_submitted_objects` | `lmcache_mp_l2_prefetch_load_submitted_objects_chunks_total` | Counter | `L2_PREFETCH_LOAD_SUBMITTED` | `+key_count` | +| `lmcache_mp.l2_prefetch_load_completed` | `lmcache_mp_l2_prefetch_load_completed_chunks_total` | Counter | `L2_PREFETCH_LOAD_COMPLETED` | `+loaded_count` | +| `lmcache_mp.l2_load_completed` | `lmcache_mp_l2_load_completed_requests_total` | Counter (attr: `l2_name`) | `L2_LOAD_TASK_COMPLETED` | +1 per event | -**What it answers:** How effective is L2 prefetching? What is the L2 hit rate? How many keys fail to load? +**What it answers:** How effective is L2 prefetching? What is the L2 hit rate? **Per-backend IOPS.** `lmcache_mp.l2_store_completed` (attr `l2_name`) counts completed L1→L2 store tasks; `lmcache_mp.l2_load_completed` (attr `l2_name`) counts completed per-adapter L2→L1 load tasks. Derive per-backend ops/sec on the dashboard with -`rate(lmcache_mp_l2_store_completed_total{l2_name="..."}[1m])` +`rate(lmcache_mp_l2_store_completed_requests_total{l2_name="..."}[1m])` (and the equivalent for loads). No separate `*_iops` metric is exported — the raw counter keeps the window choice in the dashboard. @@ -205,8 +175,8 @@ scrape time with `metric_relabel_configs` if storage cost matters. | OTel metric name | Prometheus name | Type | Source event | Calculation | |---|---|---|---|---| -| `lmcache_mp.lookup_requested_tokens` | `lmcache_mp_lookup_requested_tokens_total` | Counter (attrs: `model_name`, `cache_salt`) | `MP_LOOKUP_PREFETCH_END` | `+requested_tokens` | -| `lmcache_mp.lookup_hit_tokens` | `lmcache_mp_lookup_hit_tokens_total` | Counter (attrs: `model_name`, `cache_salt`) | `MP_LOOKUP_PREFETCH_END` | `+hit_tokens` | +| `lmcache_mp.lookup_requested` | `lmcache_mp_lookup_requested_tokens_total` | Counter (attrs: `model_name`, `cache_salt`) | `MP_LOOKUP_PREFETCH_END` | `+requested_tokens` | +| `lmcache_mp.lookup_hit` | `lmcache_mp_lookup_hit_tokens_total` | Counter (attrs: `model_name`, `cache_salt`) | `MP_LOOKUP_PREFETCH_END` | `+hit_tokens` | **What it answers:** What fraction of tokens requested by a lookup were served from cache (L1 or L2)? @@ -233,7 +203,7 @@ sum(rate(lmcache_mp_lookup_hit_tokens_total[5m])) by (model_name) | OTel metric name | Prometheus name | Type | Source event | Calculation | Tags | |---|---|---|---|---|---| -| `lmcache_mp.l2_prefetch_failure` | `lmcache_mp_l2_prefetch_failure_total` | Counter | `L2_PREFETCH_FAILED` | `+count` per `(reason, model_name)` bucket | `reason` ∈ {`l1_oom`, `not_found`}, `model_name` | +| `lmcache_mp.l2_prefetch_failure` | `lmcache_mp_l2_prefetch_failure_chunks_total` | Counter | `L2_PREFETCH_FAILED` | `+count` per `(reason, model_name)` bucket | `reason` ∈ {`l1_oom`, `not_found`}, `model_name` | **What it answers:** For keys L2 reported present at lookup but failed to land in L1: was L1 full (`l1_oom`), or did the adapter fail to produce the data (`not_found`)? @@ -255,9 +225,9 @@ per-instance and per-model Prometheus metric slicing (e.g. | OTel metric name | Prometheus name | Type | Source event | Calculation | |---|---|---|---|---| -| `lmcache_mp.l0_block_lifetime_seconds` | `lmcache_mp_l0_block_lifetime_seconds` | Histogram | `MP_VLLM_BLOCK_ALLOCATION` (eviction detected) | `eviction_time - alloc_time` per sampled block | -| `lmcache_mp.l0_block_idle_before_evict_seconds` | `lmcache_mp_l0_block_idle_before_evict_seconds` | Histogram | `MP_VLLM_BLOCK_ALLOCATION` (eviction detected) | `eviction_time - last_access_time` per sampled block | -| `lmcache_mp.l0_block_reuse_gap_seconds` | `lmcache_mp_l0_block_reuse_gap_seconds` | Histogram | `MP_VLLM_BLOCK_ALLOCATION` (cache hit) | Time gaps between consecutive accesses from access history | +| `lmcache_mp.l0_block_lifetime` | `lmcache_mp_l0_block_lifetime_seconds` | Histogram | `MP_VLLM_BLOCK_ALLOCATION` (eviction detected) | `eviction_time - alloc_time` per sampled block | +| `lmcache_mp.l0_block_idle_before_evict` | `lmcache_mp_l0_block_idle_before_evict_seconds` | Histogram | `MP_VLLM_BLOCK_ALLOCATION` (eviction detected) | `eviction_time - last_access_time` per sampled block | +| `lmcache_mp.l0_block_reuse_gap` | `lmcache_mp_l0_block_reuse_gap_seconds` | Histogram | `MP_VLLM_BLOCK_ALLOCATION` (cache hit) | Time gaps between consecutive accesses from access history | **What it answers:** How long do GPU blocks live before eviction? How idle are they? How frequently are cached blocks reused? Which instance/model is experiencing the most churn? @@ -279,8 +249,8 @@ per-worker, per-device, and per-model slicing in Prometheus (e.g. | OTel metric name | Prometheus name | Type | Source event | Calculation | |---|---|---|---|---| -| `lmcache_mp.l0_l1_store_throughput_gbs` | `lmcache_mp_l0_l1_store_throughput_gbs` | Histogram | `MP_STORE_START` → `MP_STORE_END` | `total_bytes / (end_ts - start_ts) / 1e9` per request | -| `lmcache_mp.l0_l1_load_throughput_gbs` | `lmcache_mp_l0_l1_load_throughput_gbs` | Histogram | `MP_RETRIEVE_START` → `MP_RETRIEVE_END` | `total_bytes / (end_ts - start_ts) / 1e9` per request | +| `lmcache_mp.l0_l1_store_throughput` | `lmcache_mp_l0_l1_store_throughput_GBs` | Histogram | `MP_STORE_START` → `MP_STORE_END` | `total_bytes / (end_ts - start_ts) / 1e9` per request | +| `lmcache_mp.l0_l1_load_throughput` | `lmcache_mp_l0_l1_load_throughput_GBs` | Histogram | `MP_RETRIEVE_START` → `MP_RETRIEVE_END` | `total_bytes / (end_ts - start_ts) / 1e9` per request | **What it answers:** What GPU↔CPU throughput is each vLLM worker actually achieving for KV store/load? Does it match the theoretical PCIe bandwidth? @@ -312,8 +282,8 @@ registered adapter type (e.g. `"fs"`, `"nixl_store"`, `"mooncake_store"`) | OTel metric name | Prometheus name | Type | Source event | Calculation | |---|---|---|---|---| -| `lmcache_mp.l2_store_throughput_gbs` | `lmcache_mp_l2_store_throughput_gbs` | Histogram | `L2_STORE_SUBMITTED` → `L2_STORE_COMPLETED` | `total_bytes / (completed_ts - submitted_ts) / 1e9` per task | -| `lmcache_mp.l2_load_throughput_gbs` | `lmcache_mp_l2_load_throughput_gbs` | Histogram | `L2_LOAD_TASK_SUBMITTED` → `L2_LOAD_TASK_COMPLETED` | `total_bytes / (completed_ts - submitted_ts) / 1e9` per (request, adapter) pair | +| `lmcache_mp.l2_store_throughput` | `lmcache_mp_l2_store_throughput_GBs` | Histogram | `L2_STORE_SUBMITTED` → `L2_STORE_COMPLETED` | `total_bytes / (completed_ts - submitted_ts) / 1e9` per task | +| `lmcache_mp.l2_load_throughput` | `lmcache_mp_l2_load_throughput_GBs` | Histogram | `L2_LOAD_TASK_SUBMITTED` → `L2_LOAD_TASK_COMPLETED` | `total_bytes / (completed_ts - submitted_ts) / 1e9` per (request, adapter) pair | **What it answers:** What end-to-end throughput is each L2 adapter delivering? Which backends are keeping up with demand, and which are @@ -342,10 +312,12 @@ scrape time with `metric_relabel_configs` if storage cost matters). ## EventBus Self-Monitoring -Health metrics for the EventBus itself, registered by -`EventBusSelfMetricsSubscriber`. Unlike the other metrics subscribers, -these are not driven by events — they observe bus state directly via the -`EventBus` accessors and report on every OTel scrape. +Health metrics for the EventBus itself. The two gauges are registered +inside `EventBus.__init__` via `register_gauge`; the two observable +counters are registered by `EventBusSelfMetricsSubscriber`. Unlike the +other metrics subscribers, these are not driven by events — they observe +bus state directly via the `EventBus` accessors and report on every OTel +scrape. | OTel metric name | Prometheus name | Type | Source | Calculation | |---|---|---|---|---| diff --git a/docs/source/mp/observability.rst b/docs/source/mp/observability.rst index b79cc17abff..5799b87e298 100644 --- a/docs/source/mp/observability.rst +++ b/docs/source/mp/observability.rst @@ -112,7 +112,7 @@ collector. All metrics use the ``lmcache_mp.`` prefix (multiprocess). On Prometheus, dots are converted to underscores and counters get a ``_total`` suffix -(e.g. ``lmcache_mp_l1_read_keys_total``). +(e.g. ``lmcache_mp_l1_read_chunks_total``). .. _mp-observability-resource: @@ -140,35 +140,6 @@ and propagate to every exported datapoint via OTLP. On Prometheus, SDK resource attributes surface on the ``target_info`` series rather than on each time-series — this is standard OTel behavior. -StorageManager Metrics -~~~~~~~~~~~~~~~~~~~~~~ - -.. list-table:: - :header-rows: 1 - :widths: 40 15 45 - - * - Metric - - Type - - Description - * - ``lmcache_mp.sm_read_requests`` - - Counter - - Number of read (prefetch) requests received by the StorageManager. - * - ``lmcache_mp.sm_read_succeed_keys`` - - Counter - - Number of keys successfully read from LMCache. - * - ``lmcache_mp.sm_read_failed_keys`` - - Counter - - Number of keys that failed to read. - * - ``lmcache_mp.sm_write_requests`` - - Counter - - Number of write (reserve) requests. - * - ``lmcache_mp.sm_write_succeed_keys`` - - Counter - - Number of keys successfully reserved for write. - * - ``lmcache_mp.sm_write_failed_keys`` - - Counter - - Number of keys that failed to reserve (OOM, write conflict). - L1 Metrics ~~~~~~~~~~ @@ -179,15 +150,15 @@ L1 Metrics * - Metric - Type - Description - * - ``lmcache_mp.l1_read_keys`` + * - ``lmcache_mp.l1_read`` - Counter - - Number of keys read from L1. - * - ``lmcache_mp.l1_write_keys`` + - Number of chunks read from L1. + * - ``lmcache_mp.l1_write`` - Counter - - Number of keys written to L1. - * - ``lmcache_mp.l1_evicted_keys`` + - Number of chunks written to L1. + * - ``lmcache_mp.l1_evicted`` - Counter - - Number of keys evicted by the EvictionController. + - Number of chunks evicted by the EvictionController. * - ``lmcache_mp.l1_eviction_loop_ticks`` - Counter - L1 eviction-loop iterations (every cycle, regardless of whether @@ -216,16 +187,16 @@ memory overhead. * - Metric - Type - Description - * - ``lmcache_mp.l1_chunk_lifetime_seconds`` + * - ``lmcache_mp.l1_chunk_lifetime`` - Histogram - Time from allocation to eviction per sampled chunk. - * - ``lmcache_mp.l1_chunk_idle_before_evict_seconds`` + * - ``lmcache_mp.l1_chunk_idle_before_evict`` - Histogram - Time from last access to eviction per sampled chunk. - * - ``lmcache_mp.l1_chunk_reuse_gap_seconds`` + * - ``lmcache_mp.l1_chunk_reuse_gap`` - Histogram - Time gap between consecutive touches (read or write) of the same chunk. - * - ``lmcache_mp.l1_chunk_evict_reuse_gap_seconds`` + * - ``lmcache_mp.l1_chunk_evict_reuse_gap`` - Histogram - Time from eviction to next reuse (capped at 300 s). @@ -251,12 +222,12 @@ for chunks that pass the (deterministic, hash-based) sampling gate. * - Metric - Type - Description - * - ``lmcache_mp.real_reuse_gap_seconds`` + * - ``lmcache_mp.real_reuse_gap`` - Histogram (tag: ``cache_salt``) - Time gap between a chunk's last access (read or write) and its next read. Captures storage cost — how long a stored chunk sat between accesses. Emitted only on read events. - * - ``lmcache_mp.real_reuse_gap_chunks`` + * - ``lmcache_mp.real_reuse_gap_objects`` - Histogram (tag: ``cache_salt``) - Per-``cache_salt`` access-counter gap between two reads of the same chunk. Captures storage volume — how many chunk-accesses @@ -273,49 +244,43 @@ L2 Metrics * - Metric - Type - Description - * - ``lmcache_mp.l2_store_tasks`` + * - ``lmcache_mp.l2_store_submitted`` - Counter - - Number of L2 store tasks submitted. - * - ``lmcache_mp.l2_store_keys`` + - Number of L2 store requests submitted. + * - ``lmcache_mp.l2_store_submitted_objects`` - Counter - - Number of keys submitted for L2 store. + - Number of chunks submitted for L2 store. * - ``lmcache_mp.l2_store_completed`` - Counter (attr: ``l2_name``) - - Number of L2 store tasks completed, labeled by adapter type. - * - ``lmcache_mp.l2_store_succeeded_keys`` + - Number of L2 store requests completed, labeled by adapter type. + * - ``lmcache_mp.l2_store_completed_objects`` - Counter - - Number of keys successfully stored to L2. - * - ``lmcache_mp.l2_store_failed_keys`` - - Counter - - Number of keys that failed to store to L2. - * - ``lmcache_mp.l2_prefetch_lookups`` + - Number of chunks successfully stored to L2. + * - ``lmcache_mp.l2_prefetch_lookup`` - Counter - Number of L2 prefetch lookup requests. - * - ``lmcache_mp.l2_prefetch_lookup_keys`` - - Counter - - Number of keys submitted for L2 prefetch lookup. - * - ``lmcache_mp.l2_prefetch_hit_keys`` + * - ``lmcache_mp.l2_prefetch_lookup_objects`` - Counter - - Number of prefix keys found in L2 lookup. - * - ``lmcache_mp.l2_prefetch_load_tasks`` + - Number of chunks submitted for L2 prefetch lookup. + * - ``lmcache_mp.l2_prefetch_hit`` - Counter - - Number of L2 prefetch load tasks submitted. - * - ``lmcache_mp.l2_prefetch_load_keys`` + - Number of prefix chunks found in L2 lookup. + * - ``lmcache_mp.l2_prefetch_load_submitted`` - Counter - - Number of keys submitted for L2 load. - * - ``lmcache_mp.l2_prefetch_loaded_keys`` + - Number of L2 prefetch load requests submitted. + * - ``lmcache_mp.l2_prefetch_load_submitted_objects`` - Counter - - Number of keys successfully loaded from L2. - * - ``lmcache_mp.l2_prefetch_failed_keys`` + - Number of chunks submitted for L2 load. + * - ``lmcache_mp.l2_prefetch_load_completed`` - Counter - - Number of keys that failed to load from L2. + - Number of chunks successfully loaded from L2. * - ``lmcache_mp.l2_load_completed`` - Counter (attr: ``l2_name``) - - Number of per-adapter L2 load tasks completed, labeled by adapter type. + - Number of per-adapter L2 load requests completed, labeled by adapter type. The ``l2_name``-labeled counters (``l2_store_completed`` and ``l2_load_completed``) exist so dashboards can compute per-backend IOPS on -demand via ``rate(lmcache_mp_l2_store_completed_total{l2_name="..."}[1m])`` +demand via ``rate(lmcache_mp_l2_store_completed_requests_total{l2_name="..."}[1m])`` (and the equivalent for loads). No separate ``*_iops`` metric is exported; keeping the raw counter lets dashboard users pick their own window. @@ -353,7 +318,7 @@ Prometheus ``/metrics`` endpoint. stay near zero in healthy operation. * - ``lmcache_mp.l2_prefetch_failure`` - Counter - - Keys that L2 reported present at lookup but failed to land in L1. + - Chunks that L2 reported present at lookup but failed to land in L1. Tagged by ``reason`` ∈ {``l1_oom``, ``not_found``} plus ``model_name``. ``l1_oom`` means L1 had no room to receive the prefetched object; ``not_found`` means the adapter returned no @@ -382,11 +347,11 @@ intentionally excluded — it is vLLM-owned and not observable from LMCache. * - Metric - Type - Description - * - ``lmcache_mp.lookup_requested_tokens`` + * - ``lmcache_mp.lookup_requested`` - Counter (attrs: ``model_name``, ``cache_salt``) - Total tokens submitted for lookup (denominator of the L1+L2 token-level hit rate). Only chunk-aligned tokens are counted. - * - ``lmcache_mp.lookup_hit_tokens`` + * - ``lmcache_mp.lookup_hit`` - Counter (attrs: ``model_name``, ``cache_salt``) - Total tokens found in L1 or L2 during lookup (numerator of the L1+L2 token-level hit rate). Counts the contiguous prefix hit only. @@ -434,13 +399,13 @@ in Prometheus (e.g. * - Metric - Type - Description - * - ``lmcache_mp.l0_block_lifetime_seconds`` + * - ``lmcache_mp.l0_block_lifetime`` - Histogram - Time from allocation to eviction per sampled GPU block. - * - ``lmcache_mp.l0_block_idle_before_evict_seconds`` + * - ``lmcache_mp.l0_block_idle_before_evict`` - Histogram - Time from last access to eviction per sampled GPU block. - * - ``lmcache_mp.l0_block_reuse_gap_seconds`` + * - ``lmcache_mp.l0_block_reuse_gap`` - Histogram - Time gaps between consecutive accesses of the same GPU block. @@ -459,7 +424,7 @@ All throughput histograms are emitted with ``engine_id`` (vLLM worker instance id), ``device`` (e.g. ``"cuda:3"``), and ``model_name`` OTel attributes, enabling per-worker, per-device, and per-model slicing in Prometheus (e.g. -``lmcache_mp_l0_l1_store_throughput_gbs{engine_id="0",device="cuda:3",model_name="meta-llama/Llama-3.1-8B"}``). +``lmcache_mp_l0_l1_store_throughput_GB_per_second{engine_id="0",device="cuda:3",model_name="meta-llama/Llama-3.1-8B"}``). .. list-table:: :header-rows: 1 @@ -468,23 +433,23 @@ Prometheus (e.g. * - Metric - Type - Description - * - ``lmcache_mp.l0_l1_store_throughput_gbs`` + * - ``lmcache_mp.l0_l1_store_throughput`` - Histogram - GPU→CPU (L0→L1) store throughput in GB/s per request. - * - ``lmcache_mp.l0_l1_load_throughput_gbs`` + * - ``lmcache_mp.l0_l1_load_throughput`` - Histogram - CPU→GPU (L1→L0) load throughput in GB/s per request. L1 ↔ L2 Throughput Histograms ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Per-task throughput of L1↔L2 transfers via +Per-request throughput of L1↔L2 transfers via ``L2ThroughputSubscriber``. The store path correlates ``L2_STORE_SUBMITTED`` → ``L2_STORE_COMPLETED`` by ``(adapter_index, task_id)``. The load path correlates the per-adapter ``L2_LOAD_TASK_SUBMITTED`` → ``L2_LOAD_TASK_COMPLETED`` events by ``(request_id, adapter_index)``; the request-level -``L2_PREFETCH_LOAD_*`` events used by the key-count counters aggregate +``L2_PREFETCH_LOAD_*`` events used by the chunk-count counters aggregate across adapters and cannot be attributed to a specific ``l2_name``. Timestamps span **submit → complete**, so the duration includes adapter @@ -496,7 +461,7 @@ need pure copy-time throughput. All L1↔L2 throughput histograms carry a single ``l2_name`` OTel attribute — the registered adapter type (e.g. ``"fs"``, ``"nixl_store"``, ``"mooncake_store"``) — enabling per-backend slicing in Prometheus (e.g. -``lmcache_mp_l2_store_throughput_gbs{l2_name="nixl_store"}``). +``lmcache_mp_l2_store_throughput_GB_per_second{l2_name="nixl_store"}``). .. list-table:: :header-rows: 1 @@ -505,10 +470,10 @@ attribute — the registered adapter type (e.g. ``"fs"``, ``"nixl_store"``, * - Metric - Type - Description - * - ``lmcache_mp.l2_store_throughput_gbs`` + * - ``lmcache_mp.l2_store_throughput`` - Histogram - - L1→L2 store throughput in GB/s per task. - * - ``lmcache_mp.l2_load_throughput_gbs`` + - L1→L2 store throughput in GB/s per request. + * - ``lmcache_mp.l2_load_throughput`` - Histogram - L2→L1 load throughput in GB/s per (request, adapter) pair. diff --git a/examples/observability/grafana/provisioning/dashboards/lmcache.json b/examples/observability/grafana/provisioning/dashboards/lmcache.json index 59d2258e871..0f7edb448f5 100644 --- a/examples/observability/grafana/provisioning/dashboards/lmcache.json +++ b/examples/observability/grafana/provisioning/dashboards/lmcache.json @@ -120,19 +120,19 @@ "targets": [ { "editorMode": "code", - "expr": "sum by (service_instance_id) (rate(lmcache_mp_l1_allocation_failure_total[$__rate_interval])) or vector(0)", + "expr": "sum by (service_instance_id) (rate(lmcache_mp_l1_allocation_failure_chunks_total[$__rate_interval])) or vector(0)", "legendFormat": "L1 alloc failure [{{service_instance_id}}]", "refId": "A" }, { "editorMode": "code", - "expr": "sum by (service_instance_id) (rate(lmcache_mp_l1_read_failure_total[$__rate_interval])) or vector(0)", + "expr": "sum by (service_instance_id) (rate(lmcache_mp_l1_read_failure_chunks_total[$__rate_interval])) or vector(0)", "legendFormat": "L1 read failure [{{service_instance_id}}]", "refId": "B" }, { "editorMode": "code", - "expr": "sum by (service_instance_id) (rate(lmcache_mp_l2_prefetch_failure_total[$__rate_interval])) or vector(0)", + "expr": "sum by (service_instance_id) (rate(lmcache_mp_l2_prefetch_failure_chunks_total[$__rate_interval])) or vector(0)", "legendFormat": "L2 prefetch failure [{{service_instance_id}}]", "refId": "C" } @@ -464,19 +464,19 @@ "targets": [ { "editorMode": "code", - "expr": "rate(lmcache_mp_l1_read_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l1_read_chunks_total[$__rate_interval])", "legendFormat": "L1 reads/s [{{service_instance_id}}]", "refId": "A" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l1_write_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l1_write_chunks_total[$__rate_interval])", "legendFormat": "L1 writes/s [{{service_instance_id}}]", "refId": "B" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l1_evicted_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l1_evicted_chunks_total[$__rate_interval])", "legendFormat": "L1 evictions/s [{{service_instance_id}}]", "refId": "C" } @@ -584,33 +584,27 @@ "targets": [ { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_store_tasks_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_store_submitted_requests_total[$__rate_interval])", "legendFormat": "tasks submitted/s", "refId": "A" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_store_completed_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_store_completed_requests_total[$__rate_interval])", "legendFormat": "tasks completed/s ({{l2_name}})", "refId": "B" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_store_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_store_submitted_objects_chunks_total[$__rate_interval])", "legendFormat": "keys submitted/s", "refId": "C" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_store_succeeded_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_store_completed_objects_chunks_total[$__rate_interval])", "legendFormat": "succeeded keys/s", "refId": "D" - }, - { - "editorMode": "code", - "expr": "rate(lmcache_mp_l2_store_failed_keys_total[$__rate_interval])", - "legendFormat": "failed keys/s", - "refId": "E" } ], "title": "L2 Store Pipeline", @@ -716,43 +710,43 @@ "targets": [ { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_prefetch_lookups_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_prefetch_lookup_requests_total[$__rate_interval])", "legendFormat": "lookups/s", "refId": "A" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_prefetch_lookup_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_prefetch_lookup_objects_chunks_total[$__rate_interval])", "legendFormat": "lookup keys/s", "refId": "B" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_prefetch_hit_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_prefetch_hit_chunks_total[$__rate_interval])", "legendFormat": "hit keys/s", "refId": "C" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_prefetch_load_tasks_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_prefetch_load_submitted_requests_total[$__rate_interval])", "legendFormat": "load tasks/s", "refId": "D" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_prefetch_load_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_prefetch_load_submitted_objects_chunks_total[$__rate_interval])", "legendFormat": "load keys submitted/s", "refId": "E" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_prefetch_loaded_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_prefetch_load_completed_chunks_total[$__rate_interval])", "legendFormat": "loaded keys/s", "refId": "F" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_prefetch_failed_keys_total[$__rate_interval])", + "expr": "sum(rate(lmcache_mp_l2_prefetch_failure_chunks_total[$__rate_interval]))", "legendFormat": "failed keys/s", "refId": "G" } @@ -1111,13 +1105,13 @@ "targets": [ { "editorMode": "code", - "expr": "rate(lmcache_mp_l0_l1_store_throughput_gbs_GB_per_second_sum[$__rate_interval]) / rate(lmcache_mp_l0_l1_store_throughput_gbs_GB_per_second_count[$__rate_interval])", + "expr": "rate(lmcache_mp_l0_l1_store_throughput_GB_per_second_sum[$__rate_interval]) / rate(lmcache_mp_l0_l1_store_throughput_GB_per_second_count[$__rate_interval])", "legendFormat": "L0\u2192L1 store ({{device}})", "refId": "A" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l0_l1_load_throughput_gbs_GB_per_second_sum[$__rate_interval]) / rate(lmcache_mp_l0_l1_load_throughput_gbs_GB_per_second_count[$__rate_interval])", + "expr": "rate(lmcache_mp_l0_l1_load_throughput_GB_per_second_sum[$__rate_interval]) / rate(lmcache_mp_l0_l1_load_throughput_GB_per_second_count[$__rate_interval])", "legendFormat": "L1\u2192L0 load ({{device}})", "refId": "B" } @@ -1210,13 +1204,13 @@ "targets": [ { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_store_throughput_gbs_GB_per_second_sum[$__rate_interval]) / rate(lmcache_mp_l2_store_throughput_gbs_GB_per_second_count[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_store_throughput_GB_per_second_sum[$__rate_interval]) / rate(lmcache_mp_l2_store_throughput_GB_per_second_count[$__rate_interval])", "legendFormat": "L1\u2192L2 store ({{l2_name}})", "refId": "A" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_load_throughput_gbs_GB_per_second_sum[$__rate_interval]) / rate(lmcache_mp_l2_load_throughput_gbs_GB_per_second_count[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_load_throughput_GB_per_second_sum[$__rate_interval]) / rate(lmcache_mp_l2_load_throughput_GB_per_second_count[$__rate_interval])", "legendFormat": "L2\u2192L1 load ({{l2_name}})", "refId": "B" } @@ -1516,13 +1510,13 @@ "targets": [ { "editorMode": "code", - "expr": "rate(lmcache_mp_l1_read_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l1_read_chunks_total[$__rate_interval])", "legendFormat": "L1 read IOPS [{{service_instance_id}}]", "refId": "A" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l1_write_keys_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l1_write_chunks_total[$__rate_interval])", "legendFormat": "L1 write IOPS [{{service_instance_id}}]", "refId": "B" } @@ -1615,13 +1609,13 @@ "targets": [ { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_store_completed_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_store_completed_requests_total[$__rate_interval])", "legendFormat": "L2 store IOPS ({{l2_name}})", "refId": "A" }, { "editorMode": "code", - "expr": "rate(lmcache_mp_l2_load_completed_total[$__rate_interval])", + "expr": "rate(lmcache_mp_l2_load_completed_requests_total[$__rate_interval])", "legendFormat": "L2 load IOPS ({{l2_name}})", "refId": "B" } @@ -2068,7 +2062,7 @@ }, { "editorMode": "code", - "expr": "rate(lmcache_mp_real_reuse_gap_chunks_sum[$__rate_interval]) / rate(lmcache_mp_real_reuse_gap_chunks_count[$__rate_interval])", + "expr": "rate(lmcache_mp_real_reuse_gap_objects_chunks_sum[$__rate_interval]) / rate(lmcache_mp_real_reuse_gap_objects_chunks_count[$__rate_interval])", "legendFormat": "avg reuse gap chunks ({{cache_salt}})", "refId": "B" } diff --git a/lmcache/v1/mp_observability/config.py b/lmcache/v1/mp_observability/config.py index e4bbe54c1e3..275d17e4926 100644 --- a/lmcache/v1/mp_observability/config.py +++ b/lmcache/v1/mp_observability/config.py @@ -349,7 +349,6 @@ def init_observability( L2ThroughputSubscriber, LookupMetricsSubscriber, SMLifecycleSubscriber, - SMMetricsSubscriber, ) sample_rate = obs_config.metrics_sample_rate @@ -363,7 +362,6 @@ def init_observability( bus.register_subscriber(L2FailureMetricsSubscriber()) bus.register_subscriber(L2ThroughputSubscriber()) bus.register_subscriber(LookupMetricsSubscriber()) - bus.register_subscriber(SMMetricsSubscriber()) bus.register_subscriber(SMLifecycleSubscriber(sample_rate=sample_rate)) bus.register_subscriber(BlendMetricsSubscriber()) bus.register_subscriber(EngineMetricsSubscriber()) diff --git a/lmcache/v1/mp_observability/event_bus.py b/lmcache/v1/mp_observability/event_bus.py index 864ad309b18..2152dc5c1b8 100644 --- a/lmcache/v1/mp_observability/event_bus.py +++ b/lmcache/v1/mp_observability/event_bus.py @@ -17,6 +17,7 @@ # First Party from lmcache.logging import init_logger from lmcache.v1.mp_observability.event import Event, EventType +from lmcache.v1.mp_observability.otel_init import register_gauge try: # Third Party @@ -118,6 +119,27 @@ def __init__(self, config: EventBusConfig | None = None) -> None: self._last_discard_warning: float = 0.0 self._subscriber_exception_counts: dict[str, int] = {} + self._register_self_gauges() + + def _register_self_gauges(self) -> None: + """Register the two self-monitoring gauges via ``register_gauge``.""" + register_gauge( + "lmcache.event_bus", + "lmcache_mp.event_bus.queue_depth", + "Events currently queued in the EventBus.", + self.queue_depth, + ) + register_gauge( + "lmcache.event_bus", + "lmcache_mp.event_bus.drain_lag_seconds", + ( + "Seconds since the oldest queued event was published; 0.0 " + "when empty. Rising values mean the drain thread is " + "falling behind." + ), + self.oldest_event_lag_seconds, + ) + # -- Public API -------------------------------------------------------- def subscribe(self, event_type: EventType, callback: EventCallback) -> None: diff --git a/lmcache/v1/mp_observability/subscribers/__init__.py b/lmcache/v1/mp_observability/subscribers/__init__.py index 5f6247f806f..edcc6d0132e 100644 --- a/lmcache/v1/mp_observability/subscribers/__init__.py +++ b/lmcache/v1/mp_observability/subscribers/__init__.py @@ -10,7 +10,6 @@ L0LifecycleSubscriber, L1LifecycleSubscriber, L1MetricsSubscriber, - SMMetricsSubscriber, ) from lmcache.v1.mp_observability.subscribers.tracing import ( MPServerTracingSubscriber, @@ -24,5 +23,4 @@ "MPServerLoggingSubscriber", "MPServerTracingSubscriber", "SMLoggingSubscriber", - "SMMetricsSubscriber", ] diff --git a/lmcache/v1/mp_observability/subscribers/metrics/__init__.py b/lmcache/v1/mp_observability/subscribers/metrics/__init__.py index 005c0f41300..94121dce46c 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/__init__.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/__init__.py @@ -36,7 +36,6 @@ from lmcache.v1.mp_observability.subscribers.metrics.lookup import ( LookupMetricsSubscriber, ) -from lmcache.v1.mp_observability.subscribers.metrics.sm import SMMetricsSubscriber from lmcache.v1.mp_observability.subscribers.metrics.sm_lifecycle import ( SMLifecycleSubscriber, ) @@ -56,5 +55,4 @@ "L2ThroughputSubscriber", "LookupMetricsSubscriber", "SMLifecycleSubscriber", - "SMMetricsSubscriber", ] diff --git a/lmcache/v1/mp_observability/subscribers/metrics/event_bus.py b/lmcache/v1/mp_observability/subscribers/metrics/event_bus.py index bb7a86e7e68..42b9f0dbe96 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/event_bus.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/event_bus.py @@ -2,10 +2,8 @@ """EventBus self-metrics subscriber. -Surfaces four metrics describing the EventBus's own health (per #3108): +Surfaces two metrics describing the EventBus's own health (per #3108): -- ``lmcache_mp.event_bus.queue_depth`` (gauge) -- ``lmcache_mp.event_bus.drain_lag_seconds`` (gauge) - ``lmcache_mp.event_bus.dropped_events_total`` (observable counter) - ``lmcache_mp.event_bus.subscriber_exceptions`` (observable counter, attr ``subscriber_name``) @@ -37,22 +35,6 @@ class EventBusSelfMetricsSubscriber(EventSubscriber): def __init__(self, bus: EventBus) -> None: meter = metrics.get_meter("lmcache.event_bus") - meter.create_observable_gauge( - "lmcache_mp.event_bus.queue_depth", - callbacks=[lambda _o: [metrics.Observation(bus.queue_depth())]], - description="Events currently queued in the EventBus.", - ) - meter.create_observable_gauge( - "lmcache_mp.event_bus.drain_lag_seconds", - callbacks=[ - lambda _o: [metrics.Observation(bus.oldest_event_lag_seconds())] - ], - description=( - "Seconds since the oldest queued event was published; " - "0.0 when empty. Rising values mean the drain thread is " - "falling behind." - ), - ) meter.create_observable_counter( "lmcache_mp.event_bus.dropped_events_total", callbacks=[lambda _o: [metrics.Observation(bus.dropped_events_count())]], diff --git a/lmcache/v1/mp_observability/subscribers/metrics/l0_l1_throughput.py b/lmcache/v1/mp_observability/subscribers/metrics/l0_l1_throughput.py index fe45c41f740..ded9ebb733a 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/l0_l1_throughput.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/l0_l1_throughput.py @@ -4,8 +4,8 @@ Emits two OTel histograms in GB/s, labeled by ``engine_id``, ``device``, and ``model_name``: - - ``lmcache_mp.l0_l1_store_throughput_gbs`` — GPU→CPU (L0→L1) store - - ``lmcache_mp.l0_l1_load_throughput_gbs`` — CPU→GPU (L1→L0) load + - ``lmcache_mp.l0_l1_store_throughput`` — GPU→CPU (L0→L1) store + - ``lmcache_mp.l0_l1_load_throughput`` — CPU→GPU (L1→L0) load Implementation: - Correlates ``MP_STORE_START`` → ``MP_STORE_END`` and @@ -45,7 +45,7 @@ def __init__(self) -> None: meter = metrics.get_meter("lmcache_mp.perf") self._store_hist = meter.create_histogram( - "lmcache_mp.l0_l1_store_throughput_gbs", + "lmcache_mp.l0_l1_store_throughput", description=( "Histogram of L0→L1 (GPU→CPU) store throughput in GB/s, " "measured per request as total_bytes / (end_ts - start_ts) " @@ -54,7 +54,7 @@ def __init__(self) -> None: unit="GB/s", ) self._load_hist = meter.create_histogram( - "lmcache_mp.l0_l1_load_throughput_gbs", + "lmcache_mp.l0_l1_load_throughput", description=( "Histogram of L1→L0 (CPU→GPU) load throughput in GB/s, " "measured per request as total_bytes / (end_ts - start_ts) " diff --git a/lmcache/v1/mp_observability/subscribers/metrics/l0_lifecycle.py b/lmcache/v1/mp_observability/subscribers/metrics/l0_lifecycle.py index d2272babf78..4024e296cac 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/l0_lifecycle.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/l0_lifecycle.py @@ -76,9 +76,9 @@ class L0LifecycleSubscriber(EventSubscriber): """Tracks GPU (L0) KV cache block lifecycle via shadow monitoring. Metrics (all histograms, in seconds): - - ``lmcache_mp.l0_block_lifetime_seconds`` - - ``lmcache_mp.l0_block_idle_before_evict_seconds`` - - ``lmcache_mp.l0_block_reuse_gap_seconds`` + - ``lmcache_mp.l0_block_lifetime`` + - ``lmcache_mp.l0_block_idle_before_evict`` + - ``lmcache_mp.l0_block_reuse_gap`` Parameters: sample_rate: Fraction of blocks to track (0, 1.0]. Default 0.01 @@ -100,7 +100,7 @@ def __init__(self, sample_rate: float = 0.01) -> None: meter = metrics.get_meter("lmcache.l0") self._lifetime_hist = meter.create_histogram( - "lmcache_mp.l0_block_lifetime_seconds", + "lmcache_mp.l0_block_lifetime", description=( "Histogram of GPU KV cache block lifetime from " "allocation to eviction (seconds)." @@ -108,14 +108,14 @@ def __init__(self, sample_rate: float = 0.01) -> None: unit="s", ) self._idle_hist = meter.create_histogram( - "lmcache_mp.l0_block_idle_before_evict_seconds", + "lmcache_mp.l0_block_idle_before_evict", description=( "Histogram of idle time before GPU KV cache block eviction (seconds)." ), unit="s", ) self._reuse_gap_hist = meter.create_histogram( - "lmcache_mp.l0_block_reuse_gap_seconds", + "lmcache_mp.l0_block_reuse_gap", description=( "Histogram of time gaps between consecutive GPU KV " "cache block accesses (seconds)." diff --git a/lmcache/v1/mp_observability/subscribers/metrics/l1.py b/lmcache/v1/mp_observability/subscribers/metrics/l1.py index a1db2668c64..dcdf8315e56 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/l1.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/l1.py @@ -17,24 +17,27 @@ class L1MetricsSubscriber(EventSubscriber): """Maintains OTel counters for L1Manager operations. Metric parity with the old ``L1ManagerStatsLogger``: - - ``lmcache_mp.l1_read_keys`` — keys read from L1 - - ``lmcache_mp.l1_write_keys`` — keys written to L1 - - ``lmcache_mp.l1_evicted_keys`` — keys evicted from L1 + - ``lmcache_mp.l1_read`` — chunks read from L1 + - ``lmcache_mp.l1_write`` — chunks written to L1 + - ``lmcache_mp.l1_evicted`` — chunks evicted from L1 """ def __init__(self) -> None: meter = metrics.get_meter("lmcache.l1") self._read_counter = meter.create_counter( - "lmcache_mp.l1_read_keys", - description="Total keys read from L1", + "lmcache_mp.l1_read", + description="Total chunks read from L1", + unit="chunks", ) self._write_counter = meter.create_counter( - "lmcache_mp.l1_write_keys", - description="Total keys written to L1", + "lmcache_mp.l1_write", + description="Total chunks written to L1", + unit="chunks", ) self._evicted_counter = meter.create_counter( - "lmcache_mp.l1_evicted_keys", - description="Total keys evicted from L1", + "lmcache_mp.l1_evicted", + description="Total chunks evicted from L1", + unit="chunks", ) def get_subscriptions(self) -> dict[EventType, EventCallback]: diff --git a/lmcache/v1/mp_observability/subscribers/metrics/l1_failures.py b/lmcache/v1/mp_observability/subscribers/metrics/l1_failures.py index 01644aa2f65..d4c8896b3b5 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/l1_failures.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/l1_failures.py @@ -44,6 +44,7 @@ def __init__(self) -> None: "reserve_write. Tagged by ``during`` = l1_store | l2_prefetch " "and ``model_name``." ), + unit="chunks", ) self._read_counter = meter.create_counter( "lmcache_mp.l1_read_failure", @@ -54,6 +55,7 @@ def __init__(self) -> None: "near zero in healthy operation; non-zero indicates a " "lookup/reserve race or unexpected eviction." ), + unit="chunks", ) def get_subscriptions(self) -> dict[EventType, EventCallback]: diff --git a/lmcache/v1/mp_observability/subscribers/metrics/l1_lifecycle.py b/lmcache/v1/mp_observability/subscribers/metrics/l1_lifecycle.py index 57094c798de..dd87116acfb 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/l1_lifecycle.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/l1_lifecycle.py @@ -36,10 +36,10 @@ class L1LifecycleSubscriber(EventSubscriber): """Tracks L1 chunk lifecycle via shadow monitoring. Histograms (chunk lifecycle): - - ``lmcache_mp.l1_chunk_lifetime_seconds`` — allocation to eviction - - ``lmcache_mp.l1_chunk_idle_before_evict_seconds`` — last access to eviction - - ``lmcache_mp.l1_chunk_reuse_gap_seconds`` — gap between consecutive touches - - ``lmcache_mp.l1_chunk_evict_reuse_gap_seconds`` — eviction to + - ``lmcache_mp.l1_chunk_lifetime`` — allocation to eviction + - ``lmcache_mp.l1_chunk_idle_before_evict`` — last access to eviction + - ``lmcache_mp.l1_chunk_reuse_gap`` — gap between consecutive touches + - ``lmcache_mp.l1_chunk_evict_reuse_gap`` — eviction to next reuse (capped at ``max_evict_reuse_wait``) Parameters: @@ -64,19 +64,19 @@ def __init__( self._sample_threshold = int(sample_rate * self._sample_prime) meter = metrics.get_meter("lmcache.l1") self._lifetime_hist = meter.create_histogram( - "lmcache_mp.l1_chunk_lifetime_seconds", + "lmcache_mp.l1_chunk_lifetime", description=( "Histogram of L1 chunk lifetime from allocation to eviction (seconds)." ), unit="s", ) self._idle_hist = meter.create_histogram( - "lmcache_mp.l1_chunk_idle_before_evict_seconds", + "lmcache_mp.l1_chunk_idle_before_evict", description=("Histogram of idle time before L1 chunk eviction (seconds)."), unit="s", ) self._reuse_gap_hist = meter.create_histogram( - "lmcache_mp.l1_chunk_reuse_gap_seconds", + "lmcache_mp.l1_chunk_reuse_gap", description=( "Histogram of time gaps between consecutive " "touches (write or read) of the same L1 chunk (seconds)." @@ -84,7 +84,7 @@ def __init__( unit="s", ) self._evict_reuse_gap_hist = meter.create_histogram( - "lmcache_mp.l1_chunk_evict_reuse_gap_seconds", + "lmcache_mp.l1_chunk_evict_reuse_gap", description=( "Histogram of time from L1 chunk eviction to " "next reuse. Capped at max_evict_reuse_wait." diff --git a/lmcache/v1/mp_observability/subscribers/metrics/l2.py b/lmcache/v1/mp_observability/subscribers/metrics/l2.py index ca7dc9375eb..06dd1dfda5c 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/l2.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/l2.py @@ -30,20 +30,18 @@ class L2MetricsSubscriber(EventSubscriber): """Maintains OTel counters for L2 store and prefetch operations. Metrics: - - ``lmcache_mp.l2_store_tasks`` — store tasks submitted to L2 - - ``lmcache_mp.l2_store_keys`` — keys submitted for L2 store - - ``lmcache_mp.l2_store_completed`` — store tasks completed (attr: ``l2_name``) - - ``lmcache_mp.l2_store_succeeded_keys`` — keys successfully stored to L2 - - ``lmcache_mp.l2_store_failed_keys`` — keys that failed to store to L2 - - ``lmcache_mp.l2_load_completed`` — per-adapter load tasks completed + - ``lmcache_mp.l2_store_submitted`` — store requests submitted to L2 + - ``lmcache_mp.l2_store_submitted_objects`` — chunks submitted for L2 store + - ``lmcache_mp.l2_store_completed`` — store requests completed (attr: ``l2_name``) + - ``lmcache_mp.l2_store_completed_objects`` — chunks successfully stored to L2 + - ``lmcache_mp.l2_load_completed`` — per-adapter load tasks completed (attr: ``l2_name``) - - ``lmcache_mp.l2_prefetch_lookups`` — prefetch lookup requests - - ``lmcache_mp.l2_prefetch_lookup_keys`` — keys submitted for lookup - - ``lmcache_mp.l2_prefetch_hit_keys`` — prefix keys found in L2 - - ``lmcache_mp.l2_prefetch_load_tasks`` — load tasks submitted - - ``lmcache_mp.l2_prefetch_load_keys`` — keys submitted for load - - ``lmcache_mp.l2_prefetch_loaded_keys`` — keys successfully loaded from L2 - - ``lmcache_mp.l2_prefetch_failed_keys`` — keys that failed to load + - ``lmcache_mp.l2_prefetch_lookup`` — prefetch lookup requests + - ``lmcache_mp.l2_prefetch_lookup_objects`` — chunks submitted for lookup + - ``lmcache_mp.l2_prefetch_hit`` — prefix chunks found in L2 + - ``lmcache_mp.l2_prefetch_load_submitted`` — load tasks submitted + - ``lmcache_mp.l2_prefetch_load_submitted_objects`` — chunks submitted for load + - ``lmcache_mp.l2_prefetch_load_completed`` — chunks successfully loaded from L2 The ``l2_name``-labeled counters (``l2_store_completed``, ``l2_load_completed``) let dashboards compute per-backend IOPS via @@ -54,64 +52,67 @@ def __init__(self) -> None: meter = metrics.get_meter("lmcache.l2") # Store counters - self._store_tasks = meter.create_counter( - "lmcache_mp.l2_store_tasks", - description="Total L2 store tasks submitted", + self._store_submitted = meter.create_counter( + "lmcache_mp.l2_store_submitted", + description="Total L2 store requests submitted", + unit="requests", ) - self._store_keys = meter.create_counter( - "lmcache_mp.l2_store_keys", - description="Total keys submitted for L2 store", + self._store_submitted_objects = meter.create_counter( + "lmcache_mp.l2_store_submitted_objects", + description="Total chunks submitted for L2 store", + unit="chunks", ) self._store_completed = meter.create_counter( "lmcache_mp.l2_store_completed", - description="Total L2 store tasks completed", + description="Total L2 store requests completed", + unit="requests", ) - self._store_succeeded_keys = meter.create_counter( - "lmcache_mp.l2_store_succeeded_keys", - description="Total keys successfully stored to L2", - ) - self._store_failed_keys = meter.create_counter( - "lmcache_mp.l2_store_failed_keys", - description="Total keys that failed to store to L2", + self._store_completed_objects = meter.create_counter( + "lmcache_mp.l2_store_completed_objects", + description="Total chunks successfully stored to L2", + unit="chunks", ) # Per-adapter load task counter (for IOPS via rate()). - # The existing prefetch counters count KEYS and aggregate across - # adapters at the request level; this one counts TASKS and is - # labeled by ``l2_name`` so dashboards can slice per backend. + # Labeled by ``l2_name`` so dashboards can slice per backend. self._load_completed = meter.create_counter( "lmcache_mp.l2_load_completed", description="Total L2 load tasks completed (per-adapter)", + unit="requests", ) - # Prefetch counters - self._prefetch_lookups = meter.create_counter( - "lmcache_mp.l2_prefetch_lookups", - description="Total L2 prefetch lookup requests", - ) - self._prefetch_lookup_keys = meter.create_counter( - "lmcache_mp.l2_prefetch_lookup_keys", - description="Total keys submitted for L2 prefetch lookup", + # Prefetch lookup counters + self._prefetch_lookup_submitted = meter.create_counter( + "lmcache_mp.l2_prefetch_lookup", + description="Total L2 prefetch lookup requests submitted", + unit="requests", ) - self._prefetch_hit_keys = meter.create_counter( - "lmcache_mp.l2_prefetch_hit_keys", - description="Total prefix keys found in L2 lookup", + self._prefetch_lookup_submitted_objects = meter.create_counter( + "lmcache_mp.l2_prefetch_lookup_objects", + description="Total chunks submitted for L2 prefetch lookup", + unit="chunks", ) - self._prefetch_load_tasks = meter.create_counter( - "lmcache_mp.l2_prefetch_load_tasks", - description="Total L2 prefetch load tasks submitted", + self._prefetch_lookup_hit = meter.create_counter( + "lmcache_mp.l2_prefetch_hit", + description="Total prefix chunks found in L2 lookup", + unit="chunks", ) - self._prefetch_load_keys = meter.create_counter( - "lmcache_mp.l2_prefetch_load_keys", - description="Total keys submitted for L2 load", + + # Prefetch load counters + self._prefetch_load_submitted = meter.create_counter( + "lmcache_mp.l2_prefetch_load_submitted", + description="Total L2 prefetch load requests submitted (per-adapter)", + unit="requests", ) - self._prefetch_loaded_keys = meter.create_counter( - "lmcache_mp.l2_prefetch_loaded_keys", - description="Total keys successfully loaded from L2", + self._prefetch_load_submitted_objects = meter.create_counter( + "lmcache_mp.l2_prefetch_load_submitted_objects", + description="Total chunks submitted for L2 load", + unit="chunks", ) - self._prefetch_failed_keys = meter.create_counter( - "lmcache_mp.l2_prefetch_failed_keys", - description="Total keys that failed to load from L2", + self._prefetch_load_completed = meter.create_counter( + "lmcache_mp.l2_prefetch_load_completed", + description="Total chunks successfully loaded from L2", + unit="chunks", ) def get_subscriptions(self) -> dict[EventType, EventCallback]: @@ -126,29 +127,27 @@ def get_subscriptions(self) -> dict[EventType, EventCallback]: } def _on_store_submitted(self, event: Event) -> None: - self._store_tasks.add(1) - self._store_keys.add(event.metadata["key_count"]) + self._store_submitted.add(1) + self._store_submitted_objects.add(event.metadata["key_count"]) def _on_store_completed(self, event: Event) -> None: attrs = _l2_name_attrs(event) self._store_completed.add(1, attributes=attrs) - self._store_succeeded_keys.add(event.metadata["succeeded_count"]) - self._store_failed_keys.add(event.metadata["failed_count"]) + self._store_completed_objects.add(event.metadata["succeeded_count"]) def _on_load_task_completed(self, event: Event) -> None: self._load_completed.add(1, attributes=_l2_name_attrs(event)) def _on_lookup_submitted(self, event: Event) -> None: - self._prefetch_lookups.add(1) - self._prefetch_lookup_keys.add(event.metadata["key_count"]) + self._prefetch_lookup_submitted.add(1) + self._prefetch_lookup_submitted_objects.add(event.metadata["key_count"]) def _on_lookup_completed(self, event: Event) -> None: - self._prefetch_hit_keys.add(event.metadata["prefix_hit_count"]) + self._prefetch_lookup_hit.add(event.metadata["prefix_hit_count"]) def _on_load_submitted(self, event: Event) -> None: - self._prefetch_load_tasks.add(event.metadata["adapter_count"]) - self._prefetch_load_keys.add(event.metadata["key_count"]) + self._prefetch_load_submitted.add(event.metadata["adapter_count"]) + self._prefetch_load_submitted_objects.add(event.metadata["key_count"]) def _on_load_completed(self, event: Event) -> None: - self._prefetch_loaded_keys.add(event.metadata["loaded_count"]) - self._prefetch_failed_keys.add(event.metadata["failed_count"]) + self._prefetch_load_completed.add(event.metadata["loaded_count"]) diff --git a/lmcache/v1/mp_observability/subscribers/metrics/l2_failures.py b/lmcache/v1/mp_observability/subscribers/metrics/l2_failures.py index 875d043befb..77f103a5708 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/l2_failures.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/l2_failures.py @@ -44,6 +44,7 @@ def __init__(self) -> None: "into L1. Tagged by ``reason`` = l1_oom | not_found and " "``model_name``." ), + unit="chunks", ) def get_subscriptions(self) -> dict[EventType, EventCallback]: diff --git a/lmcache/v1/mp_observability/subscribers/metrics/l2_throughput.py b/lmcache/v1/mp_observability/subscribers/metrics/l2_throughput.py index b7e2d14c4b4..ed415936e5f 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/l2_throughput.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/l2_throughput.py @@ -4,8 +4,8 @@ Emits two OTel histograms in GB/s, labeled by ``l2_name`` (the registered adapter type, e.g. ``"fs"``, ``"nixl_store"``): - - ``lmcache_mp.l2_store_throughput_gbs`` — L1→L2 store - - ``lmcache_mp.l2_load_throughput_gbs`` — L2→L1 load + - ``lmcache_mp.l2_store_throughput`` — L1→L2 store + - ``lmcache_mp.l2_load_throughput`` — L2→L1 load Implementation: - Store path correlates ``L2_STORE_SUBMITTED`` → ``L2_STORE_COMPLETED`` @@ -59,7 +59,7 @@ def __init__(self) -> None: meter = metrics.get_meter("lmcache_mp.perf") self._store_hist = meter.create_histogram( - "lmcache_mp.l2_store_throughput_gbs", + "lmcache_mp.l2_store_throughput", description=( "Histogram of L1->L2 store throughput in GB/s, measured " "per task as total_bytes / (completed_ts - " @@ -69,7 +69,7 @@ def __init__(self) -> None: unit="GB/s", ) self._load_hist = meter.create_histogram( - "lmcache_mp.l2_load_throughput_gbs", + "lmcache_mp.l2_load_throughput", description=( "Histogram of L2->L1 load throughput in GB/s, measured " "per (request, adapter) pair as total_bytes / " diff --git a/lmcache/v1/mp_observability/subscribers/metrics/lookup.py b/lmcache/v1/mp_observability/subscribers/metrics/lookup.py index 4857e4b5004..31bc7579ce8 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/lookup.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/lookup.py @@ -53,10 +53,10 @@ class LookupMetricsSubscriber(EventSubscriber): """Maintains OTel counters for L1+L2 token-level cache hit rate. Metrics (both labeled by ``model_name`` and ``cache_salt``): - - ``lmcache_mp.lookup_requested_tokens`` — tokens submitted for lookup + - ``lmcache_mp.lookup_requested`` — tokens submitted for lookup (denominator). Counts only the chunk-aligned portion; sub-chunk trailing tokens are excluded because they cannot hit by design. - - ``lmcache_mp.lookup_hit_tokens`` — tokens found in L1+L2 during the + - ``lmcache_mp.lookup_hit`` — tokens found in L1+L2 during the lookup (numerator). Counts the contiguous prefix hit only. """ @@ -64,7 +64,7 @@ def __init__(self) -> None: meter = metrics.get_meter("lmcache.lookup") self._requested_tokens = meter.create_counter( - "lmcache_mp.lookup_requested_tokens", + "lmcache_mp.lookup_requested", description=( "Total tokens submitted for lookup (denominator of the " "L1+L2 token-level hit rate). Only chunk-aligned tokens " @@ -73,7 +73,7 @@ def __init__(self) -> None: unit="tokens", ) self._hit_tokens = meter.create_counter( - "lmcache_mp.lookup_hit_tokens", + "lmcache_mp.lookup_hit", description=( "Total tokens found in L1+L2 during lookup (numerator of " "the L1+L2 token-level hit rate). Counts the contiguous " diff --git a/lmcache/v1/mp_observability/subscribers/metrics/sm.py b/lmcache/v1/mp_observability/subscribers/metrics/sm.py deleted file mode 100644 index 01dfa52e01f..00000000000 --- a/lmcache/v1/mp_observability/subscribers/metrics/sm.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -"""StorageManager metrics subscriber — OTel counters for SM events.""" - -# Future -from __future__ import annotations - -# Third Party -from opentelemetry import metrics - -# First Party -from lmcache.v1.mp_observability.event import Event, EventType -from lmcache.v1.mp_observability.event_bus import EventCallback, EventSubscriber - - -class SMMetricsSubscriber(EventSubscriber): - """Maintains OTel counters for StorageManager operations. - - Metric parity with the old ``StorageManagerStatsLogger``: - - ``lmcache_mp.sm_read_requests`` — SM read (prefetch) requests - - ``lmcache_mp.sm_read_succeed_keys`` — keys that were cache hits - - ``lmcache_mp.sm_read_failed_keys`` — keys that were cache misses - - ``lmcache_mp.sm_write_requests`` — SM write (reserve) requests - - ``lmcache_mp.sm_write_succeed_keys`` — keys successfully allocated - - ``lmcache_mp.sm_write_failed_keys`` — keys that failed allocation - """ - - def __init__(self) -> None: - meter = metrics.get_meter("lmcache.sm") - self._read_requests = meter.create_counter( - "lmcache_mp.sm_read_requests", - description="Total StorageManager read (prefetch) requests", - ) - self._read_succeed = meter.create_counter( - "lmcache_mp.sm_read_succeed_keys", - description="Total keys that were cache hits in SM read", - ) - self._read_failed = meter.create_counter( - "lmcache_mp.sm_read_failed_keys", - description="Total keys that were cache misses in SM read", - ) - self._write_requests = meter.create_counter( - "lmcache_mp.sm_write_requests", - description="Total StorageManager write (reserve) requests", - ) - self._write_succeed = meter.create_counter( - "lmcache_mp.sm_write_succeed_keys", - description="Total keys successfully allocated for write in SM", - ) - self._write_failed = meter.create_counter( - "lmcache_mp.sm_write_failed_keys", - description="Total keys that failed allocation for write in SM", - ) - - def get_subscriptions(self) -> dict[EventType, EventCallback]: - return { - EventType.SM_READ_PREFETCHED: self._on_read_prefetched, - EventType.SM_WRITE_RESERVED: self._on_write_reserved, - } - - def _on_read_prefetched(self, event: Event) -> None: - self._read_requests.add(1) - self._read_succeed.add(len(event.metadata["succeeded_keys"])) - self._read_failed.add(len(event.metadata["failed_keys"])) - - def _on_write_reserved(self, event: Event) -> None: - self._write_requests.add(1) - self._write_succeed.add(len(event.metadata["succeeded_keys"])) - self._write_failed.add(len(event.metadata["failed_keys"])) diff --git a/lmcache/v1/mp_observability/subscribers/metrics/sm_lifecycle.py b/lmcache/v1/mp_observability/subscribers/metrics/sm_lifecycle.py index 68fd5e727f5..ecff664621b 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/sm_lifecycle.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/sm_lifecycle.py @@ -68,7 +68,7 @@ def __init__(self, sample_rate: float = 0.01) -> None: self._sample_threshold = int(sample_rate * self._sample_prime) meter = metrics.get_meter("lmcache.sm") self._real_reuse_gap_seconds_hist = meter.create_histogram( - "lmcache_mp.real_reuse_gap_seconds", + "lmcache_mp.real_reuse_gap", description=( "Gap between a chunk's last access (read or write) and " "next read. Storage cost. Tagged with cache_salt." @@ -76,7 +76,7 @@ def __init__(self, sample_rate: float = 0.01) -> None: unit="s", ) self._real_reuse_gap_chunks_hist = meter.create_histogram( - "lmcache_mp.real_reuse_gap_chunks", + "lmcache_mp.real_reuse_gap_objects", description=( "Per-cache_salt access-counter gap between two reads of " "the same chunk. Storage volume. Tagged with cache_salt." diff --git a/tests/v1/mp_observability/subscribers/metrics/test_event_bus_self_metrics.py b/tests/v1/mp_observability/subscribers/metrics/test_event_bus_self_metrics.py index 373fe4cbbd5..e7f9f34cdcf 100644 --- a/tests/v1/mp_observability/subscribers/metrics/test_event_bus_self_metrics.py +++ b/tests/v1/mp_observability/subscribers/metrics/test_event_bus_self_metrics.py @@ -29,13 +29,6 @@ def _make_mock_metrics() -> tuple[MagicMock, MagicMock]: return mock_metrics, mock_meter -def _gauge_callbacks(meter: MagicMock) -> dict: - return { - c.args[0]: c.kwargs["callbacks"][0] - for c in meter.create_observable_gauge.call_args_list - } - - def _counter_callbacks(meter: MagicMock) -> dict: return { c.args[0]: c.kwargs["callbacks"][0] @@ -44,31 +37,16 @@ def _counter_callbacks(meter: MagicMock) -> dict: class TestRegistration: - def test_registers_two_gauges_and_two_counters(self): + def test_registers_two_counters(self): bus = EventBus(EventBusConfig(enabled=True)) mock_metrics, meter = _make_mock_metrics() with patch(_PATCH_TARGET, mock_metrics): EventBusSelfMetricsSubscriber(bus) - assert set(_gauge_callbacks(meter)) == { - "lmcache_mp.event_bus.queue_depth", - "lmcache_mp.event_bus.drain_lag_seconds", - } assert set(_counter_callbacks(meter)) == { "lmcache_mp.event_bus.dropped_events_total", "lmcache_mp.event_bus.subscriber_exceptions", } - def test_queue_depth_callback_reflects_bus(self): - bus = EventBus(EventBusConfig(enabled=True)) - bus.publish(Event(event_type=EventType.L1_READ_FINISHED, session_id="s1")) - bus.publish(Event(event_type=EventType.L1_READ_FINISHED, session_id="s2")) - - mock_metrics, meter = _make_mock_metrics() - with patch(_PATCH_TARGET, mock_metrics): - EventBusSelfMetricsSubscriber(bus) - cb = _gauge_callbacks(meter)["lmcache_mp.event_bus.queue_depth"] - assert cb(None) == [(2, None)] - def test_dropped_counter_callback_reflects_drops(self): bus = EventBus(EventBusConfig(enabled=True, max_queue_size=2)) for _ in range(5): diff --git a/tests/v1/mp_observability/subscribers/metrics/test_l0_l1_throughput.py b/tests/v1/mp_observability/subscribers/metrics/test_l0_l1_throughput.py index 373f95997b8..4672a3f7031 100644 --- a/tests/v1/mp_observability/subscribers/metrics/test_l0_l1_throughput.py +++ b/tests/v1/mp_observability/subscribers/metrics/test_l0_l1_throughput.py @@ -22,8 +22,8 @@ from tests.v1.mp_observability.subscribers.metrics.otel_setup import reader as _reader _DRAIN_WAIT = 0.15 -_STORE_METRIC = "lmcache_mp.l0_l1_store_throughput_gbs" -_LOAD_METRIC = "lmcache_mp.l0_l1_load_throughput_gbs" +_STORE_METRIC = "lmcache_mp.l0_l1_store_throughput" +_LOAD_METRIC = "lmcache_mp.l0_l1_load_throughput" # --------------------------------------------------------------------------- diff --git a/tests/v1/mp_observability/subscribers/metrics/test_l0_lifecycle.py b/tests/v1/mp_observability/subscribers/metrics/test_l0_lifecycle.py index 6f6df475892..045dbc22287 100644 --- a/tests/v1/mp_observability/subscribers/metrics/test_l0_lifecycle.py +++ b/tests/v1/mp_observability/subscribers/metrics/test_l0_lifecycle.py @@ -114,7 +114,7 @@ def subscriber(bus): class TestL0NewAllocation: def test_new_block_no_metrics_emitted(self, bus, subscriber): - count_before = _get_histogram_count("lmcache_mp.l0_block_lifetime_seconds") + count_before = _get_histogram_count("lmcache_mp.l0_block_lifetime") bus.start() bus.publish( _make_allocation_event( @@ -124,7 +124,7 @@ def test_new_block_no_metrics_emitted(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - count_after = _get_histogram_count("lmcache_mp.l0_block_lifetime_seconds") + count_after = _get_histogram_count("lmcache_mp.l0_block_lifetime") assert count_after == count_before def test_shadow_map_populated(self, bus, subscriber): @@ -158,7 +158,7 @@ def test_prefix_sharing_no_reuse_gap(self, bus, subscriber): ) time.sleep(_DRAIN_WAIT) - reuse_before = _get_histogram_count("lmcache_mp.l0_block_reuse_gap_seconds") + reuse_before = _get_histogram_count("lmcache_mp.l0_block_reuse_gap") # Request B also uses block 5 with same tokens (prefix sharing). bus.publish( @@ -167,7 +167,7 @@ def test_prefix_sharing_no_reuse_gap(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - reuse_after = _get_histogram_count("lmcache_mp.l0_block_reuse_gap_seconds") + reuse_after = _get_histogram_count("lmcache_mp.l0_block_reuse_gap") # No reuse gap should be recorded — this is prefix sharing. assert reuse_after == reuse_before @@ -274,7 +274,7 @@ def test_true_reuse_after_release(self, bus, subscriber): class TestL0EvictionDetection: def test_different_tokens_triggers_eviction(self, bus, subscriber): - count_before = _get_histogram_count("lmcache_mp.l0_block_lifetime_seconds") + count_before = _get_histogram_count("lmcache_mp.l0_block_lifetime") bus.start() bus.publish( @@ -288,13 +288,11 @@ def test_different_tokens_triggers_eviction(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - count_after = _get_histogram_count("lmcache_mp.l0_block_lifetime_seconds") + count_after = _get_histogram_count("lmcache_mp.l0_block_lifetime") assert count_after == count_before + 1 def test_eviction_records_idle_time(self, bus, subscriber): - idle_before = _get_histogram_count( - "lmcache_mp.l0_block_idle_before_evict_seconds" - ) + idle_before = _get_histogram_count("lmcache_mp.l0_block_idle_before_evict") bus.start() bus.publish( @@ -308,9 +306,7 @@ def test_eviction_records_idle_time(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - idle_after = _get_histogram_count( - "lmcache_mp.l0_block_idle_before_evict_seconds" - ) + idle_after = _get_histogram_count("lmcache_mp.l0_block_idle_before_evict") assert idle_after == idle_before + 1 def test_eviction_clears_old_owners(self, bus, subscriber): @@ -362,7 +358,7 @@ def test_reuse_gap_after_release_and_reuse(self, bus, subscriber): ) time.sleep(_DRAIN_WAIT) - gap_before = _get_histogram_count("lmcache_mp.l0_block_reuse_gap_seconds") + gap_before = _get_histogram_count("lmcache_mp.l0_block_reuse_gap") # Now evict to flush reuse gaps. bus.publish( @@ -373,7 +369,7 @@ def test_reuse_gap_after_release_and_reuse(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - gap_after = _get_histogram_count("lmcache_mp.l0_block_reuse_gap_seconds") + gap_after = _get_histogram_count("lmcache_mp.l0_block_reuse_gap") # 2 true reuses → 1 reuse gap. assert gap_after == gap_before + 1 @@ -472,7 +468,7 @@ def test_eviction_emits_instance_id_and_model_name(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - attrs_list = _get_histogram_attrs("lmcache_mp.l0_block_lifetime_seconds") + attrs_list = _get_histogram_attrs("lmcache_mp.l0_block_lifetime") matching = [ a for a in attrs_list diff --git a/tests/v1/mp_observability/subscribers/metrics/test_l1_lifecycle.py b/tests/v1/mp_observability/subscribers/metrics/test_l1_lifecycle.py index 965a63b528b..f5a8c351bd5 100644 --- a/tests/v1/mp_observability/subscribers/metrics/test_l1_lifecycle.py +++ b/tests/v1/mp_observability/subscribers/metrics/test_l1_lifecycle.py @@ -95,7 +95,7 @@ def test_write_populates_shadow(self, bus, subscriber): assert "life-b" in subscriber._shadow def test_eviction_records_lifetime(self, bus, subscriber): - count_before = _get_histogram_count("lmcache_mp.l1_chunk_lifetime_seconds") + count_before = _get_histogram_count("lmcache_mp.l1_chunk_lifetime") keys = ["lt-1"] bus.start() bus.publish(_make_event(EventType.L1_WRITE_FINISHED, keys)) @@ -103,13 +103,11 @@ def test_eviction_records_lifetime(self, bus, subscriber): bus.publish(_make_event(EventType.L1_KEYS_EVICTED, keys)) time.sleep(_DRAIN_WAIT) bus.stop() - count_after = _get_histogram_count("lmcache_mp.l1_chunk_lifetime_seconds") + count_after = _get_histogram_count("lmcache_mp.l1_chunk_lifetime") assert count_after == count_before + 1 def test_eviction_records_idle(self, bus, subscriber): - count_before = _get_histogram_count( - "lmcache_mp.l1_chunk_idle_before_evict_seconds" - ) + count_before = _get_histogram_count("lmcache_mp.l1_chunk_idle_before_evict") keys = ["idle-1"] bus.start() bus.publish(_make_event(EventType.L1_WRITE_FINISHED, keys)) @@ -117,9 +115,7 @@ def test_eviction_records_idle(self, bus, subscriber): bus.publish(_make_event(EventType.L1_KEYS_EVICTED, keys)) time.sleep(_DRAIN_WAIT) bus.stop() - count_after = _get_histogram_count( - "lmcache_mp.l1_chunk_idle_before_evict_seconds" - ) + count_after = _get_histogram_count("lmcache_mp.l1_chunk_idle_before_evict") assert count_after == count_before + 1 def test_eviction_removes_from_shadow(self, bus, subscriber): @@ -147,7 +143,7 @@ def test_eviction_without_write_no_crash(self, bus, subscriber): class TestL1ReuseGap: def test_read_after_write_records_reuse_gap(self, bus, subscriber): - count_before = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap_seconds") + count_before = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap") keys = ["rg-1"] bus.start() bus.publish(_make_event(EventType.L1_WRITE_FINISHED, keys)) @@ -155,11 +151,11 @@ def test_read_after_write_records_reuse_gap(self, bus, subscriber): bus.publish(_make_event(EventType.L1_READ_FINISHED, keys)) time.sleep(_DRAIN_WAIT) bus.stop() - count_after = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap_seconds") + count_after = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap") assert count_after == count_before + 1 def test_rewrite_records_reuse_gap(self, bus, subscriber): - count_before = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap_seconds") + count_before = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap") keys = ["rg-2"] bus.start() bus.publish(_make_event(EventType.L1_WRITE_FINISHED, keys)) @@ -167,17 +163,17 @@ def test_rewrite_records_reuse_gap(self, bus, subscriber): bus.publish(_make_event(EventType.L1_WRITE_FINISHED, keys)) time.sleep(_DRAIN_WAIT) bus.stop() - count_after = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap_seconds") + count_after = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap") assert count_after == count_before + 1 def test_read_untracked_key_no_gap(self, bus, subscriber): """Reading a key never written should not record a gap.""" - count_before = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap_seconds") + count_before = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap") bus.start() bus.publish(_make_event(EventType.L1_READ_FINISHED, ["nowrite-1"])) time.sleep(_DRAIN_WAIT) bus.stop() - count_after = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap_seconds") + count_after = _get_histogram_count("lmcache_mp.l1_chunk_reuse_gap") assert count_after == count_before @@ -188,9 +184,7 @@ def test_read_untracked_key_no_gap(self, bus, subscriber): class TestL1EvictReuseGap: def test_rewrite_after_eviction_records_gap(self, bus, subscriber): - count_before = _get_histogram_count( - "lmcache_mp.l1_chunk_evict_reuse_gap_seconds" - ) + count_before = _get_histogram_count("lmcache_mp.l1_chunk_evict_reuse_gap") keys = ["erg-1"] bus.start() bus.publish(_make_event(EventType.L1_WRITE_FINISHED, keys)) @@ -200,9 +194,7 @@ def test_rewrite_after_eviction_records_gap(self, bus, subscriber): bus.publish(_make_event(EventType.L1_WRITE_FINISHED, keys)) time.sleep(_DRAIN_WAIT) bus.stop() - count_after = _get_histogram_count( - "lmcache_mp.l1_chunk_evict_reuse_gap_seconds" - ) + count_after = _get_histogram_count("lmcache_mp.l1_chunk_evict_reuse_gap") assert count_after == count_before + 1 def test_evicted_key_tracked_in_evicted_at(self, bus, subscriber): @@ -272,7 +264,7 @@ def test_deterministic_sampling_is_consistent(self, bus): def test_unsampled_key_ignored_on_eviction(self, bus, sampled_subscriber): """Evicting an unsampled key should not record lifetime.""" - count_before = _get_histogram_count("lmcache_mp.l1_chunk_lifetime_seconds") + count_before = _get_histogram_count("lmcache_mp.l1_chunk_lifetime") keys = ["skip-ev-1"] bus.start() bus.publish(_make_event(EventType.L1_WRITE_FINISHED, keys)) @@ -280,7 +272,7 @@ def test_unsampled_key_ignored_on_eviction(self, bus, sampled_subscriber): bus.publish(_make_event(EventType.L1_KEYS_EVICTED, keys)) time.sleep(_DRAIN_WAIT) bus.stop() - count_after = _get_histogram_count("lmcache_mp.l1_chunk_lifetime_seconds") + count_after = _get_histogram_count("lmcache_mp.l1_chunk_lifetime") assert count_after == count_before diff --git a/tests/v1/mp_observability/subscribers/metrics/test_l2.py b/tests/v1/mp_observability/subscribers/metrics/test_l2.py index 3b9ed42c410..6512a424287 100644 --- a/tests/v1/mp_observability/subscribers/metrics/test_l2.py +++ b/tests/v1/mp_observability/subscribers/metrics/test_l2.py @@ -131,8 +131,8 @@ def test_store_submitted_counts(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.l2_store_tasks"] == 2 - assert delta["lmcache_mp.l2_store_keys"] == 15 + assert delta["lmcache_mp.l2_store_submitted"] == 2 + assert delta["lmcache_mp.l2_store_submitted_objects"] == 15 def test_store_completed_success(self, bus, subscriber, snapshot): bus.start() @@ -147,8 +147,7 @@ def test_store_completed_success(self, bus, subscriber, snapshot): delta = snapshot() assert delta["lmcache_mp.l2_store_completed"] == 1 - assert delta["lmcache_mp.l2_store_succeeded_keys"] == 8 - assert delta.get("lmcache_mp.l2_store_failed_keys", 0) == 0 + assert delta["lmcache_mp.l2_store_completed_objects"] == 8 def test_store_completed_with_failures(self, bus, subscriber, snapshot): bus.start() @@ -163,8 +162,7 @@ def test_store_completed_with_failures(self, bus, subscriber, snapshot): delta = snapshot() assert delta["lmcache_mp.l2_store_completed"] == 1 - assert delta["lmcache_mp.l2_store_succeeded_keys"] == 3 - assert delta["lmcache_mp.l2_store_failed_keys"] == 7 + assert delta["lmcache_mp.l2_store_completed_objects"] == 3 def test_store_full_lifecycle(self, bus, subscriber, snapshot): """Simulate warmup: submit 20 keys, all succeed.""" @@ -185,11 +183,10 @@ def test_store_full_lifecycle(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.l2_store_tasks"] == 1 - assert delta["lmcache_mp.l2_store_keys"] == 20 + assert delta["lmcache_mp.l2_store_submitted"] == 1 + assert delta["lmcache_mp.l2_store_submitted_objects"] == 20 assert delta["lmcache_mp.l2_store_completed"] == 1 - assert delta["lmcache_mp.l2_store_succeeded_keys"] == 20 - assert delta.get("lmcache_mp.l2_store_failed_keys", 0) == 0 + assert delta["lmcache_mp.l2_store_completed_objects"] == 20 # --------------------------------------------------------------------------- @@ -210,8 +207,8 @@ def test_lookup_submitted_counts(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.l2_prefetch_lookups"] == 1 - assert delta["lmcache_mp.l2_prefetch_lookup_keys"] == 12 + assert delta["lmcache_mp.l2_prefetch_lookup"] == 1 + assert delta["lmcache_mp.l2_prefetch_lookup_objects"] == 12 def test_lookup_completed_counts_hits(self, bus, subscriber, snapshot): bus.start() @@ -225,7 +222,7 @@ def test_lookup_completed_counts_hits(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.l2_prefetch_hit_keys"] == 10 + assert delta["lmcache_mp.l2_prefetch_hit"] == 10 def test_load_submitted_counts(self, bus, subscriber, snapshot): bus.start() @@ -239,8 +236,8 @@ def test_load_submitted_counts(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.l2_prefetch_load_tasks"] == 2 - assert delta["lmcache_mp.l2_prefetch_load_keys"] == 10 + assert delta["lmcache_mp.l2_prefetch_load_submitted"] == 2 + assert delta["lmcache_mp.l2_prefetch_load_submitted_objects"] == 10 def test_load_completed_counts(self, bus, subscriber, snapshot): bus.start() @@ -254,8 +251,7 @@ def test_load_completed_counts(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.l2_prefetch_loaded_keys"] == 9 - assert delta["lmcache_mp.l2_prefetch_failed_keys"] == 1 + assert delta["lmcache_mp.l2_prefetch_load_completed"] == 9 def test_prefetch_full_lifecycle(self, bus, subscriber, snapshot): """Simulate query: lookup 20 keys, 18 prefix hits, all 18 load OK.""" @@ -288,13 +284,12 @@ def test_prefetch_full_lifecycle(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.l2_prefetch_lookups"] == 1 - assert delta["lmcache_mp.l2_prefetch_lookup_keys"] == 20 - assert delta["lmcache_mp.l2_prefetch_hit_keys"] == 18 - assert delta["lmcache_mp.l2_prefetch_load_tasks"] == 1 - assert delta["lmcache_mp.l2_prefetch_load_keys"] == 18 - assert delta["lmcache_mp.l2_prefetch_loaded_keys"] == 18 - assert delta.get("lmcache_mp.l2_prefetch_failed_keys", 0) == 0 + assert delta["lmcache_mp.l2_prefetch_lookup"] == 1 + assert delta["lmcache_mp.l2_prefetch_lookup_objects"] == 20 + assert delta["lmcache_mp.l2_prefetch_hit"] == 18 + assert delta["lmcache_mp.l2_prefetch_load_submitted"] == 1 + assert delta["lmcache_mp.l2_prefetch_load_submitted_objects"] == 18 + assert delta["lmcache_mp.l2_prefetch_load_completed"] == 18 # --------------------------------------------------------------------------- @@ -435,10 +430,10 @@ def test_multiple_store_events_accumulate(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.l2_store_tasks"] == 5 - assert delta["lmcache_mp.l2_store_keys"] == 15 + assert delta["lmcache_mp.l2_store_submitted"] == 5 + assert delta["lmcache_mp.l2_store_submitted_objects"] == 15 assert delta["lmcache_mp.l2_store_completed"] == 5 - assert delta["lmcache_mp.l2_store_succeeded_keys"] == 15 + assert delta["lmcache_mp.l2_store_completed_objects"] == 15 def test_multiple_prefetch_events_accumulate(self, bus, subscriber, snapshot): bus.start() @@ -471,10 +466,9 @@ def test_multiple_prefetch_events_accumulate(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.l2_prefetch_lookups"] == 3 - assert delta["lmcache_mp.l2_prefetch_lookup_keys"] == 30 - assert delta["lmcache_mp.l2_prefetch_hit_keys"] == 24 - assert delta["lmcache_mp.l2_prefetch_load_tasks"] == 3 - assert delta["lmcache_mp.l2_prefetch_load_keys"] == 24 - assert delta["lmcache_mp.l2_prefetch_loaded_keys"] == 21 - assert delta["lmcache_mp.l2_prefetch_failed_keys"] == 3 + assert delta["lmcache_mp.l2_prefetch_lookup"] == 3 + assert delta["lmcache_mp.l2_prefetch_lookup_objects"] == 30 + assert delta["lmcache_mp.l2_prefetch_hit"] == 24 + assert delta["lmcache_mp.l2_prefetch_load_submitted"] == 3 + assert delta["lmcache_mp.l2_prefetch_load_submitted_objects"] == 24 + assert delta["lmcache_mp.l2_prefetch_load_completed"] == 21 diff --git a/tests/v1/mp_observability/subscribers/metrics/test_l2_throughput.py b/tests/v1/mp_observability/subscribers/metrics/test_l2_throughput.py index b8a2e0b34bc..e1d4c3cb5b3 100644 --- a/tests/v1/mp_observability/subscribers/metrics/test_l2_throughput.py +++ b/tests/v1/mp_observability/subscribers/metrics/test_l2_throughput.py @@ -22,8 +22,8 @@ from tests.v1.mp_observability.subscribers.metrics.otel_setup import reader as _reader _DRAIN_WAIT = 0.15 -_STORE_METRIC = "lmcache_mp.l2_store_throughput_gbs" -_LOAD_METRIC = "lmcache_mp.l2_load_throughput_gbs" +_STORE_METRIC = "lmcache_mp.l2_store_throughput" +_LOAD_METRIC = "lmcache_mp.l2_load_throughput" # --------------------------------------------------------------------------- diff --git a/tests/v1/mp_observability/subscribers/metrics/test_lookup.py b/tests/v1/mp_observability/subscribers/metrics/test_lookup.py index c106942d15c..635b8aa9ffb 100644 --- a/tests/v1/mp_observability/subscribers/metrics/test_lookup.py +++ b/tests/v1/mp_observability/subscribers/metrics/test_lookup.py @@ -127,8 +127,8 @@ def test_full_hit(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.lookup_requested_tokens"] == 1024 - assert delta["lmcache_mp.lookup_hit_tokens"] == 1024 + assert delta["lmcache_mp.lookup_requested"] == 1024 + assert delta["lmcache_mp.lookup_hit"] == 1024 def test_partial_hit(self, bus, subscriber, snapshot): """A prefix of the requested tokens is served from cache.""" @@ -148,8 +148,8 @@ def test_partial_hit(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.lookup_requested_tokens"] == 1024 - assert delta["lmcache_mp.lookup_hit_tokens"] == 512 + assert delta["lmcache_mp.lookup_requested"] == 1024 + assert delta["lmcache_mp.lookup_hit"] == 512 def test_full_miss(self, bus, subscriber, snapshot): """Nothing is cached; both counters still receive the denominator.""" @@ -169,8 +169,8 @@ def test_full_miss(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta["lmcache_mp.lookup_requested_tokens"] == 1024 - assert delta.get("lmcache_mp.lookup_hit_tokens", 0) == 0 + assert delta["lmcache_mp.lookup_requested"] == 1024 + assert delta.get("lmcache_mp.lookup_hit", 0) == 0 def test_early_exit_contributes_zero(self, bus, subscriber, snapshot): """Early-exit lookups (no layout_desc or empty chunk_hashes) emit @@ -192,8 +192,8 @@ def test_early_exit_contributes_zero(self, bus, subscriber, snapshot): bus.stop() delta = snapshot() - assert delta.get("lmcache_mp.lookup_requested_tokens", 0) == 0 - assert delta.get("lmcache_mp.lookup_hit_tokens", 0) == 0 + assert delta.get("lmcache_mp.lookup_requested", 0) == 0 + assert delta.get("lmcache_mp.lookup_hit", 0) == 0 def test_multiple_lookups_accumulate(self, bus, subscriber, snapshot): """Counters should accumulate across multiple completed lookups.""" @@ -229,9 +229,9 @@ def test_multiple_lookups_accumulate(self, bus, subscriber, snapshot): delta = snapshot() # 3*512 + 2*1024 = 1536 + 2048 = 3584 - assert delta["lmcache_mp.lookup_requested_tokens"] == 3584 + assert delta["lmcache_mp.lookup_requested"] == 3584 # 3*512 + 2*256 = 1536 + 512 = 2048 - assert delta["lmcache_mp.lookup_hit_tokens"] == 2048 + assert delta["lmcache_mp.lookup_hit"] == 2048 # --------------------------------------------------------------------------- @@ -279,12 +279,10 @@ def test_carries_model_name_and_cache_salt(self, bus, subscriber): ("cache_salt", "tenant-A"), ("model_name", "llama-3.1-8b"), ) - before_req = before.get("lmcache_mp.lookup_requested_tokens", {}).get( - attr_key, 0 - ) - before_hit = before.get("lmcache_mp.lookup_hit_tokens", {}).get(attr_key, 0) - after_req = after.get("lmcache_mp.lookup_requested_tokens", {}).get(attr_key, 0) - after_hit = after.get("lmcache_mp.lookup_hit_tokens", {}).get(attr_key, 0) + before_req = before.get("lmcache_mp.lookup_requested", {}).get(attr_key, 0) + before_hit = before.get("lmcache_mp.lookup_hit", {}).get(attr_key, 0) + after_req = after.get("lmcache_mp.lookup_requested", {}).get(attr_key, 0) + after_hit = after.get("lmcache_mp.lookup_hit", {}).get(attr_key, 0) assert after_req == before_req + 1024 assert after_hit == before_hit + 768 @@ -325,7 +323,7 @@ def test_different_models_accumulate_independently(self, bus, subscriber): after = _read_counters_by_attrs() a_key = (("cache_salt", "salt-1"), ("model_name", "model-A")) b_key = (("cache_salt", "salt-1"), ("model_name", "model-B")) - req = "lmcache_mp.lookup_requested_tokens" + req = "lmcache_mp.lookup_requested" assert ( after.get(req, {}).get(a_key, 0) == before.get(req, {}).get(a_key, 0) + 512 ) @@ -355,7 +353,7 @@ def test_missing_labels_record_without_attrs(self, bus, subscriber): after = _read_counters_by_attrs() empty_key: tuple = () - req = "lmcache_mp.lookup_requested_tokens" + req = "lmcache_mp.lookup_requested" assert ( after.get(req, {}).get(empty_key, 0) == before.get(req, {}).get(empty_key, 0) + 128 diff --git a/tests/v1/mp_observability/subscribers/metrics/test_sm.py b/tests/v1/mp_observability/subscribers/metrics/test_sm.py deleted file mode 100644 index 6cf16ecd058..00000000000 --- a/tests/v1/mp_observability/subscribers/metrics/test_sm.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 - -"""Tests for SMMetricsSubscriber.""" - -# Standard -import time - -# Third Party -import pytest - -# First Party -from lmcache.v1.mp_observability.event import Event, EventType -from lmcache.v1.mp_observability.event_bus import EventBus, EventBusConfig -from lmcache.v1.mp_observability.subscribers.metrics.sm import ( - SMMetricsSubscriber, -) - - -def _make_keys(count: int) -> list: - """Create a list of placeholder key objects for testing.""" - return [f"key-{i}" for i in range(count)] - - -def _make_sm_event( - event_type: EventType, - succeeded: list, - failed: list, -) -> Event: - return Event( - event_type=event_type, - metadata={"succeeded_keys": succeeded, "failed_keys": failed}, - ) - - -@pytest.fixture -def bus(): - return EventBus(EventBusConfig(enabled=True, max_queue_size=100)) - - -@pytest.fixture -def subscriber(bus): - sub = SMMetricsSubscriber() - bus.register_subscriber(sub) - return sub - - -class TestSMMetricsSubscriber: - def test_read_prefetched_increments_counters(self, bus, subscriber): - bus.start() - bus.publish( - _make_sm_event( - EventType.SM_READ_PREFETCHED, - succeeded=_make_keys(3), - failed=_make_keys(2), - ) - ) - time.sleep(0.15) - bus.stop() - - def test_write_reserved_increments_counters(self, bus, subscriber): - bus.start() - bus.publish( - _make_sm_event( - EventType.SM_WRITE_RESERVED, - succeeded=_make_keys(5), - failed=_make_keys(1), - ) - ) - time.sleep(0.15) - bus.stop() - - def test_no_subscription_for_finished_events(self, subscriber): - subs = subscriber.get_subscriptions() - assert EventType.SM_READ_PREFETCHED_FINISHED not in subs - assert EventType.SM_WRITE_FINISHED not in subs - - def test_subscriptions_cover_expected_events(self, subscriber): - subs = subscriber.get_subscriptions() - assert EventType.SM_READ_PREFETCHED in subs - assert EventType.SM_WRITE_RESERVED in subs - - def test_multiple_events_accumulate(self, bus, subscriber): - bus.start() - for _ in range(10): - bus.publish( - _make_sm_event( - EventType.SM_READ_PREFETCHED, - succeeded=_make_keys(1), - failed=[], - ) - ) - time.sleep(0.15) - bus.stop() - - def test_does_not_crash_on_empty_keys(self, bus, subscriber): - bus.start() - bus.publish( - _make_sm_event( - EventType.SM_READ_PREFETCHED, - succeeded=[], - failed=[], - ) - ) - time.sleep(0.15) - bus.stop() diff --git a/tests/v1/mp_observability/subscribers/metrics/test_sm_lifecycle.py b/tests/v1/mp_observability/subscribers/metrics/test_sm_lifecycle.py index 9572eb39bdc..43595cb525d 100644 --- a/tests/v1/mp_observability/subscribers/metrics/test_sm_lifecycle.py +++ b/tests/v1/mp_observability/subscribers/metrics/test_sm_lifecycle.py @@ -4,10 +4,10 @@ Both histograms are driven by StorageManager read/write events: -- ``lmcache_mp.real_reuse_gap_seconds`` — wall-clock gap between a +- ``lmcache_mp.real_reuse_gap`` — wall-clock gap between a chunk's most recent access (read or write) and the next read. Captures storage cost. -- ``lmcache_mp.real_reuse_gap_chunks`` — per-``cache_salt`` chunk-event +- ``lmcache_mp.real_reuse_gap_objects`` — per-``cache_salt`` chunk-event gap between two reads of the same chunk. The counter advances on every chunk access (read AND write, all chunks); the histogram records gaps only on read events for sampled chunks. @@ -118,8 +118,8 @@ def subscriber(bus): class TestReadEmitsGap: def test_read_then_read_records_gap(self, bus, subscriber): - before_t = _totals_by_salt("lmcache_mp.real_reuse_gap_seconds") - before_c = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before_t = _totals_by_salt("lmcache_mp.real_reuse_gap") + before_c = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_read([_Key("h1", cache_salt="t-a")])) # counter=1, seed @@ -127,14 +127,16 @@ def test_read_then_read_records_gap(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - time_d = _delta(before_t, _totals_by_salt("lmcache_mp.real_reuse_gap_seconds")) - chunks_d = _delta(before_c, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + time_d = _delta(before_t, _totals_by_salt("lmcache_mp.real_reuse_gap")) + chunks_d = _delta( + before_c, _totals_by_salt("lmcache_mp.real_reuse_gap_objects") + ) assert time_d.get("t-a", (0.0, 0))[1] == 1 assert chunks_d.get("t-a", (0.0, 0)) == (1.0, 1) def test_write_then_read_records_gap(self, bus, subscriber): """Write seeds the anchor; first read records gap = read - write.""" - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_write([_Key("h1", cache_salt="t-w")])) # counter=1, seed @@ -142,21 +144,21 @@ def test_write_then_read_records_gap(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) assert delta.get("t-w", (0.0, 0)) == (1.0, 1) def test_first_access_is_cold_no_emission(self, bus, subscriber): """The very first event for a chunk seeds; no histogram sample.""" - before_t = _totals_by_salt("lmcache_mp.real_reuse_gap_seconds") - before_c = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before_t = _totals_by_salt("lmcache_mp.real_reuse_gap") + before_c = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_read([_Key("cold", cache_salt="t-cold")])) time.sleep(_DRAIN_WAIT) bus.stop() - t_d = _delta(before_t, _totals_by_salt("lmcache_mp.real_reuse_gap_seconds")) - c_d = _delta(before_c, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + t_d = _delta(before_t, _totals_by_salt("lmcache_mp.real_reuse_gap")) + c_d = _delta(before_c, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) assert t_d.get("t-cold", (0.0, 0))[1] == 0 assert c_d.get("t-cold", (0.0, 0))[1] == 0 @@ -164,7 +166,7 @@ def test_first_access_is_cold_no_emission(self, bus, subscriber): class TestWriteDoesNotEmit: def test_write_then_write_no_emission(self, bus, subscriber): """Write → write does not record a sample.""" - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_write([_Key("k", cache_salt="t-ww")])) @@ -172,12 +174,12 @@ def test_write_then_write_no_emission(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) assert delta.get("t-ww", (0.0, 0))[1] == 0 def test_read_then_write_no_emission(self, bus, subscriber): """Read → write does not record a sample. Write only updates anchor.""" - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_read([_Key("k", cache_salt="t-rw")])) # seed @@ -185,12 +187,12 @@ def test_read_then_write_no_emission(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) assert delta.get("t-rw", (0.0, 0))[1] == 0 def test_write_updates_anchor_for_next_read(self, bus, subscriber): """After read → write, the next read measures from the write.""" - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_read([_Key("k", cache_salt="t-x")])) # counter=1 @@ -201,7 +203,7 @@ def test_write_updates_anchor_for_next_read(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) # Only sample is the final read of k, gap = 5 - 3 = 2. assert delta.get("t-x", (0.0, 0)) == (2.0, 1) @@ -209,7 +211,7 @@ def test_write_updates_anchor_for_next_read(self, bus, subscriber): class TestChunksGapCounter: def test_counter_advances_for_all_accesses(self, bus, subscriber): """Counter bumps on every read AND write, all chunks under salt.""" - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_read([_Key("target", cache_salt="t-b")])) # counter=1 @@ -222,12 +224,12 @@ def test_counter_advances_for_all_accesses(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) assert delta.get("t-b", (0.0, 0)) == (5.0, 1) def test_cache_salt_isolates_counter(self, bus, subscriber): """Per-salt counter: t-x accesses don't bump t-y's gap.""" - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_read([_Key("target", cache_salt="t-y")])) # y=1 @@ -244,14 +246,14 @@ def test_cache_salt_isolates_counter(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) assert delta.get("t-y", (0.0, 0)) == (1.0, 1) # tenant-x has no second read of any chunk → no sample. assert delta.get("t-x", (0.0, 0))[1] == 0 def test_cache_salt_isolates_chunk_identity(self, bus, subscriber): """Same chunk_hash under different cache_salts = different keys.""" - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_read([_Key("shared", cache_salt="t-p")])) @@ -259,7 +261,7 @@ def test_cache_salt_isolates_chunk_identity(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) # Both tenants saw "shared" first time → no samples. assert delta.get("t-p", (0.0, 0))[1] == 0 assert delta.get("t-q", (0.0, 0))[1] == 0 @@ -270,7 +272,7 @@ def test_unsampled_first_sighting_is_dropped(self, bus): """Near-zero sample rate → cold seed skipped → no later sample.""" sub = SMLifecycleSubscriber(sample_rate=1e-9) bus.register_subscriber(sub) - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() for _ in range(3): @@ -278,7 +280,7 @@ def test_unsampled_first_sighting_is_dropped(self, bus): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) assert delta.get("t-u", (0.0, 0))[1] == 0 def test_unsampled_chunks_still_advance_counter(self, bus): @@ -295,7 +297,7 @@ def test_unsampled_chunks_still_advance_counter(self, bus): # (LRU eviction) does not break the counter semantics. sub = SMLifecycleSubscriber(sample_rate=1.0) bus.register_subscriber(sub) - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_read([_Key("k", cache_salt="t-cnt")])) # 1, seed @@ -307,7 +309,7 @@ def test_unsampled_chunks_still_advance_counter(self, bus): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) assert delta.get("t-cnt", (0.0, 0)) == (11.0, 1) @@ -315,7 +317,7 @@ class TestTPFanout: def test_read_dedupes_tp_fanout(self, bus, subscriber): """Same logical chunk fanned across TP ranks counts as one access.""" keys = [_Key("tp-chunk", cache_salt="t-tp", kv_rank=r) for r in range(4)] - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") bus.start() bus.publish(_l1_read(keys)) # counter=1 (deduped) @@ -323,14 +325,14 @@ def test_read_dedupes_tp_fanout(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) # 4 ObjectKeys per event, 1 logical chunk, 2 events → counter=2, # one sample with gap=1. assert delta.get("t-tp", (0.0, 0)) == (1.0, 1) def test_write_dedupes_tp_fanout(self, bus, subscriber): """Write fanout produces a single anchor, not one per rank.""" - before = _totals_by_salt("lmcache_mp.real_reuse_gap_chunks") + before = _totals_by_salt("lmcache_mp.real_reuse_gap_objects") keys = [_Key("tp", cache_salt="t-tpw", kv_rank=r) for r in range(4)] bus.start() @@ -339,7 +341,7 @@ def test_write_dedupes_tp_fanout(self, bus, subscriber): time.sleep(_DRAIN_WAIT) bus.stop() - delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_chunks")) + delta = _delta(before, _totals_by_salt("lmcache_mp.real_reuse_gap_objects")) assert delta.get("t-tpw", (0.0, 0)) == (1.0, 1) diff --git a/tests/v1/mp_observability/test_event_bus.py b/tests/v1/mp_observability/test_event_bus.py index 09dc1b96209..0b8c550bed8 100644 --- a/tests/v1/mp_observability/test_event_bus.py +++ b/tests/v1/mp_observability/test_event_bus.py @@ -3,7 +3,7 @@ """Tests for EventBus, EventSubscriber, and singleton management.""" # Standard -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import time # Third Party @@ -511,3 +511,29 @@ def test_block_allocation_not_delivered_to_other_subscriber(self, bus): bus.stop() assert len(sub.events) == 0 + + +_REGISTER_GAUGE = "lmcache.v1.mp_observability.event_bus.register_gauge" + + +class TestSelfMonitoringGauges: + def test_init_registers_both_gauges(self): + with patch(_REGISTER_GAUGE) as mock_register: + EventBus(EventBusConfig(enabled=True)) + names = [c.args[1] for c in mock_register.call_args_list] + assert set(names) == { + "lmcache_mp.event_bus.queue_depth", + "lmcache_mp.event_bus.drain_lag_seconds", + } + + def test_queue_depth_callback_reflects_bus(self): + captured: dict[str, object] = {} + + def _capture(_meter, name, _desc, func): + captured[name] = func + + with patch(_REGISTER_GAUGE, side_effect=_capture): + bus = EventBus(EventBusConfig(enabled=True)) + bus.publish(_make_event(session_id="s1")) + bus.publish(_make_event(session_id="s2")) + assert captured["lmcache_mp.event_bus.queue_depth"]() == 2 From 8ece5f0a8020e4f15c3197f654133874b10066e8 Mon Sep 17 00:00:00 2001 From: Jiayue Chen <96101996+glbyktjys@users.noreply.github.com> Date: Tue, 19 May 2026 16:56:56 -0700 Subject: [PATCH 37/69] Fix the translation icon together with the screen layout adjustment (#3334) * Fix the translation icon together with the screen layout adjustment Signed-off-by: Jiayue Chen --- docs/source/_static/custom.css | 25 +++++++++++++------------ docs/source/_static/custom.js | 27 ++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index f649ddda57a..bb44ef86dda 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -68,28 +68,29 @@ div.highlight button.copybtn + button.copybtn { font-size: 10px; } +/* Language switcher (中文 | Eng) — lives in the top nav bar, + or floats at top-right if the nav bar isn't found. */ .lmcache-language-switcher { - position: fixed; - top: 10px; - right: 72px; - z-index: 60; display: inline-flex; align-items: center; + justify-content: center; gap: 8px; - min-height: 36px; - padding: 0 8px; + margin: 0 6px; + padding: 4px 10px; white-space: nowrap; - font-size: 18px; + font-size: 16px; font-weight: 700; line-height: 1; color: var(--foreground); + border-radius: 8px; } -@media (max-width: 640px) { - .lmcache-language-switcher { - right: 56px; - font-size: 15px; - } +/* When there's no nav bar, float at top-right instead */ +.lmcache-language-switcher--fallback { + position: fixed; + top: 10px; + right: 72px; + z-index: 60; } .lmcache-language-switcher a { diff --git a/docs/source/_static/custom.js b/docs/source/_static/custom.js index 06e83d200f3..9bd4e2320f9 100644 --- a/docs/source/_static/custom.js +++ b/docs/source/_static/custom.js @@ -150,12 +150,37 @@ function addLanguageSwitcher() { switcher.appendChild(chineseLink); switcher.appendChild(divider); switcher.appendChild(englishLink); - document.body.appendChild(switcher); + + // Place the switcher in the top nav bar with the other icons. + // If the nav bar isn't there, show it as a floating button instead. + var navbar = findDocsNavbar(); + if (navbar) { + navbar.appendChild(switcher); + } else { + switcher.classList.add("lmcache-language-switcher--fallback"); + document.body.appendChild(switcher); + } fallbackMissingLanguagePage(chineseLink, "zh_CN"); fallbackMissingLanguagePage(englishLink, "en"); } +/** + * Locate the top nav bar that holds the GitHub / profile / theme-toggle + * icons. Prefers the structural `header nav` selector; falls back to + * the GitHub link's parent if the theme markup differs. + * + * @returns {HTMLElement | null} The nav bar element, or null if not found. + */ +function findDocsNavbar() { + var navbar = document.querySelector("header nav"); + if (navbar) { + return navbar; + } + var githubLink = document.querySelector('a[title="Visit GitHub"]'); + return githubLink ? githubLink.parentElement : null; +} + /** * Remove all goblin easter egg elements from the current page. * From 5b71110ade44ec04d9bbce8eabe925f50e11f2f0 Mon Sep 17 00:00:00 2001 From: Yihua Cheng Date: Tue, 19 May 2026 20:14:20 -0500 Subject: [PATCH 38/69] =?UTF-8?q?docs:=20daily=20drift=20check=20=E2=80=94?= =?UTF-8?q?=20multi-process=20mode=20(2026-05-18)=20(#3309)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ApostaC --- docs/design/ARCHITECTURE_MULTI_HARDWARE.md | 34 +++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/docs/design/ARCHITECTURE_MULTI_HARDWARE.md b/docs/design/ARCHITECTURE_MULTI_HARDWARE.md index 142bd988cbd..fb88412ccf5 100644 --- a/docs/design/ARCHITECTURE_MULTI_HARDWARE.md +++ b/docs/design/ARCHITECTURE_MULTI_HARDWARE.md @@ -13,7 +13,7 @@ │ └──────────────────┴──────────────────┘ │ │ │ │ │ torch_dev (unified entry) │ -│ torch_device_type ("cuda"/"xpu"/"hpu") │ +│ torch_device_type ("cuda"/"xpu"/"hpu"/"cpu") │ │ │ │ [Monkey Patch Point] │ │ New hardware can be added by extending _detect_device() │ @@ -93,8 +93,40 @@ torch_device_type == "cuda" --> VLLMPagedMemGPUConnectorV2/V3 torch_device_type == "xpu" --> VLLMPagedMemXPUConnectorV2 torch_device_type == "hpu" --> VLLMPagedMemHPUConnector +torch_device_type == "cpu" --> (no GPU connector; raises RuntimeError) ``` +## CPU-Only Stub Fallback + +`_detect_device()` also accepts a CPU-only environment where none of the +supported accelerators (CUDA, XPU, HPU) is available. In that case +`torch_device_type` is `"cpu"` and `torch_dev` is either: + +- `lmcache.v1.platform.cpu.stub_cpu_device.StubCPUDevice` — when `torch` + is importable but no GPU is detected. The stub implements the subset of + the `torch.cuda` / `torch.xpu` / `torch.hpu` surface used by the middle + layer (`Event`, `Stream`, `device`, `synchronize`, `set_device`, + `current_device`, `device_count`, `get_device_properties`, + `empty_cache`), as no-op or constant returns. `is_available()` is + `False`, so any `hasattr(torch_dev, 'xxx')` consumer that gates on the + real device's availability stays on the degraded path. +- `None` — when `torch` itself is not importable (the `lmcache-cli` + slim install). The CLI surface (`lmcache ping`, `lmcache describe`, + `lmcache query`, `lmcache bench engine`) tolerates this; engine and + storage paths do not. + +The stub is intended for L1-adapter-only flows (e.g., end-to-end MP +server smoke tests on a CPU-only host) and CLI loading without torch. It +is **not** a CPU connector: there is no entry for `"cpu"` in +`gpu_connector/__init__.py`, so calling `CreateGPUConnector` with +`torch_device_type == "cpu"` raises `RuntimeError("No supported cpu +connector found.")`. + +`normalize_kv_and_discover_format` also hardcodes `kv_layout = "HND"` +when `torch_device_type == "cpu"`, because vLLM's +`get_kv_cache_layout()` reports `NHD` for its CPU attention backend +which is wrong for that backend's actual KV cache layout. + ## Adding New Hardware 1. Add detection branch in `__init__.py` `_detect_device()` From 46c9b5be9d5a2038191044a3c242a1b5ac05bd0e Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Wed, 20 May 2026 14:34:22 +0800 Subject: [PATCH 39/69] Unconditionally compile common c++ extensions in setup.py (#3302) * Unconditionally compile common c++ extensions in setup.py Signed-off-by: Tony Lin * add "mooncake ext is optional" comment back Signed-off-by: Tony Lin --------- Signed-off-by: Tony Lin --- csrc/storage_backends/README.md | 2 +- setup.py | 242 ++++++++++++++++---------------- 2 files changed, 121 insertions(+), 123 deletions(-) diff --git a/csrc/storage_backends/README.md b/csrc/storage_backends/README.md index c50f7ee6dad..565ca92ae89 100644 --- a/csrc/storage_backends/README.md +++ b/csrc/storage_backends/README.md @@ -157,7 +157,7 @@ PYBIND11_MODULE(lmcache_mybackend, m) { Add your sources to `setup.py` alongside the existing Redis extension: ```python -# In cuda_extension() and rocm_extension(): +# In _common_cpp_extensions(): mybackend_sources = [ "csrc/storage_backends/mybackend/pybind.cpp", "csrc/storage_backends/mybackend/connector.cpp", diff --git a/setup.py b/setup.py index 2124e21e96d..836cf6148af 100644 --- a/setup.py +++ b/setup.py @@ -1,19 +1,25 @@ # SPDX-License-Identifier: Apache-2.0 # Standard from pathlib import Path +from typing import TYPE_CHECKING import os import sys # Third Party from setuptools import find_packages, setup +if TYPE_CHECKING: + # Third Party + from setuptools.extension import Extension + ROOT_DIR = Path(__file__).parent HIPIFY_DIR = os.path.join(ROOT_DIR, "csrc/") HIPIFY_OUT_DIR = os.path.join(ROOT_DIR, "csrc_hip/") # python -m build --sdist # will run python setup.py sdist --dist-dir dist -BUILDING_SDIST = "sdist" in sys.argv or os.environ.get("NO_CUDA_EXT", "0") == "1" +BUILDING_SDIST = "sdist" in sys.argv +NO_CUDA_EXT = os.environ.get("NO_CUDA_EXT", "0") == "1" # New environment variable to choose between CUDA, HIP, and SYCL BUILD_WITH_HIP = os.environ.get("BUILD_WITH_HIP", "0") == "1" @@ -116,29 +122,33 @@ def _mooncake_extension( ] -def cuda_extension() -> tuple[list, dict]: +def _common_cpp_extensions( + extra_cxx_flags: list[str], fs_extra_cxx_flags: list[str] | None = None +) -> tuple[list["Extension"], dict[str, type]]: + """Build pure C++ extensions that do not depend on any GPU backend. + + Args: + extra_cxx_flags: Additional C++ compiler flags to apply to the + `native_storage_ops` and `lmcache_redis` pure C++ extensions. + fs_extra_cxx_flags: Additional C++ compiler flags to apply to the + `lmcache_fs` extension. Defaults to `extra_cxx_flags` when not set. + + Notes: + `fs_extra_cxx_flags` exists to preserve pre-refactor SYCL compile-flag + behavior where `lmcache_fs` intentionally omitted the ABI define. + + Returns: + A tuple of: + - list: CppExtension modules for native storage backends, + including optional mooncake when enabled. + - dict: cmdclass containing BuildExtension. + """ # Third Party - from torch.utils import cpp_extension # Import here + from torch.utils import cpp_extension - print("Building CUDA extensions") - global ENABLE_CXX11_ABI - if ENABLE_CXX11_ABI: - flag_cxx_abi = "-D_GLIBCXX_USE_CXX11_ABI=1" - else: - flag_cxx_abi = "-D_GLIBCXX_USE_CXX11_ABI=0" + if fs_extra_cxx_flags is None: + fs_extra_cxx_flags = extra_cxx_flags - cuda_sources = [ - "csrc/pybind.cpp", - "csrc/mem_kernels.cu", - "csrc/mp_mem_kernels.cu", - "csrc/cal_cdf.cu", - "csrc/ac_enc.cu", - "csrc/ac_dec.cu", - "csrc/pos_kernels.cu", - "csrc/mem_alloc.cpp", - "csrc/utils.cpp", - "csrc/event_recorder.cpp", - ] storage_manager_sources = [ "csrc/storage_manager/bitmap.cpp", "csrc/storage_manager/pybind.cpp", @@ -158,20 +168,12 @@ def cuda_extension() -> tuple[list, dict]: "csrc/storage_backends/mooncake/connector.cpp", ] ext_modules = [ - cpp_extension.CUDAExtension( - "lmcache.c_ops", - sources=cuda_sources, - extra_compile_args={ - "cxx": [flag_cxx_abi, "-std=c++17"], - "nvcc": [flag_cxx_abi], - }, - ), cpp_extension.CppExtension( "lmcache.native_storage_ops", sources=storage_manager_sources, include_dirs=["csrc/storage_manager"], extra_compile_args={ - "cxx": [flag_cxx_abi, "-O3", "-std=c++17"], + "cxx": extra_cxx_flags + ["-O3", "-std=c++17"], }, ), cpp_extension.CppExtension( @@ -179,7 +181,7 @@ def cuda_extension() -> tuple[list, dict]: sources=redis_sources, include_dirs=["csrc/storage_backends", "csrc/storage_backends/redis"], extra_compile_args={ - "cxx": [flag_cxx_abi, "-O3", "-std=c++17"], + "cxx": extra_cxx_flags + ["-O3", "-std=c++17"], }, ), cpp_extension.CppExtension( @@ -187,18 +189,55 @@ def cuda_extension() -> tuple[list, dict]: sources=fs_sources, include_dirs=["csrc/storage_backends", "csrc/storage_backends/fs"], extra_compile_args={ - "cxx": [flag_cxx_abi, "-O3", "-std=c++17"], + "cxx": fs_extra_cxx_flags + ["-O3", "-std=c++17"], }, ), ] # Mooncake extension is optional. ext_modules.extend( - _mooncake_extension(cpp_extension, mooncake_sources, [flag_cxx_abi]) + _mooncake_extension(cpp_extension, mooncake_sources, extra_cxx_flags) ) cmdclass = {"build_ext": cpp_extension.BuildExtension} return ext_modules, cmdclass +def cuda_extension() -> tuple[list, dict]: + # Third Party + from torch.utils import cpp_extension # Import here + + print("Building CUDA extensions") + global ENABLE_CXX11_ABI + if ENABLE_CXX11_ABI: + flag_cxx_abi = "-D_GLIBCXX_USE_CXX11_ABI=1" + else: + flag_cxx_abi = "-D_GLIBCXX_USE_CXX11_ABI=0" + + cuda_sources = [ + "csrc/pybind.cpp", + "csrc/mem_kernels.cu", + "csrc/mp_mem_kernels.cu", + "csrc/cal_cdf.cu", + "csrc/ac_enc.cu", + "csrc/ac_dec.cu", + "csrc/pos_kernels.cu", + "csrc/mem_alloc.cpp", + "csrc/utils.cpp", + "csrc/event_recorder.cpp", + ] + ext_modules = [ + cpp_extension.CUDAExtension( + "lmcache.c_ops", + sources=cuda_sources, + extra_compile_args={ + "cxx": [flag_cxx_abi, "-std=c++17"], + "nvcc": [flag_cxx_abi], + }, + ), + ] + cmdclass = {"build_ext": cpp_extension.BuildExtension} + return ext_modules, cmdclass + + def rocm_extension() -> tuple[list, dict]: # Third Party from torch.utils import cpp_extension # Import here @@ -217,24 +256,6 @@ def rocm_extension() -> tuple[list, dict]: "csrc/utils_hip.cpp", "csrc/event_recorder.cpp", ] - storage_manager_sources = [ - "csrc/storage_manager/bitmap.cpp", - "csrc/storage_manager/pybind.cpp", - "csrc/storage_manager/ttl_lock.cpp", - "csrc/storage_manager/utils.cpp", - ] - redis_sources = [ - "csrc/storage_backends/redis/pybind.cpp", - "csrc/storage_backends/redis/connector.cpp", - ] - fs_sources = [ - "csrc/storage_backends/fs/pybind.cpp", - "csrc/storage_backends/fs/connector.cpp", - ] - mooncake_sources = [ - "csrc/storage_backends/mooncake/pybind.cpp", - "csrc/storage_backends/mooncake/connector.cpp", - ] # For HIP, we generally use CppExtension and let hipcc handle things. # Ensure CXX environment variable is set to hipcc when running this build. # e.g., CXX=hipcc python setup.py install @@ -266,33 +287,7 @@ def rocm_extension() -> tuple[list, dict]: # libraries=['amdhip64'] # Or other relevant HIP libs if needed define_macros=define_macros, ), - cpp_extension.CppExtension( - "lmcache.native_storage_ops", - sources=storage_manager_sources, - include_dirs=["csrc/storage_manager"], - extra_compile_args={ - "cxx": ["-O3", "-std=c++17"], - }, - ), - cpp_extension.CppExtension( - "lmcache.lmcache_redis", - sources=redis_sources, - include_dirs=["csrc/storage_backends", "csrc/storage_backends/redis"], - extra_compile_args={ - "cxx": ["-O3", "-std=c++17"], - }, - ), - cpp_extension.CppExtension( - "lmcache.lmcache_fs", - sources=fs_sources, - include_dirs=["csrc/storage_backends", "csrc/storage_backends/fs"], - extra_compile_args={ - "cxx": ["-O3", "-std=c++17"], - }, - ), ] - # Mooncake extension is optional. - ext_modules.extend(_mooncake_extension(cpp_extension, mooncake_sources, [])) cmdclass = {"build_ext": cpp_extension.BuildExtension} return ext_modules, cmdclass @@ -317,20 +312,6 @@ def sycl_extension() -> tuple[list, dict]: "csrc/sycl/pybind_sycl.cpp", "csrc/sycl/mem_kernels_sycl.cpp", ] - storage_manager_sources = [ - "csrc/storage_manager/bitmap.cpp", - "csrc/storage_manager/pybind.cpp", - "csrc/storage_manager/ttl_lock.cpp", - "csrc/storage_manager/utils.cpp", - ] - redis_sources = [ - "csrc/storage_backends/redis/pybind.cpp", - "csrc/storage_backends/redis/connector.cpp", - ] - fs_sources = [ - "csrc/storage_backends/fs/pybind.cpp", - "csrc/storage_backends/fs/connector.cpp", - ] # Use CppExtension with DPC++ compiler (set CXX=icpx before invoking). # The -fsycl flag enables SYCL compilation and linking. # Intel XPU optimizations: @@ -367,30 +348,6 @@ def sycl_extension() -> tuple[list, dict]: }, extra_link_args=["-fsycl"], ), - cpp_extension.CppExtension( - "lmcache.native_storage_ops", - sources=storage_manager_sources, - include_dirs=["csrc/storage_manager"], - extra_compile_args={ - "cxx": ["-D_GLIBCXX_USE_CXX11_ABI=1", "-O3", "-std=c++17"], - }, - ), - cpp_extension.CppExtension( - "lmcache.lmcache_redis", - sources=redis_sources, - include_dirs=["csrc/storage_backends", "csrc/storage_backends/redis"], - extra_compile_args={ - "cxx": ["-D_GLIBCXX_USE_CXX11_ABI=1", "-O3", "-std=c++17"], - }, - ), - cpp_extension.CppExtension( - "lmcache.lmcache_fs", - sources=fs_sources, - include_dirs=["csrc/storage_backends", "csrc/storage_backends/fs"], - extra_compile_args={ - "cxx": ["-O3", "-std=c++17"], - }, - ), ] cmdclass = {"build_ext": cpp_extension.BuildExtension} return ext_modules, cmdclass @@ -401,17 +358,58 @@ def source_dist_extension() -> tuple[list, dict]: return [], {} -if __name__ == "__main__": +def _get_common_cpp_flags() -> list[str]: + """Select common pure C++ ABI flags based on the configured build backend. + + Returns: + A list of compiler flags for pure C++ extensions. + """ + if BUILD_WITH_HIP: + return [] + if BUILD_WITH_SYCL: + return ["-D_GLIBCXX_USE_CXX11_ABI=1"] + if ENABLE_CXX11_ABI: + return ["-D_GLIBCXX_USE_CXX11_ABI=1"] + return ["-D_GLIBCXX_USE_CXX11_ABI=0"] + + +def _collect_extensions() -> tuple[list, dict]: + """Collect extension modules according to current setup.py build settings. + + Returns: + A tuple of: + - list: extension modules selected for the current build mode. + - dict: cmdclass containing BuildExtension when extensions are built. + + Notes: + - `sdist` builds skip all extension compilation. + - `NO_CUDA_EXT=1` keeps pure C++ extensions and skips GPU extensions. + - Otherwise, pure C++ extensions are combined with one GPU backend + extension set (CUDA, ROCm, or SYCL). + """ if BUILDING_SDIST: - get_extension = source_dist_extension - elif BUILD_WITH_SYCL: - get_extension = sycl_extension + return source_dist_extension() + + common_cpp_flags = _get_common_cpp_flags() + # Preserve historical SYCL compatibility: lmcache_fs was compiled without + # _GLIBCXX_USE_CXX11_ABI in pre-refactor builds. + fs_cpp_flags = [] if BUILD_WITH_SYCL else common_cpp_flags + ext_modules, cmdclass = _common_cpp_extensions(common_cpp_flags, fs_cpp_flags) + if NO_CUDA_EXT: + return ext_modules, cmdclass + + if BUILD_WITH_SYCL: + gpu_ext_modules, cmdclass = sycl_extension() elif BUILD_WITH_HIP: - get_extension = rocm_extension + gpu_ext_modules, cmdclass = rocm_extension() else: - get_extension = cuda_extension + gpu_ext_modules, cmdclass = cuda_extension() + ext_modules.extend(gpu_ext_modules) + return ext_modules, cmdclass - ext_modules, cmdclass = get_extension() + +if __name__ == "__main__": + ext_modules, cmdclass = _collect_extensions() install_requires = _read_requirements(ROOT_DIR / "requirements" / "common.txt") if BUILD_WITH_HIP: From 9a2fd944a00eccfcec48a3f7e8c724320b1c5c67 Mon Sep 17 00:00:00 2001 From: chunxiaozheng Date: Wed, 20 May 2026 14:56:27 +0800 Subject: [PATCH 40/69] [MP][bench] add l2 adapter benchmark cli (#3243) * [MP][bench] add l2 adapter benchmark cli Signed-off-by: idellzheng * bugfix: fix timeout error Signed-off-by: idellzheng * optimize Signed-off-by: idellzheng * bugfix: fix print info error Signed-off-by: idellzheng * update docs Signed-off-by: idellzheng * update docs Signed-off-by: idellzheng * update Signed-off-by: idellzheng * add format/output/quiet Signed-off-by: idellzheng * update configuration metrics Signed-off-by: idellzheng * update name Signed-off-by: idellzheng * delete unuse code Signed-off-by: idellzheng --------- Signed-off-by: idellzheng --- docs/source/cli/bench_kvcache.rst | 3 + docs/source/cli/bench_l2.rst | 340 ++++++++++ docs/source/cli/index.rst | 4 +- lmcache/cli/commands/bench/__init__.py | 9 +- .../bench/l2_adapter_bench/__init__.py | 2 + .../bench/l2_adapter_bench/command.py | 601 ++++++++++++++++++ .../commands/bench/l2_adapter_bench/data.py | 137 ++++ .../commands/bench/l2_adapter_bench/result.py | 174 +++++ .../commands/bench/l2_adapter_bench/runner.py | 318 +++++++++ 9 files changed, 1586 insertions(+), 2 deletions(-) create mode 100644 docs/source/cli/bench_l2.rst create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/__init__.py create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/command.py create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/data.py create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/result.py create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/runner.py diff --git a/docs/source/cli/bench_kvcache.rst b/docs/source/cli/bench_kvcache.rst index 22e44b1e6bd..ae03d6bcdbd 100644 --- a/docs/source/cli/bench_kvcache.rst +++ b/docs/source/cli/bench_kvcache.rst @@ -188,5 +188,8 @@ See also * :doc:`bench` -- ``lmcache bench engine`` for engine-side workload benchmarks. +* :doc:`bench_l2` -- ``lmcache bench l2`` for + store / lookup / load throughput benchmarks against an L2 cache + adapter. * :doc:`kvcache` -- ``lmcache kvcache`` for managing KV cache state on a running server (clear, etc.). diff --git a/docs/source/cli/bench_l2.rst b/docs/source/cli/bench_l2.rst new file mode 100644 index 00000000000..1dd75631963 --- /dev/null +++ b/docs/source/cli/bench_l2.rst @@ -0,0 +1,340 @@ +.. _lmcache-bench-l2: + +lmcache bench l2 +================ + +The ``lmcache bench l2`` command benchmarks an L2 cache adapter +(e.g. the local-filesystem adapter) end-to-end through the same +``parse_args_to_l2_adapters_config`` + ``create_l2_adapter`` pipeline that +LMCache uses in production. Any registered adapter type can be tested +without code changes: you describe the adapter with a single JSON spec +and pick the operations to exercise. + +.. code-block:: bash + + lmcache bench l2 [options] + +Unlike :ref:`lmcache bench engine `, this command +does **not** require an inference engine or an LMCache MP server. It +only needs the adapter's own backing storage to be reachable (for the +``fs`` adapter, that simply means a writable directory). + + +What it does +------------ + +For each measured operation the tool drives the adapter directly via +its public submit/wait API: + +* ``Store`` -- ``submit_store_task`` writes ``num_keys`` MemoryObjs per + submit and waits for the store eventfd. +* ``Lookup`` -- ``submit_lookup_and_lock_task`` checks key existence + (no payload transfer) and waits for the lookup eventfd. +* ``Load`` -- ``submit_load_task`` reads ``num_keys`` MemoryObjs per + submit and waits for the load eventfd. + +Each measured **round** issues ``--in-flight`` submits sequentially from +a single producer thread and then waits for all of them to complete; the +round duration is the wall-clock time from the first submit until the +last completion. Warmup rounds run before measurement and their results +are discarded from the final summary. + +All three operations share the same key idx universe, so running +``--only store`` followed by ``--only load`` (or ``--only lookup``) with +identical other flags hits exactly the same keys. This makes the +benchmark useful as a quick regression test for adapters that should +support a clean store -> load round-trip. + +.. note:: + + When ``--only`` is not given, the three operations are run **in a + single process in the order** ``store -> lookup -> load``. For + adapters whose backing storage sits behind an OS-level cache -- + most notably the local-filesystem (``fs``) adapter, which is + subject to the Linux **page cache** -- this means ``lookup`` and + ``load`` will almost always observe the data that ``store`` just + wrote still hot in RAM, and the reported numbers reflect + page-cache throughput rather than the underlying device. + + To benchmark each operation against a cold cache, run them + separately with ``--only`` and drop the OS caches in between, for + example:: + + lmcache bench l2 --l2-adapter '...' --only store + sync && echo 3 | sudo tee /proc/sys/vm/drop_caches + lmcache bench l2 --l2-adapter '...' --only lookup + sync && echo 3 | sudo tee /proc/sys/vm/drop_caches + lmcache bench l2 --l2-adapter '...' --only load + + For adapters that bypass the page cache (e.g. ``fs`` with + ``"use_odirect": true``) or that talk to a remote service without + a local cache, the default combined run is usually fine. +----------- + +Benchmark the local filesystem adapter with default parameters: + +.. code-block:: bash + + lmcache bench l2 \ + --l2-adapter '{"type":"fs","base_path":"/tmp/lmcache-bench"}' + +This runs all three operations (store, lookup, load) with one warmup +round and one measurement round. + +Stress the adapter with more in-flight submits and larger payloads: + +.. code-block:: bash + + lmcache bench l2 \ + --l2-adapter '{"type":"fs","base_path":"/data/lmcache-bench","relative_tmp_dir":"tmp"}' \ + --num-keys 32 --in-flight 4 \ + --data-size-kb 512 \ + --rounds 5 --warmup-rounds 1 + +Run only one operation (useful to isolate store vs. load throughput): + +.. code-block:: bash + + lmcache bench l2 \ + --l2-adapter '{"type":"fs","base_path":"/tmp/lmcache-bench"}' \ + --only store + +Lookup with a controlled hit rate (the benchmark splits the lookup keys +between a potentially-existing range and a guaranteed-non-existent +range): + +.. code-block:: bash + + lmcache bench l2 \ + --l2-adapter '{"type":"fs","base_path":"/tmp/lmcache-bench"}' \ + --only lookup --lookup-max-hit-rate 0.5 + +Enable a store -> load round-trip data integrity check on the last +measured round: + +.. code-block:: bash + + lmcache bench l2 \ + --l2-adapter '{"type":"fs","base_path":"/tmp/lmcache-bench"}' \ + --no-skip-verify + +If you prefer to keep the JSON spec out of the command line, set the +``L2_ADAPTER_JSON`` environment variable instead of passing +``--l2-adapter``: + +.. code-block:: bash + + export L2_ADAPTER_JSON='{"type":"fs","base_path":"/tmp/lmcache-bench"}' + lmcache bench l2 --num-keys 32 --in-flight 2 + + +Options +------- + +.. list-table:: + :header-rows: 1 + :widths: 30 15 55 + + * - Flag + - Default + - Description + * - ``--l2-adapter JSON`` + - *(unset)* + - L2 adapter spec as JSON with a ``"type"`` field plus + adapter-specific configs, e.g. + ``'{"type":"fs","base_path":"/tmp/bench"}'``. May be passed + multiple times; only the first spec is benchmarked. If not + provided, falls back to the ``L2_ADAPTER_JSON`` environment + variable. Either the flag or the env var is **required**. + * - ``--num-keys N`` + - ``32`` + - Number of keys per submit. + * - ``--in-flight N`` + - ``1`` + - In-flight submits per round. Each round issues this many + submits sequentially from a single producer thread, then waits + for all of them. + * - ``--data-size-kb N`` + - ``256`` + - Data size per key, in KiB. + * - ``--rounds N`` + - ``1`` + - Measurement rounds per operation. + * - ``--warmup-rounds N`` + - ``1`` + - Warmup rounds run before measurement; their results are + discarded. + * - ``--lookup-max-hit-rate F`` + - ``0.0`` + - Upper bound on the lookup hit rate, in ``[0, 1]``. The benchmark + requests ``floor(N * rate)`` keys from the + potentially-existing range and ``N - hit`` keys from a + guaranteed-non-existent range, where ``N`` is the total number + of lookup keys. The actual hit rate may be lower if those keys + were never stored in this run. + * - ``--skip-verify`` / ``--no-skip-verify`` + - ``--skip-verify`` + - Skip the store -> load round-trip data integrity check (the + default). Pass ``--no-skip-verify`` to enable verification on + the last measured round; this requires both ``store`` and + ``load`` to be exercised. + * - ``--only {lookup,store,load}`` + - *(unset)* + - Run only the specified operation. When omitted, all three + operations are run in the order ``store -> lookup -> load``. + + +Adapter JSON spec +----------------- + +The ``--l2-adapter`` JSON is parsed by +``lmcache.v1.distributed.l2_adapters.config.parse_args_to_l2_adapters_config``, +the same entry point LMCache uses everywhere else. The minimum required +field is ``type``; all remaining fields are forwarded to the adapter +implementation as keyword arguments. + +Example for the local-filesystem adapter: + +.. code-block:: json + + { + "type": "fs", + "base_path": "/data/lmcache-bench", + "relative_tmp_dir": "tmp", + "read_ahead_size": null, + "use_odirect": false + } + +See the source under ``lmcache/v1/distributed/l2_adapters/`` for the +full list of adapter types and their accepted fields. + + +Example output +-------------- + +Per-round progress (suppressed by ``-q`` if you wire it through): + +.. code-block:: text + + ============================================================ + L2 Adapter Benchmark + ============================================================ + Adapter config : FSL2AdapterConfig + L2 adapter JSON : {"type":"fs","base_path":"/data/lmcache-bench","relative_tmp_dir":"tmp"} + Keys / submit : 32 + In-flight / round : 3 + Keys / round : 96 + Data size / key : 256 KB + Data / round : 24.00 MB + Rounds : 1 (+ 1 warmup) + Lookup max hit rate : 0.00% + ============================================================ + + [Init] Creating adapter... + [Init] Adapter created successfully (FSL2Adapter). + + [Store] Running 1 warmup + 1 measurement rounds... + [Store] Round 1: 47.83 ms, success_keys=96/96 + [Store] Round 2: 46.19 ms, success_keys=96/96 + + [Lookup] Running 1 warmup + 1 measurement rounds... + [Lookup] Round 1: 5.36 ms, found=96/96 + [Lookup] Round 2: 5.03 ms, found=96/96 + + [Load] Running 1 warmup + 1 measurement rounds... + [Load] Round 1: 18.15 ms, loaded=96/96 + [Load] Round 2: 17.63 ms, loaded=96/96 + +Final summary (one section per exercised operation): + +.. code-block:: text + + ====== L2 Adapter Benchmark Result (FSL2Adapter) ======= + ----------------------- Configuration ------------------- + Adapter: FSL2Adapter + Keys / submit: 32 + In-flight / round: 3 + Data size / key (KB): 256 + Measurement rounds: 1 + Warmup rounds: 1 + Lookup max hit rate: 0.0 + --------------------------- Store ----------------------- + Operation: Store + Rounds: 1 + Keys / round: 96 + Total keys: 96 + Total success: 96 + Duration avg (ms): 46.19 + ... + Throughput avg (MB/s): 519.62 + Avg ops/s: 2078.50 + Avg latency / key (ms): 0.481 + --------------------------- Lookup ---------------------- + ... + ---------------------------- Load ----------------------- + ... + ========================================================= + +Each operation section reports per-round duration statistics +(avg / min / max / p50 / p99 / std), aggregate throughput +(``avg_throughput_mbps`` -- 0 for ``Lookup`` since it has no payload), +average key-rate (``avg_ops_per_sec``), and a per-key latency. + +For ``Lookup``, three additional fields are reported when +``--lookup-max-hit-rate`` is non-zero or some keys were found: + +* ``Expected max hit rate`` -- the configured upper bound. +* ``Expected hit keys`` -- ``floor(total_keys * rate)``, scaled for + the measured rounds only. +* ``Actual hit rate`` -- the measured hit rate over the kept rounds. + + +Round-trip verification +----------------------- + +When ``--no-skip-verify`` is passed and both ``store`` and ``load`` were +run, the benchmark compares the load buffers from the last measured +round against the byte pattern that ``store`` wrote (see +``make_memory_objects`` in +``lmcache/cli/commands/bench/l2_adapter_bench/data.py``): + +.. code-block:: text + + [Verify] Checking store -> load data integrity for last measured round... + [Verify] OK + +Verification is **off** by default because the stricter byte pattern +also forces every key to allocate its own ``data_size`` buffer +(otherwise the runner is free to reuse a single shared buffer across +keys to keep the memory footprint small). + + +Exit codes +---------- + +.. list-table:: + :header-rows: 1 + :widths: 15 85 + + * - Code + - Meaning + * - ``0`` + - All requested operations completed and (when enabled) the + round-trip verification passed. + * - ``1`` + - Adapter creation failed, round-trip verification failed, or + an operation hit a fatal error (e.g. all rounds timed out). + * - ``2`` + - The ``--l2-adapter`` JSON / ``L2_ADAPTER_JSON`` env var was + missing or could not be parsed. + + +See also +-------- + +* :doc:`bench` -- ``lmcache bench engine`` for engine-side workload + benchmarks. +* :doc:`bench_kvcache` -- end-to-end sanity test against an LMCache MP + server. +* :doc:`kvcache` -- ``lmcache kvcache`` for managing KV cache state on + a running server. diff --git a/docs/source/cli/index.rst b/docs/source/cli/index.rst index 5aad36b1f02..43b19d194a5 100644 --- a/docs/source/cli/index.rst +++ b/docs/source/cli/index.rst @@ -28,7 +28,8 @@ Available Commands - Liveness check for LMCache or vLLM servers. * - ``bench`` - Run sustained performance benchmarks against an inference engine, - or an end-to-end sanity test against an LMCache MP server. + an end-to-end sanity test against an LMCache MP server, or a + throughput/latency benchmark against an L2 cache adapter. * - ``kvcache`` - Manage KV cache state (e.g. clear L1 cache). * - ``server`` @@ -41,4 +42,5 @@ For a comprehensive guide with examples, see :doc:`/getting_started/cli`. bench bench_kvcache + bench_l2 kvcache diff --git a/lmcache/cli/commands/bench/__init__.py b/lmcache/cli/commands/bench/__init__.py index 0055411167b..7043ed92382 100644 --- a/lmcache/cli/commands/bench/__init__.py +++ b/lmcache/cli/commands/bench/__init__.py @@ -27,6 +27,10 @@ StatsCollector, ) from lmcache.cli.commands.bench.engine_bench.workloads import create_workload +from lmcache.cli.commands.bench.l2_adapter_bench.command import ( + register_l2_parser, + run_l2_adapter_bench, +) from lmcache.cli.commands.test_cache import TestCacheCommand from lmcache.logging import init_logger @@ -61,10 +65,12 @@ def register(self, subparsers: argparse._SubParsersAction) -> None: inner = parser.add_subparsers( dest="bench_target", required=True, - metavar="{engine,kvcache}", + metavar="{engine,kvcache,l2}", ) + # TODO(chunxiaozheng): move engine and kvcache to sub module too self._register_engine(inner) self._register_kvcache(inner) + register_l2_parser(inner, self.execute) def _register_engine( self, @@ -378,6 +384,7 @@ def execute(self, args: argparse.Namespace) -> None: handlers = { "engine": self._bench_engine, "kvcache": self._bench_kvcache, + "l2": lambda a: run_l2_adapter_bench(self, a), } handler = handlers.get(args.bench_target) if handler is None: diff --git a/lmcache/cli/commands/bench/l2_adapter_bench/__init__.py b/lmcache/cli/commands/bench/l2_adapter_bench/__init__.py new file mode 100644 index 00000000000..9f6dff5ba6d --- /dev/null +++ b/lmcache/cli/commands/bench/l2_adapter_bench/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +"""L2 adapter benchmark subpackage for ``lmcache bench l2``.""" diff --git a/lmcache/cli/commands/bench/l2_adapter_bench/command.py b/lmcache/cli/commands/bench/l2_adapter_bench/command.py new file mode 100644 index 00000000000..0befc1e0576 --- /dev/null +++ b/lmcache/cli/commands/bench/l2_adapter_bench/command.py @@ -0,0 +1,601 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``lmcache bench l2`` subcommand implementation. + +This module owns the full registration + execution flow for the L2 +adapter benchmark. ``BenchCommand`` only forwards CLI dispatch to +:func:`run_l2_adapter_bench` and parser registration to +:func:`register_l2_parser`. +""" + +# Future +from __future__ import annotations + +# Standard +from typing import TYPE_CHECKING +import argparse +import os +import sys + +# First Party +# Reuse the common helper that wires up ``--format / --output / +# --quiet`` onto a subparser. ``BenchCommand.register`` is overridden +# and creates inner subparsers manually, bypassing the auto-wiring +# that ``BaseCommand.register`` normally performs, so we attach those +# common flags ourselves only on the L2 subparser. The ``engine`` and +# ``kvcache`` subparsers intentionally stay untouched. +from lmcache.cli.commands.base import _add_output_args + +if TYPE_CHECKING: + # First Party + from lmcache.cli.commands.base import BaseCommand + from lmcache.cli.commands.bench.l2_adapter_bench.result import BenchResult + + +# --------------------------------------------------------------------------- +# Parser registration +# --------------------------------------------------------------------------- + + +def register_l2_parser( + subparsers: argparse._SubParsersAction, + dispatch_func, +) -> argparse.ArgumentParser: + """Register the ``lmcache bench l2`` subcommand parser. + + Args: + subparsers: The ``bench`` subparsers action. + dispatch_func: Function to bind via ``set_defaults(func=...)``. + Typically ``BenchCommand.execute`` so that the outer + dispatcher can route the call back into + :func:`run_l2_adapter_bench`. + + Returns: + The created ``ArgumentParser`` (mostly for testing). + """ + parser = subparsers.add_parser( + "l2", + help="Benchmark an L2 adapter (store / lookup / load).", + description=( + "Benchmark L2 adapters using the standard LMCache adapter " + "configuration mechanism (parse_args_to_l2_adapters_config " + "+ create_l2_adapter). Any registered adapter type can be " + "tested without code changes." + ), + ) + + parser.add_argument( + "--l2-adapter", + dest="l2_adapter", + action="append", + default=None, + type=str, + metavar="JSON", + help=( + 'L2 adapter spec as JSON with a "type" field and adapter-' + 'specific configs, e.g. \'{"type":"fs","base_path":"/tmp/' + "bench\"}'. If not provided, falls back to L2_ADAPTER_JSON " + "environment variable." + ), + ) + parser.add_argument( + "--num-keys", + type=int, + default=32, + help="Keys per submit (default: 32).", + ) + parser.add_argument( + "--in-flight", + type=int, + default=1, + help=( + "In-flight submits per round. Each round issues this many " + "submits sequentially from a single producer thread, then " + "waits for all of them (default: 1)." + ), + ) + parser.add_argument( + "--data-size-kb", + type=int, + default=256, + help="Data size per key in KB (default: 256).", + ) + parser.add_argument( + "--rounds", + type=int, + default=1, + help="Measurement rounds per operation (default: 1).", + ) + parser.add_argument( + "--warmup-rounds", + type=int, + default=1, + help="Warmup rounds before measurement (default: 1).", + ) + parser.add_argument( + "--lookup-max-hit-rate", + type=float, + default=0.0, + help=( + "Upper bound on the lookup hit rate, in [0, 1]. The " + "benchmark will request floor(N * rate) keys from the " + "potentially-existing range and (N - hit) keys from a " + "guaranteed-non-existent range, where N is the total " + "number of lookup keys (rounds * in_flight * num_keys). " + "The actual hit rate may be lower if those keys were " + "never stored. Default: 0.0." + ), + ) + # Round-trip verification is OFF by default (cheaper memory + # footprint: see make_memory_objects' share_buffer layout). + # Use --no-skip-verify to enable verification. + parser.add_argument( + "--skip-verify", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "Skip round-trip data verification (default). " + "Pass --no-skip-verify to enable verification." + ), + ) + parser.add_argument( + "--only", + choices=["lookup", "store", "load"], + default=None, + help="Run only the specified operation (default: run all).", + ) + + # Common ``--format / --output / --quiet`` flags. Attached only + # to the L2 subparser; the ``engine`` and ``kvcache`` subparsers + # intentionally keep their existing arguments unchanged. + _add_output_args(parser) + + parser.set_defaults(func=dispatch_func) + return parser + + +# --------------------------------------------------------------------------- +# Core benchmark runner +# --------------------------------------------------------------------------- + + +def run_l2_adapter_bench(command: "BaseCommand", args: argparse.Namespace) -> None: + """Run the L2 adapter benchmark. + + Args: + command: The owning :class:`BaseCommand` instance, used only + to obtain a configured :class:`Metrics` object via + ``command.create_metrics``. + args: Parsed CLI arguments from the ``bench l2`` subparser. + """ + # Lazy imports: keep CLI loadable without torch / native deps. + # Third Party + import torch + + # First Party + from lmcache.cli.commands.bench.l2_adapter_bench.data import ( + create_l1_memory_desc, + make_memory_objects, + make_object_keys, + verify_round_trip, + ) + from lmcache.cli.commands.bench.l2_adapter_bench.runner import ( + bench_load, + bench_lookup, + bench_store, + ) + from lmcache.v1.distributed.l2_adapters import create_l2_adapter + from lmcache.v1.distributed.l2_adapters.config import ( + parse_args_to_l2_adapters_config, + ) + + kb = 1024 + mb = 1024 * 1024 + data_size = args.data_size_kb * kb + in_flight = args.in_flight + num_keys = args.num_keys + rounds = args.rounds + warmup = args.warmup_rounds + total_rounds = warmup + rounds + max_hit_rate = max(0.0, min(1.0, args.lookup_max_hit_rate)) + quiet = getattr(args, "quiet", False) + + # Keys per round (one in-flight wave) and total measured keys per + # operation. Warmup rounds extend the consumed idx range. + keys_per_round = in_flight * num_keys + total_run_keys = total_rounds * keys_per_round # warmup + measured + + def log(msg: str) -> None: + # Per-round progress log; suppressed by --quiet. + if not quiet: + print(msg) + + # Resolve L2 adapter JSON: CLI arg takes priority, then env var + l2_adapter_specs = args.l2_adapter + if not l2_adapter_specs: + env_json = os.environ.get("L2_ADAPTER_JSON") + if env_json: + l2_adapter_specs = [env_json] + else: + print( + "Error: No L2 adapter configuration provided.\n" + "Use --l2-adapter JSON or set L2_ADAPTER_JSON " + "environment variable.", + file=sys.stderr, + ) + sys.exit(2) + + # Parse adapter config using the standard LMCache mechanism + ns = argparse.Namespace(l2_adapter=l2_adapter_specs) + try: + l2_cfg = parse_args_to_l2_adapters_config(ns) + except (ValueError, KeyError) as e: + print(f"Error parsing L2 adapter config: {e}", file=sys.stderr) + sys.exit(2) + + if not l2_cfg.adapters: + print("Error: no L2 adapter configs parsed", file=sys.stderr) + sys.exit(2) + + # Use the first adapter config for benchmarking + adapter_cfg = l2_cfg.adapters[0] + + # Backing L1 memory buffer for adapters that need an L1 desc. + # Sized for one in-flight wave of store + load buffers. + l1_buffer = torch.empty(2 * keys_per_round * data_size, dtype=torch.uint8) + l1_memory_desc = create_l1_memory_desc(l1_buffer) + + log("\n[Init] Creating adapter...") + try: + adapter = create_l2_adapter(adapter_cfg, l1_memory_desc=l1_memory_desc) + log(f"[Init] Adapter created successfully ({type(adapter).__name__}).\n") + except Exception as e: + print(f"[Init] Failed to create adapter: {e}", file=sys.stderr) + sys.exit(1) + + # ------------------------------------------------------------------ + # Idx layout + # ------------------------------------------------------------------ + # All ops live in the same idx universe so that ``--only store`` + # followed by ``--only load`` (or lookup) with the same flags hits + # the exact same keys. + # + # Round r (0-indexed, warmup rounds first) consumes the idx slice + # [r * keys_per_round, (r+1) * keys_per_round) + # split into ``in_flight`` contiguous batches of ``num_keys`` each. + # + # Lookup additionally splits each round into a hit-portion (drawn + # from the same idx range as store/load) and a miss-portion drawn + # from a guaranteed-non-existent range starting at + # ``total_run_keys``. + # ------------------------------------------------------------------ + + def _build_round_keys(r: int) -> list[list]: + """Build per-submit key batches for round *r* (store/load).""" + base = r * keys_per_round + return [ + make_object_keys(num_keys, key_offset=base + i * num_keys) + for i in range(in_flight) + ] + + def _build_round_objs() -> list[list]: + """Allocate per-submit object batches for one round. + + Every key in every batch gets its OWN ``data_size`` tensor, + pre-filled with a distinguishing byte pattern. This keeps + ``verify_round_trip`` meaningful (it can detect cross-key + corruption after a store -> load cycle) and keeps the + memory layout consistent regardless of whether verify is + actually run. + + Per-round (per direction) memory: + ``in_flight * num_keys * data_size`` bytes. + """ + return [make_memory_objects(num_keys, data_size) for _ in range(in_flight)] + + # Lookup hit/miss split per round. + per_round_hit = int(keys_per_round * max_hit_rate) + per_round_miss = keys_per_round - per_round_hit + # Total expected hit count over measured rounds only. + expected_hit_count = per_round_hit * rounds + # Origin of the guaranteed-miss idx range. + miss_origin = total_run_keys + + def _build_lookup_round_keys(r: int) -> list[list]: + """Build per-submit lookup key batches for round *r*. + + Hit slice for round r: + [r * per_round_hit, (r+1) * per_round_hit) + Miss slice for round r (disjoint from any store/load idx): + [miss_origin + r * per_round_miss, + miss_origin + (r+1) * per_round_miss) + + The combined ``keys_per_round`` keys are concatenated then + split into ``in_flight`` chunks of ``num_keys`` each. + """ + hit_base = r * per_round_hit + miss_base = miss_origin + r * per_round_miss + keys_round: list = [] + keys_round.extend(make_object_keys(per_round_hit, key_offset=hit_base)) + keys_round.extend(make_object_keys(per_round_miss, key_offset=miss_base)) + # Split into in_flight equal-sized batches of num_keys. + return [keys_round[i * num_keys : (i + 1) * num_keys] for i in range(in_flight)] + + # Per-direction object batches for store / load. + # + # Allocation strategy: + # * Lazy: only allocate when the corresponding direction is + # actually exercised. With ``--only store`` we never touch + # load buffers (and vice versa), saving + # ``in_flight * num_keys * data_size`` bytes of host memory. + # * Cross-round reuse: once allocated, the same batches are + # fed into every round; only the keys change per round. The + # L2 adapter does not care about object identity across + # rounds, and re-allocating these buffers each round would + # just be wasted work. + store_obj_batches: list[list] | None = None + load_obj_batches: list[list] | None = None + + def _store_objs(_r: int) -> list[list]: + nonlocal store_obj_batches + if store_obj_batches is None: + store_obj_batches = _build_round_objs() + return store_obj_batches + + def _load_objs(_r: int) -> list[list]: + nonlocal load_obj_batches + if load_obj_batches is None: + load_obj_batches = _build_round_objs() + return load_obj_batches + + results: list = [] + failed = False + + # Track the very last measured store round so we can verify it + # against the matching load round (round-trip integrity check). + last_store_round_keys: list[list] | None = None + last_load_round_keys: list[list] | None = None + + try: + # ---- Store ---- + if args.only is None or args.only == "store": + log(f"[Store] Running {warmup} warmup + {rounds} measurement rounds...") + all_store = bench_store( + adapter, + in_flight=in_flight, + num_keys=num_keys, + data_size=data_size, + rounds=total_rounds, + keys_for_round=_build_round_keys, + objs_for_round=_store_objs, + log=log, + ) + results.append(_strip_warmup(all_store, warmup)) + # Last measured store round is total_rounds - 1. + last_store_round_keys = _build_round_keys(total_rounds - 1) + log("") + + # ---- Lookup ---- + if args.only is None or args.only == "lookup": + log(f"[Lookup] Running {warmup} warmup + {rounds} measurement rounds...") + all_lookup = bench_lookup( + adapter, + in_flight=in_flight, + num_keys=num_keys, + rounds=total_rounds, + keys_for_round=_build_lookup_round_keys, + log=log, + expected_max_hit_rate=max_hit_rate, + expected_hit_count=expected_hit_count, + ) + results.append(_strip_warmup(all_lookup, warmup)) + log("") + + # ---- Load ---- + if args.only is None or args.only == "load": + log(f"[Load] Running {warmup} warmup + {rounds} measurement rounds...") + all_load = bench_load( + adapter, + in_flight=in_flight, + num_keys=num_keys, + data_size=data_size, + rounds=total_rounds, + keys_for_round=_build_round_keys, + objs_for_round=_load_objs, + log=log, + ) + results.append(_strip_warmup(all_load, warmup)) + last_load_round_keys = _build_round_keys(total_rounds - 1) + log("") + + # ---- Round-trip verification (last measured round only) ---- + if ( + not args.skip_verify + and store_obj_batches is not None + and load_obj_batches is not None + and last_store_round_keys is not None + and last_load_round_keys is not None + ): + # Sanity: store and load used the same key idx range for + # the last measured round, and load buffers now hold what + # the adapter returned. Compare against the byte pattern + # written by store (i & 0xFF, where i is position within + # the batch — see make_memory_objects). + log( + "[Verify] Checking store -> load data integrity for last " + "measured round..." + ) + flat_keys = [k for kl in last_load_round_keys for k in kl] + flat_store = [o for ol in store_obj_batches for o in ol] + flat_load = [o for ol in load_obj_batches for o in ol] + ok = verify_round_trip(flat_keys, flat_store, flat_load, log) + if not ok: + failed = True + log("") + + # ---- Summary via metrics system ---- + _emit_l2_adapter_metrics( + command=command, + args=args, + l2_adapter_json=l2_adapter_specs[0], + keys_per_round=keys_per_round, + data_per_round_mb=(keys_per_round * data_size) / mb, + results=results, + ) + finally: + log("[Cleanup] Closing adapter...") + try: + adapter.close() + except Exception as e: + print(f"[Cleanup] adapter.close() failed: {e}", file=sys.stderr) + log("[Cleanup] Done.") + + if failed: + sys.exit(1) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _strip_warmup(result: "BenchResult", warmup: int) -> "BenchResult": + """Drop the leading *warmup* rounds from a BenchResult.""" + # First Party + from lmcache.cli.commands.bench.l2_adapter_bench.result import BenchResult + + # Adjust the expected hit count proportionally for the kept rounds. + kept_rounds = max(0, len(result.round_durations) - warmup) + total_rounds = max(1, len(result.round_durations)) + scaled_expected_hit = int(result.expected_hit_count * kept_rounds / total_rounds) + + return BenchResult( + operation=result.operation, + in_flight=result.in_flight, + num_keys=result.num_keys, + data_size_bytes=result.data_size_bytes, + round_durations=result.round_durations[warmup:], + success_counts=result.success_counts[warmup:], + expected_max_hit_rate=result.expected_max_hit_rate, + expected_hit_count=scaled_expected_hit, + ) + + +def _emit_l2_adapter_metrics( + command: "BaseCommand", + args: argparse.Namespace, + l2_adapter_json: str, + keys_per_round: int, + data_per_round_mb: float, + results: list, +) -> None: + """Emit L2 adapter benchmark summary using the CLI metrics system.""" + title = "L2 Adapter Benchmark Result" + metrics = command.create_metrics(title, args, width=64) + + cfg_section = metrics.add_section("config", "Configuration") + cfg_section.add("l2_adapter_json", "L2 adapter JSON", l2_adapter_json) + cfg_section.add("num_keys", "Keys / submit", args.num_keys) + cfg_section.add("in_flight", "In-flight / round", args.in_flight) + cfg_section.add("keys_per_round", "Keys / round", keys_per_round) + cfg_section.add( + "data_size_kb", + "Data size / key (KB)", + args.data_size_kb, + ) + cfg_section.add( + "data_per_round_mb", + "Data / round (MB)", + round(data_per_round_mb, 2), + ) + cfg_section.add("measurement_rounds", "Measurement rounds", args.rounds) + cfg_section.add("warmup_rounds", "Warmup rounds", args.warmup_rounds) + # Only meaningful when lookup is actually executed; matches the + # original banner log behaviour. + if args.only is None or args.only == "lookup": + cfg_section.add( + "lookup_max_hit_rate", + "Lookup max hit rate", + round(args.lookup_max_hit_rate, 4), + ) + + for idx, r in enumerate(results): + section_id = f"op_{idx}" + section = metrics.add_section(section_id, r.operation) + section.add("operation", "Operation", r.operation) + section.add("rounds", "Rounds", len(r.round_durations)) + section.add("keys_per_round", "Keys / round", r.keys_per_round) + section.add("total_keys", "Total keys", r.total_keys) + section.add("total_success", "Total success", r.total_success) + section.add( + "duration_avg_ms", + "Duration avg (ms)", + round(r.avg_duration * 1000, 2), + ) + section.add( + "duration_min_ms", + "Duration min (ms)", + round(r.min_duration * 1000, 2), + ) + section.add( + "duration_max_ms", + "Duration max (ms)", + round(r.max_duration * 1000, 2), + ) + section.add( + "duration_p50_ms", + "Duration p50 (ms)", + round(r.p50_duration * 1000, 2), + ) + section.add( + "duration_p99_ms", + "Duration p99 (ms)", + round(r.p99_duration * 1000, 2), + ) + section.add( + "duration_std_ms", + "Duration std (ms)", + round(r.std_duration * 1000, 2), + ) + section.add( + "throughput_avg_mbps", + "Throughput avg (MB/s)", + round(r.avg_throughput_mbps, 2), + ) + section.add( + "throughput_min_mbps", + "Throughput min (MB/s)", + round(r.min_throughput_mbps, 2), + ) + section.add( + "throughput_max_mbps", + "Throughput max (MB/s)", + round(r.max_throughput_mbps, 2), + ) + section.add( + "ops_per_sec_avg", + "Avg ops/s", + round(r.avg_ops_per_sec, 2), + ) + section.add( + "latency_per_key_ms", + "Avg latency / key (ms)", + round(r.avg_latency_per_key_ms, 3), + ) + if r.expected_max_hit_rate > 0 or r.expected_hit_count > 0: + section.add( + "expected_max_hit_rate", + "Expected max hit rate", + round(r.expected_max_hit_rate, 4), + ) + section.add( + "expected_hit_count", + "Expected hit keys", + r.expected_hit_count, + ) + section.add( + "actual_hit_rate", + "Actual hit rate", + round(r.actual_hit_rate, 4), + ) + + metrics.emit() diff --git a/lmcache/cli/commands/bench/l2_adapter_bench/data.py b/lmcache/cli/commands/bench/l2_adapter_bench/data.py new file mode 100644 index 00000000000..9da2fb92423 --- /dev/null +++ b/lmcache/cli/commands/bench/l2_adapter_bench/data.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Test data construction helpers for L2 adapter benchmarks.""" + +# Future +from __future__ import annotations + +# Standard +import select + +# Third Party +import torch + +# First Party +from lmcache.v1.distributed.api import ObjectKey +from lmcache.v1.distributed.internal_api import L1MemoryDesc +from lmcache.v1.memory_management import ( + MemoryFormat, + MemoryObj, + MemoryObjMetadata, + TensorMemoryObj, +) +from lmcache.v1.platform import consume_fd + +_KB = 1024 + + +def make_object_keys( + num_keys: int, model_name: str = "bench-model", key_offset: int = 0 +) -> list[ObjectKey]: + """Generate *num_keys* unique ``ObjectKey`` instances for benchmarking. + + ``ObjectKey`` is a frozen dataclass with field order: + (chunk_hash, model_name, kv_rank). + + Args: + num_keys: Number of keys to generate. + model_name: Model name embedded in each key. + key_offset: Starting index offset to ensure uniqueness across threads. + """ + keys: list[ObjectKey] = [] + for i in range(num_keys): + idx = key_offset + i + # chunk_hash: 16 bytes derived from index to guarantee uniqueness + chunk_hash = idx.to_bytes(16, "big") + keys.append( + ObjectKey( + chunk_hash=chunk_hash, + model_name=model_name, + kv_rank=idx, + ) + ) + return keys + + +def make_memory_objects( + num_keys: int, + data_size: int, +) -> list[MemoryObj]: + """Create *num_keys* ``TensorMemoryObj`` instances of *data_size* bytes. + + Each returned object owns an independent ``data_size``-byte tensor + pre-filled with a distinguishing byte pattern (key index mod 256) + so that ``verify_round_trip`` can detect cross-key corruption after + a store -> load cycle. + + Per-call memory: ``num_keys * data_size``. + """ + # Independent buffers with distinguishing fill patterns for verify. + objects: list[MemoryObj] = [] + for i in range(num_keys): + raw_tensor = torch.empty(data_size, dtype=torch.uint8) + raw_tensor.fill_(i & 0xFF) + metadata = MemoryObjMetadata( + shape=torch.Size([data_size]), + dtype=torch.uint8, + address=raw_tensor.data_ptr(), + phy_size=data_size * raw_tensor.element_size(), + fmt=MemoryFormat.KV_2LTD, + ref_count=1, + ) + objects.append( + TensorMemoryObj( + raw_data=raw_tensor, + metadata=metadata, + parent_allocator=None, + ) + ) + return objects + + +def create_l1_memory_desc(buffer: torch.Tensor) -> L1MemoryDesc: + """Create an L1 memory descriptor for a contiguous test buffer.""" + flat_buffer = buffer.view(-1) + return L1MemoryDesc( + ptr=flat_buffer.data_ptr(), + size=flat_buffer.numel() * flat_buffer.element_size(), + align_bytes=flat_buffer.element_size(), + ) + + +def wait_eventfd(efd: int, timeout: float = 60.0) -> bool: + """Block until the eventfd is signalled or *timeout* seconds elapse. + + Uses ``select.poll`` + ``consume_fd`` for cross-platform compatibility. + + Returns True if the fd was signalled, False on timeout. + """ + poller = select.poll() + poller.register(efd, select.POLLIN) + # poll() expects timeout in milliseconds + events = poller.poll(timeout * 1000) + if events: + consume_fd(efd) + return True + return False + + +def verify_round_trip(keys, store_objects, load_objects, log) -> bool: + """Verify that loaded data matches what was stored. + + Compares the underlying ``raw_data`` tensors directly (more efficient + than converting via ``byte_array``). + """ + mismatches = 0 + for i, (s_obj, l_obj) in enumerate(zip(store_objects, load_objects, strict=True)): + if not torch.equal(s_obj.raw_data, l_obj.raw_data): + mismatches += 1 + log( + f" [Verify] Key {i}: MISMATCH " + f"(store {s_obj.get_physical_size()} bytes " + f"vs load {l_obj.get_physical_size()} bytes)" + ) + if mismatches == 0: + log(f" [Verify] All {len(keys)} keys data verified OK.") + return True + log(f" [Verify] {mismatches}/{len(keys)} keys have data mismatches!") + return False diff --git a/lmcache/cli/commands/bench/l2_adapter_bench/result.py b/lmcache/cli/commands/bench/l2_adapter_bench/result.py new file mode 100644 index 00000000000..eaadec06341 --- /dev/null +++ b/lmcache/cli/commands/bench/l2_adapter_bench/result.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Aggregated benchmark statistics for L2 adapter operations.""" + +# Future +from __future__ import annotations + +# Standard +from dataclasses import dataclass, field +import statistics + +_KB = 1024 +_MB = 1024 * 1024 + + +def _percentile(values: list[float], pct: float) -> float: + """Return the percentile *pct* (0..100) using nearest-rank. + + Returns 0.0 for an empty list. + """ + if not values: + return 0.0 + if len(values) == 1: + return values[0] + sorted_vals = sorted(values) + # Nearest-rank method: rank = ceil(pct/100 * N) + rank = max(1, int((pct / 100.0) * len(sorted_vals) + 0.999999)) + rank = min(rank, len(sorted_vals)) + return sorted_vals[rank - 1] + + +@dataclass +class BenchResult: + """Aggregated benchmark statistics for one operation type. + + Each measured *round* issues ``in_flight`` submits, where each submit + carries ``num_keys`` keys. ``round_durations[r]`` is the wall-clock + elapsed for the whole round (from issuing the first submit to all + submits of that round completing). + """ + + operation: str + in_flight: int + num_keys: int + data_size_bytes: int + round_durations: list[float] = field(default_factory=list) + success_counts: list[int] = field(default_factory=list) + # Lookup-specific metadata (left as defaults for store/load). + expected_max_hit_rate: float = 0.0 + expected_hit_count: int = 0 + + # ------------------------------------------------------------------ + # Derived counts + # ------------------------------------------------------------------ + + @property + def keys_per_round(self) -> int: + return self.in_flight * self.num_keys + + @property + def total_keys(self) -> int: + return self.keys_per_round * len(self.round_durations) + + @property + def total_data_bytes_per_round(self) -> int: + return self.keys_per_round * self.data_size_bytes + + @property + def total_data_bytes(self) -> int: + return self.total_keys * self.data_size_bytes + + @property + def total_success(self) -> int: + return sum(self.success_counts) + + # ------------------------------------------------------------------ + # Duration stats (seconds) + # ------------------------------------------------------------------ + + @property + def avg_duration(self) -> float: + return statistics.mean(self.round_durations) if self.round_durations else 0.0 + + @property + def min_duration(self) -> float: + return min(self.round_durations) if self.round_durations else 0.0 + + @property + def max_duration(self) -> float: + return max(self.round_durations) if self.round_durations else 0.0 + + @property + def std_duration(self) -> float: + if len(self.round_durations) > 1: + return statistics.stdev(self.round_durations) + return 0.0 + + @property + def p50_duration(self) -> float: + return _percentile(self.round_durations, 50.0) + + @property + def p99_duration(self) -> float: + return _percentile(self.round_durations, 99.0) + + # ------------------------------------------------------------------ + # Throughput stats (per-round, MB/s) + # ------------------------------------------------------------------ + + @property + def per_round_throughput_mbps(self) -> list[float]: + if self.data_size_bytes <= 0: + return [] + bytes_per_round = self.total_data_bytes_per_round + out: list[float] = [] + for d in self.round_durations: + if d <= 0: + out.append(float("inf")) + else: + out.append((bytes_per_round / _MB) / d) + return out + + @property + def avg_throughput_mbps(self) -> float: + vals = self.per_round_throughput_mbps + return statistics.mean(vals) if vals else 0.0 + + @property + def min_throughput_mbps(self) -> float: + vals = self.per_round_throughput_mbps + return min(vals) if vals else 0.0 + + @property + def max_throughput_mbps(self) -> float: + vals = self.per_round_throughput_mbps + return max(vals) if vals else 0.0 + + # ------------------------------------------------------------------ + # Ops/sec (key-rate) stats — useful for lookup which has no payload + # ------------------------------------------------------------------ + + @property + def per_round_ops_per_sec(self) -> list[float]: + out: list[float] = [] + for d in self.round_durations: + if d <= 0: + out.append(float("inf")) + else: + out.append(self.keys_per_round / d) + return out + + @property + def avg_ops_per_sec(self) -> float: + vals = self.per_round_ops_per_sec + return statistics.mean(vals) if vals else 0.0 + + # ------------------------------------------------------------------ + # Misc + # ------------------------------------------------------------------ + + @property + def avg_success_per_round(self) -> float: + return statistics.mean(self.success_counts) if self.success_counts else 0.0 + + @property + def avg_latency_per_key_ms(self) -> float: + if self.keys_per_round <= 0: + return 0.0 + return (self.avg_duration / self.keys_per_round) * 1000 + + @property + def actual_hit_rate(self) -> float: + if self.total_keys <= 0: + return 0.0 + return self.total_success / self.total_keys diff --git a/lmcache/cli/commands/bench/l2_adapter_bench/runner.py b/lmcache/cli/commands/bench/l2_adapter_bench/runner.py new file mode 100644 index 00000000000..378724182c4 --- /dev/null +++ b/lmcache/cli/commands/bench/l2_adapter_bench/runner.py @@ -0,0 +1,318 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Benchmark runners for L2 adapter ops. + +Each round issues ``in_flight`` submits sequentially from the calling +thread, then waits for ``in_flight`` eventfd notifications before +recording the round duration. This matches the real-world usage pattern +where multiple producers submit tasks and the L2 adapter's worker +coroutine processes them. + +The benchmark itself is single-threaded on the producer side; the +adapter internally is free to use threads / coroutines / async I/O. +""" + +# Future +from __future__ import annotations + +# Standard +from typing import Callable +import time + +# First Party +from lmcache.native_storage_ops import Bitmap +from lmcache.v1.distributed.api import ObjectKey +from lmcache.v1.memory_management import MemoryObj + +# Local +from .data import wait_eventfd +from .result import BenchResult + +# Logger callable type: takes a single string and prints / logs it. +LogFn = Callable[[str], None] + +# Provider callable signatures used by the runners. They are invoked at +# the start of every round and must return ``in_flight`` lists, one per +# in-flight submit, of length ``num_keys`` each. +KeyProvider = Callable[[int], list[list[ObjectKey]]] +ObjProvider = Callable[[int], list[list[MemoryObj]]] + + +def _bitmap_count(bitmap: Bitmap | None) -> int: + """Count how many bits are set in *bitmap*. Returns 0 when None.""" + if bitmap is None: + return 0 + return bitmap.popcount() + + +def _wait_store_finished( + adapter, task_ids: list[int], timeout: float +) -> dict[int, bool]: + """Wait for all store tasks to finish. + + Returns the accumulated ``{task_id: success}`` dict. + ``pop_completed_store_tasks`` consumes the adapter's completion + dict, so we must accumulate the results here for the caller to + use. On timeout, returns whatever was harvested so far (possibly + empty or partial); the caller can detect timeout by comparing + ``len(returned_dict)`` against ``len(task_ids)``. + """ + unfinished = len(task_ids) + efd = adapter.get_store_event_fd() + completed: dict[int, bool] = {} + while unfinished > 0: + if not wait_eventfd(efd, timeout=timeout): + return completed + batch = adapter.pop_completed_store_tasks() + completed.update(batch) + unfinished -= len(batch) + return completed + + +def _wait_load_finished( + adapter, task_ids: list[int], timeout: float +) -> dict[int, Bitmap]: + """Wait for all load tasks to finish. + + Returns ``{task_id: bitmap}``. ``query_load_result`` consumes the + per-task result, so we cache the bitmaps here for the caller. + Already-finished tasks are removed from the pending set so + subsequent wakeups don't re-query them. On timeout, returns + whatever was harvested so far; the caller can detect timeout by + comparing ``len(returned_dict)`` against ``len(task_ids)``. + """ + pending = set(task_ids) + efd = adapter.get_load_event_fd() + results: dict[int, Bitmap] = {} + while pending: + if not wait_eventfd(efd, timeout=timeout): + return results + for task_id in list(pending): + bitmap = adapter.query_load_result(task_id) + if bitmap is not None: + results[task_id] = bitmap + pending.remove(task_id) + return results + + +def _wait_lookup_finished( + adapter, task_ids: list[int], timeout: float +) -> dict[int, Bitmap]: + """Wait for all lookup-and-lock tasks to finish. + + Returns ``{task_id: bitmap}``. ``query_lookup_and_lock_result`` + consumes the per-task result, so we cache the bitmaps here for + the caller. Already-finished tasks are removed from the pending + set so subsequent wakeups don't re-query them. On timeout, + returns whatever was harvested so far; the caller can detect + timeout by comparing ``len(returned_dict)`` against + ``len(task_ids)``. + """ + pending = set(task_ids) + efd = adapter.get_lookup_and_lock_event_fd() + results: dict[int, Bitmap] = {} + while pending: + if not wait_eventfd(efd, timeout=timeout): + return results + for task_id in list(pending): + bitmap = adapter.query_lookup_and_lock_result(task_id) + if bitmap is not None: + results[task_id] = bitmap + pending.remove(task_id) + return results + + +# --------------------------------------------------------------------------- +# Store +# --------------------------------------------------------------------------- + + +def bench_store( + adapter, + in_flight: int, + num_keys: int, + data_size: int, + rounds: int, + keys_for_round: KeyProvider, + objs_for_round: ObjProvider, + log: LogFn, +) -> BenchResult: + """Benchmark ``submit_store_task``. + + For each round, ``in_flight`` independent submits are issued; the + round duration is the wall-clock time from the first submit until + every submit of that round has completed. + """ + result = BenchResult( + operation="Store", + in_flight=in_flight, + num_keys=num_keys, + data_size_bytes=data_size, + ) + + for r in range(rounds): + keys_batches = keys_for_round(r) + obj_batches = objs_for_round(r) + assert len(keys_batches) == in_flight + assert len(obj_batches) == in_flight + + t0 = time.perf_counter() + task_ids: list[int] = [] + for i in range(in_flight): + task_ids.append(adapter.submit_store_task(keys_batches[i], obj_batches[i])) + + completed = _wait_store_finished(adapter, task_ids, 120.0) + t1 = time.perf_counter() + elapsed = t1 - t0 + timed_out = len(completed) < len(task_ids) + + success_keys = sum( + len(keys_batches[i]) + for i, tid in enumerate(task_ids) + if completed.get(tid, False) + ) + + if timed_out: + log( + f" [Store] Round {r + 1}: TIMEOUT " + f"({len(completed)}/{len(task_ids)} tasks completed, " + f"success_keys={success_keys}/{in_flight * num_keys})" + ) + result.round_durations.append(float("inf")) + result.success_counts.append(success_keys) + continue + + result.round_durations.append(elapsed) + result.success_counts.append(success_keys) + log( + f" [Store] Round {r + 1}: {elapsed * 1000:.2f} ms, " + f"success_keys={success_keys}/{in_flight * num_keys}" + ) + + return result + + +# --------------------------------------------------------------------------- +# Lookup +# --------------------------------------------------------------------------- + + +def bench_lookup( + adapter, + in_flight: int, + num_keys: int, + rounds: int, + keys_for_round: KeyProvider, + log: LogFn, + expected_max_hit_rate: float = 0.0, + expected_hit_count: int = 0, +) -> BenchResult: + """Benchmark ``submit_lookup_and_lock_task``.""" + result = BenchResult( + operation="Lookup", + in_flight=in_flight, + num_keys=num_keys, + data_size_bytes=0, # lookup transfers no payload + expected_max_hit_rate=expected_max_hit_rate, + expected_hit_count=expected_hit_count, + ) + + for r in range(rounds): + keys_batches = keys_for_round(r) + assert len(keys_batches) == in_flight + + t0 = time.perf_counter() + task_ids: list[int] = [] + for i in range(in_flight): + task_ids.append(adapter.submit_lookup_and_lock_task(keys_batches[i])) + + results = _wait_lookup_finished(adapter, task_ids, 60.0) + t1 = time.perf_counter() + elapsed = t1 - t0 + timed_out = len(results) < len(task_ids) + + total_found = sum(_bitmap_count(results.get(tid)) for tid in task_ids) + + if timed_out: + log( + f" [Lookup] Round {r + 1}: TIMEOUT " + f"({len(results)}/{len(task_ids)} tasks completed, " + f"found={total_found}/{in_flight * num_keys})" + ) + result.round_durations.append(float("inf")) + result.success_counts.append(total_found) + continue + + result.round_durations.append(elapsed) + result.success_counts.append(total_found) + log( + f" [Lookup] Round {r + 1}: {elapsed * 1000:.2f} ms, " + f"found={total_found}/{in_flight * num_keys}" + ) + + return result + + +# --------------------------------------------------------------------------- +# Load +# --------------------------------------------------------------------------- + + +def bench_load( + adapter, + in_flight: int, + num_keys: int, + data_size: int, + rounds: int, + keys_for_round: KeyProvider, + objs_for_round: ObjProvider, + log: LogFn, +) -> BenchResult: + """Benchmark ``submit_load_task``.""" + result = BenchResult( + operation="Load", + in_flight=in_flight, + num_keys=num_keys, + data_size_bytes=data_size, + ) + + for r in range(rounds): + keys_batches = keys_for_round(r) + obj_batches = objs_for_round(r) + assert len(keys_batches) == in_flight + assert len(obj_batches) == in_flight + + # Reset all load buffers before each round to ensure fresh reads. + for objs in obj_batches: + for obj in objs: + obj.raw_data.zero_() + + t0 = time.perf_counter() + task_ids: list[int] = [] + for i in range(in_flight): + task_ids.append(adapter.submit_load_task(keys_batches[i], obj_batches[i])) + + results = _wait_load_finished(adapter, task_ids, 120.0) + t1 = time.perf_counter() + elapsed = t1 - t0 + timed_out = len(results) < len(task_ids) + + total_loaded = sum(_bitmap_count(results.get(tid)) for tid in task_ids) + + if timed_out: + log( + f" [Load] Round {r + 1}: TIMEOUT " + f"({len(results)}/{len(task_ids)} tasks completed, " + f"loaded={total_loaded}/{in_flight * num_keys})" + ) + result.round_durations.append(float("inf")) + result.success_counts.append(total_loaded) + continue + + result.round_durations.append(elapsed) + result.success_counts.append(total_loaded) + log( + f" [Load] Round {r + 1}: {elapsed * 1000:.2f} ms, " + f"loaded={total_loaded}/{in_flight * num_keys}" + ) + + return result From 2ba249dbf502e366de362170da69a6b73bba0c55 Mon Sep 17 00:00:00 2001 From: chunxiaozheng Date: Wed, 20 May 2026 15:12:22 +0800 Subject: [PATCH 41/69] [optimize] rename non_cuda_equivalents to python_ops_fallback (#3338) Signed-off-by: idellzheng --- docs/design/ARCHITECTURE_MULTI_HARDWARE.md | 4 ++-- lmcache/__init__.py | 4 ++-- ...on_cuda_equivalents.py => python_ops_fallback.py} | 0 tests/v1/test_c_ops_fallback_parity.py | 10 +++++----- ...da_equivalents.py => test_python_ops_fallback.py} | 12 ++++++------ 5 files changed, 15 insertions(+), 15 deletions(-) rename lmcache/{non_cuda_equivalents.py => python_ops_fallback.py} (100%) rename tests/v1/{test_non_cuda_equivalents.py => test_python_ops_fallback.py} (99%) diff --git a/docs/design/ARCHITECTURE_MULTI_HARDWARE.md b/docs/design/ARCHITECTURE_MULTI_HARDWARE.md index fb88412ccf5..e33c5e3ff15 100644 --- a/docs/design/ARCHITECTURE_MULTI_HARDWARE.md +++ b/docs/design/ARCHITECTURE_MULTI_HARDWARE.md @@ -69,7 +69,7 @@ │ │ • Layerwise │ │ • LayerwiseXPU │ │ │ │ │ │ • Buffer │ │ │ │ torch.hpu.* │ │ │ │ • SGLang │ │ torch.xpu.* │ │ │ │ -│ │ │ │ non_cuda_equiv │ │ │ │ +│ │ │ │ python_ops_fb │ │ │ │ │ │ torch.cuda.* │ │ │ │ │ │ │ │ c_ops + cupy │ │ │ │ │ │ │ └─────────────────┘ └─────────────────┘ └─────────────────┘ │ @@ -132,5 +132,5 @@ which is wrong for that backend's actual KV cache layout. 1. Add detection branch in `__init__.py` `_detect_device()` 2. Create `gpu_connector/xxx_connectors.py`, implement `GPUConnectorInterface` 3. Add routing branch in `gpu_connector/__init__.py` -4. Add kernels in `c_ops/` or fallback in `non_cuda_equivalents.py` +4. Add kernels in `c_ops/` or fallback in `python_ops_fallback.py` 5. No changes needed in middle layer code diff --git a/lmcache/__init__.py b/lmcache/__init__.py index 992a1268fb3..649fddb0c77 100644 --- a/lmcache/__init__.py +++ b/lmcache/__init__.py @@ -66,7 +66,7 @@ def _get_backend() -> Any: """ Try backends in order, first successful import wins. """ - default_module = importlib.import_module("lmcache.non_cuda_equivalents") + default_module = importlib.import_module("lmcache.python_ops_fallback") # Third Party import torch @@ -121,7 +121,7 @@ def _get_backend() -> Any: _ops = _get_backend() # override lmcache.c_ops with merged module, # in which: - # non_cuda_equivalents as base, + # python_ops_fallback as base, # use backend implementation if exists sys.modules["lmcache.c_ops"] = _ops except (ImportError, ModuleNotFoundError): diff --git a/lmcache/non_cuda_equivalents.py b/lmcache/python_ops_fallback.py similarity index 100% rename from lmcache/non_cuda_equivalents.py rename to lmcache/python_ops_fallback.py diff --git a/tests/v1/test_c_ops_fallback_parity.py b/tests/v1/test_c_ops_fallback_parity.py index 84d34333d15..b0b75c04f08 100644 --- a/tests/v1/test_c_ops_fallback_parity.py +++ b/tests/v1/test_c_ops_fallback_parity.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 """ -Verify that every public function/enum in non_cuda_equivalents that also +Verify that every public function/enum in python_ops_fallback that also exists in c_ops has a matching signature. -Does NOT require non_cuda_equivalents to implement everything in c_ops — +Does NOT require python_ops_fallback to implement everything in c_ops — only checks the intersection. If you implement a function in the fallback, its signature must match c_ops exactly. @@ -24,7 +24,7 @@ import pytest # First Party -import lmcache.non_cuda_equivalents as fallback +import lmcache.python_ops_fallback as fallback try: # First Party @@ -212,7 +212,7 @@ def _has_real_names(params): _shared_func_names if _shared_func_names else ["__placeholder__"], ) def test_function_signature_parity(func_name): - """For every function that non_cuda_equivalents chose to implement, + """For every function that python_ops_fallback chose to implement, its signature must match c_ops exactly. When c_ops has real py::arg() names → check names, count, defaults. @@ -291,7 +291,7 @@ def test_function_signature_parity(func_name): _shared_enum_names if _shared_enum_names else ["__placeholder__"], ) def test_enum_parity(enum_name): - """For every enum that non_cuda_equivalents defines, + """For every enum that python_ops_fallback defines, its members and values must match c_ops.""" if enum_name == "__placeholder__": pytest.skip("No shared enums found between c_ops and fallback") diff --git a/tests/v1/test_non_cuda_equivalents.py b/tests/v1/test_python_ops_fallback.py similarity index 99% rename from tests/v1/test_non_cuda_equivalents.py rename to tests/v1/test_python_ops_fallback.py index adef533cd32..a071ef6e17d 100644 --- a/tests/v1/test_non_cuda_equivalents.py +++ b/tests/v1/test_python_ops_fallback.py @@ -8,7 +8,7 @@ import torch # First Party -import lmcache.non_cuda_equivalents as _py_ops +import lmcache.python_ops_fallback as _py_ops # ========================================== # 0. utils functions. @@ -48,9 +48,9 @@ def _build_backend_params() -> list: Returns one entry per available backend configuration: - cuda_c_ops: uses lmcache.c_ops (requires CUDA and the CUDA extension) - - cuda_py_ops: uses lmcache.non_cuda_equivalents with GPU visible - - cpy_py_ops: uses lmcache.non_cuda_equivalents with GPU mocked away - - xpu_py_ops: uses lmcache.non_cuda_equivalents with XPU visible + - cuda_py_ops: uses lmcache.python_ops_fallback with GPU visible + - cpy_py_ops: uses lmcache.python_ops_fallback with GPU mocked away + - xpu_py_ops: uses lmcache.python_ops_fallback with XPU visible """ params = [] cuda_available = torch.cuda.is_available() @@ -1773,8 +1773,8 @@ def test_1_scenario( def test_2_compare(self, name: str) -> None: """Compare results across backends for a single scenario. - When multiple backends ran (CUDA available), asserts that non_cuda - equivalents produce numerically identical results to cuda_ops. + When multiple backends ran (CUDA available), asserts that python + fallback equivalents produce numerically identical results to cuda_ops. When only one backend ran (no CUDA), simply verifies results were stored. This test runs after test_1_scenario due to alphabetical ordering of From 2ce4f5f2e263a81149518377e27556ecb7c14461 Mon Sep 17 00:00:00 2001 From: maobaolong Date: Wed, 20 May 2026 15:55:19 +0800 Subject: [PATCH 42/69] Refactor: abstract discover_subclasses util method (#3237) * Refactor: abstract discover_subclasses util method Signed-off-by: baoloongmao * Fix Signed-off-by: baoloongmao * Fix Signed-off-by: baoloongmao * Fix Signed-off-by: baoloongmao * Fix Signed-off-by: baoloongmao * Fix Signed-off-by: baoloongmao * Raise exception on_import_error Signed-off-by: baoloongmao * Move test_cache CLI module under bench sub-package TestCacheCommand is the impl behind 'bench kvcache', not a top-level CLI verb. Putting test_cache.py under lmcache/cli/commands/ caused discover_subclasses to surface it as a stand-alone command. Move the module (and its tests) into lmcache/cli/commands/bench/ so it sits next to the engine_bench peer and is naturally invisible to the top-level scan. Signed-off-by: baoloongmao --------- Signed-off-by: baoloongmao --- lmcache/cli/commands/__init__.py | 56 +-- lmcache/cli/commands/bench/__init__.py | 4 +- .../cli/commands/{ => bench}/test_cache.py | 0 .../controller_benchmark/handlers/__init__.py | 37 +- lmcache/v1/health_monitor/base.py | 61 ++-- .../record_strategies/__init__.py | 28 +- .../v1/storage_backend/connector/__init__.py | 46 +-- lmcache/v1/utils/subclass_discovery.py | 106 ++++++ .../commands/{ => bench}/test_test_cache.py | 13 +- tests/v1/test_connector_discovery.py | 5 + tests/v1/test_subclass_discovery.py | 330 ++++++++++++++++++ 11 files changed, 541 insertions(+), 145 deletions(-) rename lmcache/cli/commands/{ => bench}/test_cache.py (100%) create mode 100644 lmcache/v1/utils/subclass_discovery.py rename tests/cli/commands/{ => bench}/test_test_cache.py (96%) create mode 100644 tests/v1/test_subclass_discovery.py diff --git a/lmcache/cli/commands/__init__.py b/lmcache/cli/commands/__init__.py index 115426b37b2..f39f3ad2c19 100644 --- a/lmcache/cli/commands/__init__.py +++ b/lmcache/cli/commands/__init__.py @@ -1,34 +1,40 @@ # SPDX-License-Identifier: Apache-2.0 """CLI subcommand package. -To add a new command: - -1. Create a module with a :class:`BaseCommand` subclass. -2. Add one import + one entry to :data:`ALL_COMMANDS` below. +To add a new top-level command, simply create a new module (or sub-package +with an ``__init__.py``) under this package that defines a concrete +:class:`BaseCommand` subclass. It will be discovered and registered +automatically — no edits to this file are required. """ # First Party from lmcache.cli.commands.base import BaseCommand -from lmcache.cli.commands.bench import BenchCommand -from lmcache.cli.commands.describe import DescribeCommand -from lmcache.cli.commands.kvcache import KVCacheCommand -from lmcache.cli.commands.mock import MockCommand -from lmcache.cli.commands.ping import PingCommand -from lmcache.cli.commands.query import QueryCommand -from lmcache.cli.commands.server import ServerCommand -from lmcache.cli.commands.tool import ToolCommand -from lmcache.cli.commands.trace import TraceCommand - -ALL_COMMANDS: list[BaseCommand] = [ - MockCommand(), - KVCacheCommand(), - DescribeCommand(), - PingCommand(), - QueryCommand(), - ServerCommand(), - BenchCommand(), - ToolCommand(), - TraceCommand(), -] +from lmcache.v1.utils.subclass_discovery import discover_subclasses + + +def _discover_commands() -> list[BaseCommand]: + """Scan direct submodules of this package and collect all concrete + :class:`BaseCommand` subclasses, returning one instance per class. + + Import errors are intentionally re-raised: a broken CLI command + module should fail loudly rather than silently disappear from the + CLI. + """ + + def _raise(module_name: str, exc: Exception) -> None: + raise exc + + return [ + cls() + for cls in discover_subclasses( + __name__, + BaseCommand, # type: ignore[type-abstract] + module_filter=lambda name: name != "base", + on_import_error=_raise, + ) + ] + + +ALL_COMMANDS: list[BaseCommand] = _discover_commands() __all__ = ["ALL_COMMANDS", "BaseCommand"] diff --git a/lmcache/cli/commands/bench/__init__.py b/lmcache/cli/commands/bench/__init__.py index 7043ed92382..fbb7c2f47cd 100644 --- a/lmcache/cli/commands/bench/__init__.py +++ b/lmcache/cli/commands/bench/__init__.py @@ -8,8 +8,8 @@ import sys # First Party -from lmcache.cli.commands import test_cache as _test_cache_mod from lmcache.cli.commands.base import BaseCommand +from lmcache.cli.commands.bench import test_cache as _test_cache_mod from lmcache.cli.commands.bench.engine_bench.config import ( EngineBenchConfig, parse_args_to_config, @@ -31,7 +31,7 @@ register_l2_parser, run_l2_adapter_bench, ) -from lmcache.cli.commands.test_cache import TestCacheCommand +from lmcache.cli.commands.bench.test_cache import TestCacheCommand from lmcache.logging import init_logger logger = init_logger(__name__) diff --git a/lmcache/cli/commands/test_cache.py b/lmcache/cli/commands/bench/test_cache.py similarity index 100% rename from lmcache/cli/commands/test_cache.py rename to lmcache/cli/commands/bench/test_cache.py diff --git a/lmcache/tools/controller_benchmark/handlers/__init__.py b/lmcache/tools/controller_benchmark/handlers/__init__.py index a382dfb3955..ac65a9e99d9 100644 --- a/lmcache/tools/controller_benchmark/handlers/__init__.py +++ b/lmcache/tools/controller_benchmark/handlers/__init__.py @@ -4,9 +4,9 @@ # Standard from typing import Dict -import importlib -import inspect -import pkgutil + +# First Party +from lmcache.v1.utils.subclass_discovery import discover_subclasses # Local from .base import OperationHandler @@ -17,29 +17,14 @@ def _discover_and_register_handlers(): """Dynamically discover and register all operation handlers""" - # Get the current package - package = __package__ - package_path = __path__ - - # Iterate through all modules in the handlers package - for _, module_name, _ in pkgutil.iter_modules(package_path): - # Skip base module - if module_name == "base": - continue - - # Import the module - module = importlib.import_module("." + module_name, package=package) - - # Find all classes that inherit from OperationHandler - for name, obj in inspect.getmembers(module, inspect.isclass): - if ( - issubclass(obj, OperationHandler) - and obj is not OperationHandler - and not inspect.isabstract(obj) - ): - # Instantiate and register the handler - handler = obj() - OPERATION_HANDLERS[handler.operation_name] = handler + for cls in discover_subclasses( + __name__, + OperationHandler, + module_filter=lambda name: name != "base", + require_defined_in_module=False, + ): + handler = cls() + OPERATION_HANDLERS[handler.operation_name] = handler # Auto-discover and register handlers on import diff --git a/lmcache/v1/health_monitor/base.py b/lmcache/v1/health_monitor/base.py index 4663ac72b0a..8b4fe2ca947 100644 --- a/lmcache/v1/health_monitor/base.py +++ b/lmcache/v1/health_monitor/base.py @@ -22,6 +22,7 @@ ThreadLevel, ThreadRunSummary, ) +from lmcache.v1.utils.subclass_discovery import discover_subclasses if TYPE_CHECKING: # First Party @@ -231,47 +232,27 @@ def _discover_health_checks(self) -> None: finds all HealthCheck subclasses and calls their `create()` method to create instances. """ - # Standard - import importlib - import inspect - import pkgutil - - # First Party - # Import the checks package - import lmcache.v1.health_monitor.checks as checks_pkg - - # Discover all modules in the checks package - for _, module_name, _ in pkgutil.iter_modules(checks_pkg.__path__): - # Skip private modules - if module_name.startswith("_"): - continue - + cls: type[HealthCheck] + for cls in discover_subclasses( + "lmcache.v1.health_monitor.checks", + HealthCheck, # type: ignore[type-abstract] + module_filter=lambda name: not name.startswith("_"), + include_abstract=True, + require_defined_in_module=False, + ): try: - module = importlib.import_module(f"{checks_pkg.__name__}.{module_name}") - - # Find all HealthCheck subclasses in the module - for _, obj in inspect.getmembers(module): - if ( - inspect.isclass(obj) - and issubclass(obj, HealthCheck) - and obj != HealthCheck - ): - try: - instances = obj.create(self._manager) - for instance in instances: - self._health_checks.append(instance) - # Initialize previous status as healthy - self._previous_check_status[instance.name()] = True - logger.info( - f"Registered health check: {instance.name()} " - f"with fallback_policy: {instance.fallback_policy}" - ) - except Exception as e: - logger.warning( - f"Failed to create health check {obj.__name__}: {e}" - ) - except ImportError as e: - logger.warning(f"Failed to import module {module_name}: {e}") + instances = cls.create(self._manager) + except Exception as e: + logger.warning(f"Failed to create health check {cls.__name__}: {e}") + continue + for instance in instances: + self._health_checks.append(instance) + # Initialize previous status as healthy + self._previous_check_status[instance.name()] = True + logger.info( + f"Registered health check: {instance.name()} " + f"with fallback_policy: {instance.fallback_policy}" + ) def get_health_checks(self) -> List[HealthCheck]: """Get all registered health checks""" diff --git a/lmcache/v1/lookup_client/record_strategies/__init__.py b/lmcache/v1/lookup_client/record_strategies/__init__.py index 479c7792467..2cbb00d9b57 100644 --- a/lmcache/v1/lookup_client/record_strategies/__init__.py +++ b/lmcache/v1/lookup_client/record_strategies/__init__.py @@ -2,9 +2,6 @@ # Standard from typing import Dict, Type -import importlib -import inspect -import pkgutil # First Party from lmcache.logging import init_logger @@ -12,27 +9,22 @@ AsyncRecorder, RecordStrategy, ) +from lmcache.v1.utils.subclass_discovery import discover_subclasses logger = init_logger(__name__) def _discover_strategies() -> Dict[str, Type[RecordStrategy]]: - strategies = {} - # First Party - from lmcache.v1.lookup_client import record_strategies - - for importer, modname, ispkg in pkgutil.iter_modules( - record_strategies.__path__, record_strategies.__name__ + "." + strategies: Dict[str, Type[RecordStrategy]] = {} + for cls in discover_subclasses( + __name__, + RecordStrategy, # type: ignore[type-abstract] + include_abstract=True, + on_import_error=lambda mod, exc: None, ): - try: - module = importlib.import_module(modname) - for name, obj in inspect.getmembers(module, inspect.isclass): - if issubclass(obj, RecordStrategy) and obj is not RecordStrategy: - # Use module name as strategy name - strategy_name = modname.split(".")[-1] - strategies[strategy_name] = obj - except Exception: - continue + # Use module short name as strategy name + strategy_name = cls.__module__.rsplit(".", 1)[-1] + strategies[strategy_name] = cls return strategies diff --git a/lmcache/v1/storage_backend/connector/__init__.py b/lmcache/v1/storage_backend/connector/__init__.py index 625b414b4c6..ac78efc2fa0 100644 --- a/lmcache/v1/storage_backend/connector/__init__.py +++ b/lmcache/v1/storage_backend/connector/__init__.py @@ -7,7 +7,6 @@ import asyncio import importlib import inspect -import pkgutil # First Party from lmcache.logging import init_logger @@ -18,6 +17,7 @@ InstrumentedRemoteConnector, ) from lmcache.v1.storage_backend.local_cpu_backend import LocalCPUBackend +from lmcache.v1.utils.subclass_discovery import discover_subclasses logger = init_logger(__name__) @@ -243,39 +243,19 @@ def __init__( def _remote_adapters_builtin_launcher(self) -> None: """Automatically load all builtin remote connector adapters.""" - # Import current package to ensure all modules are loaded - # First Party - import lmcache.v1.storage_backend.connector as connector_pkg - - # Discover all modules in the connector package - for _, module_name, _ in pkgutil.iter_modules(connector_pkg.__path__): - # Skip private modules and non-adapter modules - if module_name.startswith("_") or not module_name.endswith("_adapter"): - continue - + for cls in discover_subclasses( + "lmcache.v1.storage_backend.connector", + ConnectorAdapter, # type: ignore[type-abstract] + module_filter=lambda name: ( + not name.startswith("_") and name.endswith("_adapter") + ), + require_defined_in_module=False, + ): try: - module = importlib.import_module( - f"{connector_pkg.__name__}.{module_name}" - ) - - # Find all ConnectorAdapter subclasses in the module - for _, obj in inspect.getmembers(module): - if ( - inspect.isclass(obj) - and issubclass(obj, ConnectorAdapter) - and obj != ConnectorAdapter - ): - try: - adapter_instance = obj() - self.adapters.append(adapter_instance) - logger.info(f"Discovered adapter: {obj.__name__}") - except Exception as e: - logger.error( - "Failed to instantiate adapter " - f"{obj.__name__}: {str(e)}" - ) - except ImportError as e: - logger.warning(f"Failed to import module {module_name}: {e}") + self.adapters.append(cls()) + logger.info(f"Discovered adapter: {cls.__name__}") + except Exception as e: + logger.error(f"Failed to instantiate adapter {cls.__name__}: {str(e)}") def _remote_adapters_plugin_launcher(self, config: LMCacheEngineConfig) -> None: """Automatically load all plug and play remote connector adapters.""" diff --git a/lmcache/v1/utils/subclass_discovery.py b/lmcache/v1/utils/subclass_discovery.py new file mode 100644 index 00000000000..bb7c6905e8e --- /dev/null +++ b/lmcache/v1/utils/subclass_discovery.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Generic helper for plugin-style auto-discovery of concrete subclasses. + +This module centralises the boilerplate that several packages used to +duplicate (CLI commands, controller-benchmark handlers, lookup-client +record strategies, health-monitor checks, remote connector adapters, +...). Each of those packages walks its own submodules with +``pkgutil.iter_modules``, imports them, then iterates classes via +``inspect.getmembers`` to locate concrete subclasses of a given base +class. ``discover_subclasses`` captures that pattern in a single +well-tested place so callers stay tiny and behave consistently. +""" + +# Standard +from types import ModuleType +from typing import Callable, Iterator, Optional, TypeVar, Union +import importlib +import inspect +import pkgutil + +# First Party +from lmcache.logging import init_logger + +logger = init_logger(__name__) + +T = TypeVar("T") + + +def _resolve_package(package: Union[ModuleType, str]) -> ModuleType: + if isinstance(package, str): + return importlib.import_module(package) + return package + + +def discover_subclasses( + package: Union[ModuleType, str], + base_class: type[T], + *, + module_filter: Optional[Callable[[str], bool]] = None, + include_abstract: bool = False, + require_defined_in_module: bool = True, + on_import_error: Optional[Callable[[str, Exception], None]] = None, +) -> Iterator[type[T]]: + """Yield concrete subclasses of *base_class* found in direct + submodules of *package*. + + Each subclass is yielded **at most once**, even when re-exported + from several modules. + + Args: + package: The package to scan, either as a module object or its + fully-qualified dotted name. + base_class: The base class whose concrete subclasses to collect. + module_filter: Optional predicate over the *short* module name + (i.e. without the package prefix). Modules for which the + predicate returns ``False`` are skipped. Defaults to + ``None`` which keeps every module. + include_abstract: When ``False`` (default) classes with + unimplemented abstract methods are skipped. + require_defined_in_module: When ``True`` (default) classes that + were merely imported (re-exported) into a module are + ignored; only classes whose ``__module__`` matches the + module being scanned are yielded. Set to ``False`` to keep + the historical behaviour of accepting re-exported classes. + on_import_error: Optional callback invoked as + ``on_import_error(full_module_name, exc)`` when a submodule + fails to import. When omitted the error is logged at + ``warning`` level and discovery proceeds with the next + module. + """ + pkg = _resolve_package(package) + pkg_path = getattr(pkg, "__path__", None) + if pkg_path is None: + raise TypeError( + "discover_subclasses requires a package (with __path__), got %r" % (pkg,) + ) + + seen: set[type] = set() + for _, short_name, _ in pkgutil.iter_modules(pkg_path): + if module_filter is not None and not module_filter(short_name): + continue + full_name = "%s.%s" % (pkg.__name__, short_name) + try: + module = importlib.import_module(full_name) + except Exception as exc: + if on_import_error is not None: + on_import_error(full_name, exc) + else: + logger.warning( + "Failed to import module %s during subclass discovery: %s", + full_name, + exc, + ) + continue + + for _, obj in inspect.getmembers(module, inspect.isclass): + if not issubclass(obj, base_class) or obj is base_class: + continue + if not include_abstract and inspect.isabstract(obj): + continue + if require_defined_in_module and obj.__module__ != module.__name__: + continue + if obj in seen: + continue + seen.add(obj) + yield obj diff --git a/tests/cli/commands/test_test_cache.py b/tests/cli/commands/bench/test_test_cache.py similarity index 96% rename from tests/cli/commands/test_test_cache.py rename to tests/cli/commands/bench/test_test_cache.py index 8b3773c7aaf..e3aaedb7c99 100644 --- a/tests/cli/commands/test_test_cache.py +++ b/tests/cli/commands/bench/test_test_cache.py @@ -21,7 +21,7 @@ # First Party from lmcache.cli.commands.bench import BenchCommand -from lmcache.cli.commands.test_cache import ( +from lmcache.cli.commands.bench.test_cache import ( _allocate_gpu_kv_cache, _build_token_ids, _make_key, @@ -63,6 +63,17 @@ def test_name(self, cmd: BenchCommand) -> None: def test_help(self, cmd: BenchCommand) -> None: assert "benchmark" in cmd.help().lower() + def test_test_cache_lives_under_bench_package(self) -> None: + """``test_cache`` is the impl behind ``bench kvcache`` and must + live inside the ``bench`` sub-package, not at the top-level CLI + commands package — otherwise auto-discovery would expose + ``TestCacheCommand`` as a stand-alone verb. + """ + # First Party + from lmcache.cli.commands.bench.test_cache import TestCacheCommand + + assert TestCacheCommand.__module__ == ("lmcache.cli.commands.bench.test_cache") + # ------------------------------------------------------------------ # # Argument registration diff --git a/tests/v1/test_connector_discovery.py b/tests/v1/test_connector_discovery.py index 54871f97256..8b899c569dc 100644 --- a/tests/v1/test_connector_discovery.py +++ b/tests/v1/test_connector_discovery.py @@ -44,6 +44,9 @@ def create_connector(self, context): non_adapter_module = ModuleType("non_adapter") non_adapter_module.NotAnAdapter = object + fake_package = ModuleType("lmcache.v1.storage_backend.connector") + fake_package.__path__ = [] # type: ignore[attr-defined] + def fake_iter_modules(_path): """Yield a mix of adapter and non-adapter module names.""" yield None, "good_adapter", False @@ -54,6 +57,8 @@ def fake_iter_modules(_path): def fake_import_module(name): """Return fake modules or simulate import failure.""" + if name == "lmcache.v1.storage_backend.connector": + return fake_package if name.endswith(".good_adapter"): return good_module if name.endswith(".broken_adapter"): diff --git a/tests/v1/test_subclass_discovery.py b/tests/v1/test_subclass_discovery.py new file mode 100644 index 00000000000..84398078894 --- /dev/null +++ b/tests/v1/test_subclass_discovery.py @@ -0,0 +1,330 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for ``lmcache.v1.utils.subclass_discovery``.""" + +# Standard +from pathlib import Path +from typing import Iterable, List, Tuple +import importlib +import sys +import textwrap +import uuid + +# Third Party +import pytest + +# First Party +from lmcache.v1.utils.subclass_discovery import discover_subclasses + + +def _write_module(pkg_dir: Path, name: str, source: str) -> None: + (pkg_dir / f"{name}.py").write_text(textwrap.dedent(source)) + + +@pytest.fixture +def temp_pkg(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Create a fresh, importable Python package on disk and yield its + fully-qualified name plus its filesystem path. + + The fixture also takes care of cleaning ``sys.modules`` so each test + starts from a clean slate even when reusing the same Python + interpreter. + """ + pkg_name = f"_subclass_discovery_pkg_{uuid.uuid4().hex}" + pkg_dir = tmp_path / pkg_name + pkg_dir.mkdir() + (pkg_dir / "__init__.py").write_text("") + + # Make the parent dir importable. + monkeypatch.syspath_prepend(str(tmp_path)) + + yield pkg_name, pkg_dir + + # Drop every module that the test imported under this package so + # subsequent runs cannot observe stale state. + for mod_name in [ + m for m in list(sys.modules) if m == pkg_name or m.startswith(pkg_name + ".") + ]: + sys.modules.pop(mod_name, None) + + +def _populate_basic_pkg(pkg_dir: Path) -> None: + """A minimal layout used by the majority of the tests: + base.py defines Base + AbstractChild, child_a.py & child_b.py define + one concrete subclass each. + """ + _write_module( + pkg_dir, + "base", + """ + from abc import ABC, abstractmethod + + class Base(ABC): + @abstractmethod + def name(self) -> str: ... + + class AbstractChild(Base): + # Still abstract: does not implement name(). + pass + """, + ) + _write_module( + pkg_dir, + "child_a", + """ + from .base import Base + + class ChildA(Base): + def name(self) -> str: + return "a" + """, + ) + _write_module( + pkg_dir, + "child_b", + """ + from .base import Base + + class ChildB(Base): + def name(self) -> str: + return "b" + """, + ) + + +def _names(classes: Iterable[type]) -> List[str]: + return sorted(c.__name__ for c in classes) + + +class TestDiscoverSubclassesBasic: + def test_finds_concrete_subclasses(self, temp_pkg: Tuple[str, Path]) -> None: + pkg_name, pkg_dir = temp_pkg + _populate_basic_pkg(pkg_dir) + + base_mod = importlib.import_module(f"{pkg_name}.base") + + result = list( + discover_subclasses( + pkg_name, + base_mod.Base, + module_filter=lambda n: n != "base", + ) + ) + + assert _names(result) == ["ChildA", "ChildB"] + + def test_accepts_module_object(self, temp_pkg: Tuple[str, Path]) -> None: + pkg_name, pkg_dir = temp_pkg + _populate_basic_pkg(pkg_dir) + + pkg_mod = importlib.import_module(pkg_name) + base_mod = importlib.import_module(f"{pkg_name}.base") + + result = list( + discover_subclasses( + pkg_mod, + base_mod.Base, + module_filter=lambda n: n != "base", + ) + ) + + assert _names(result) == ["ChildA", "ChildB"] + + def test_base_class_itself_is_skipped(self, temp_pkg: Tuple[str, Path]) -> None: + pkg_name, pkg_dir = temp_pkg + _populate_basic_pkg(pkg_dir) + base_mod = importlib.import_module(f"{pkg_name}.base") + + # Even when scanning base.py the Base class itself is not yielded. + result = list( + discover_subclasses( + pkg_name, + base_mod.Base, + include_abstract=True, + ) + ) + + assert base_mod.Base not in result + + +class TestAbstractFiltering: + def test_skips_abstract_by_default(self, temp_pkg: Tuple[str, Path]) -> None: + pkg_name, pkg_dir = temp_pkg + _populate_basic_pkg(pkg_dir) + base_mod = importlib.import_module(f"{pkg_name}.base") + + result = list(discover_subclasses(pkg_name, base_mod.Base)) + + # AbstractChild lives in base.py; with default filter + # (include_abstract=False) it must be excluded. + assert "AbstractChild" not in _names(result) + assert _names(result) == ["ChildA", "ChildB"] + + def test_include_abstract_keeps_them(self, temp_pkg: Tuple[str, Path]) -> None: + pkg_name, pkg_dir = temp_pkg + _populate_basic_pkg(pkg_dir) + base_mod = importlib.import_module(f"{pkg_name}.base") + + result = list( + discover_subclasses( + pkg_name, + base_mod.Base, + include_abstract=True, + ) + ) + assert "AbstractChild" in _names(result) + + +class TestModuleFilter: + def test_module_filter_skips_modules(self, temp_pkg: Tuple[str, Path]) -> None: + pkg_name, pkg_dir = temp_pkg + _populate_basic_pkg(pkg_dir) + base_mod = importlib.import_module(f"{pkg_name}.base") + + result = list( + discover_subclasses( + pkg_name, + base_mod.Base, + module_filter=lambda n: n == "child_a", + ) + ) + assert _names(result) == ["ChildA"] + + +class TestReExportHandling: + def test_default_excludes_reexports(self, temp_pkg: Tuple[str, Path]) -> None: + """A class re-exported from another module must not be yielded + twice when ``require_defined_in_module=True`` (the default).""" + pkg_name, pkg_dir = temp_pkg + _populate_basic_pkg(pkg_dir) + # extra.py only re-imports ChildA; it does not define a new class. + _write_module( + pkg_dir, + "extra", + """ + from .child_a import ChildA # re-export + """, + ) + base_mod = importlib.import_module(f"{pkg_name}.base") + + result = list( + discover_subclasses( + pkg_name, + base_mod.Base, + module_filter=lambda n: n != "base", + ) + ) + # Each class appears at most once - the re-export does not + # cause duplication. + assert _names(result) == ["ChildA", "ChildB"] + + def test_disable_require_defined_keeps_reexports_but_dedups( + self, temp_pkg: Tuple[str, Path] + ) -> None: + pkg_name, pkg_dir = temp_pkg + _populate_basic_pkg(pkg_dir) + _write_module( + pkg_dir, + "extra", + """ + from .child_a import ChildA # re-export + """, + ) + base_mod = importlib.import_module(f"{pkg_name}.base") + + result = list( + discover_subclasses( + pkg_name, + base_mod.Base, + module_filter=lambda n: n != "base", + require_defined_in_module=False, + ) + ) + # Even though ChildA is visible in two modules, dedup ensures + # it is yielded a single time. + assert _names(result) == ["ChildA", "ChildB"] + # And it is exactly the same class object. + assert result.count(importlib.import_module(f"{pkg_name}.child_a").ChildA) == 1 + + +class TestImportErrorHandling: + def test_callback_invoked_on_failure(self, temp_pkg: Tuple[str, Path]) -> None: + pkg_name, pkg_dir = temp_pkg + _populate_basic_pkg(pkg_dir) + _write_module( + pkg_dir, + "broken", + """ + raise RuntimeError("boom") + """, + ) + base_mod = importlib.import_module(f"{pkg_name}.base") + + captured: List[Tuple[str, BaseException]] = [] + + result = list( + discover_subclasses( + pkg_name, + base_mod.Base, + module_filter=lambda n: n != "base", + on_import_error=lambda mod, exc: captured.append((mod, exc)), + ) + ) + + # Discovery still produced the healthy modules. + assert _names(result) == ["ChildA", "ChildB"] + assert len(captured) == 1 + failed_mod, failed_exc = captured[0] + assert failed_mod == f"{pkg_name}.broken" + assert isinstance(failed_exc, RuntimeError) + + def test_default_handler_logs_and_continues( + self, + temp_pkg: Tuple[str, Path], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + pkg_name, pkg_dir = temp_pkg + _populate_basic_pkg(pkg_dir) + _write_module( + pkg_dir, + "broken", + """ + raise RuntimeError("boom") + """, + ) + base_mod = importlib.import_module(f"{pkg_name}.base") + + # Spy directly on the module-level logger to avoid coupling the + # test to pytest's caplog interaction with the project's + # custom logger setup. + # First Party + from lmcache.v1.utils import subclass_discovery as sd + + warnings: List[Tuple[str, Tuple[object, ...]]] = [] + monkeypatch.setattr( + sd.logger, + "warning", + lambda msg, *args, **kwargs: warnings.append((msg, args)), + ) + + result = list( + discover_subclasses( + pkg_name, + base_mod.Base, + module_filter=lambda n: n != "base", + ) + ) + + assert _names(result) == ["ChildA", "ChildB"] + # The default handler must surface the failure via logging, + # without aborting the iteration. + assert any(f"{pkg_name}.broken" in str(args) for _, args in warnings) + + +class TestInvalidPackage: + def test_non_package_raises_type_error(self) -> None: + # A regular (non-package) module: pytest itself has no __path__. + # Standard + import io # any builtin module without __path__ + + with pytest.raises(TypeError): + list(discover_subclasses(io, object)) From fc91bfa9b3a94cecdd54d20be3821fa9872f674b Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Wed, 20 May 2026 11:31:00 -0700 Subject: [PATCH 43/69] [CI] fix incorrect version tagging (#3314) Signed-off-by: deng451e <838677410@qq.com> --- .github/workflows/sync_torch_version.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sync_torch_version.yml b/.github/workflows/sync_torch_version.yml index 4f481df5234..0bcc182f955 100644 --- a/.github/workflows/sync_torch_version.yml +++ b/.github/workflows/sync_torch_version.yml @@ -58,7 +58,7 @@ jobs: extract_torch_version() { local file="$1" - sed -nE 's/.*torch[[:space:]]*==[[:space:]]*"?([0-9]+(\.[0-9]+){1,2})"?.*/\1/p' "$file" + sed -nE 's/.*torch[[:space:]]*==[[:space:]]*([0-9]+(\.[0-9]+){1,2}).*/\1/p' "$file" } mapfile -t upstream_matches < <(extract_torch_version "$tmp_upstream") @@ -96,8 +96,8 @@ jobs: awk -v new_ver="$upstream" ' BEGIN { updated = 0 } { - if ($0 ~ /torch[[:space:]]*==[[:space:]]*"?[0-9]+(\.[0-9]+){1,2}"?/) { - sub(/torch[[:space:]]*==[[:space:]]*"?[0-9]+(\.[0-9]+){1,2}"?/, "torch == \"" new_ver "\"") + if ($0 ~ /torch[[:space:]]*==[[:space:]]*[0-9]+(\.[0-9]+){1,2}/) { + sub(/torch[[:space:]]*==[[:space:]]*[0-9]+(\.[0-9]+){1,2}/, "torch==" new_ver) updated++ } print From 259c0b39873b7ad1d5b10992d47ccf19801b00f1 Mon Sep 17 00:00:00 2001 From: Tony Lin Date: Thu, 21 May 2026 06:31:55 +0800 Subject: [PATCH 44/69] feat(mp): Non-GPU Context by pickle (#3259) * feat(mp): CPU Context by pickle Signed-off-by: Tony Lin * renaming bounce keyword to cpu context Signed-off-by: Tony Lin * fix unit test failures Signed-off-by: Tony Lin * address bot review comment Signed-off-by: Tony Lin * refactor: standardize cpu context naming conventions Signed-off-by: Tony Lin * small fix on error handling Signed-off-by: Tony Lin * refactor: polymorphic TransferContext for MP adapter transport layer Replace scattered if/else CPU/CUDA branches in vllm_multi_process_adapter with a TransferContext abstraction (ABC + CudaTransferContext + CPUTransferContext). - Add transfer_context.py with unified register/store/retrieve/poll interface - Device-type dispatch centralized in create_transfer_context() factory - Adapter delegates all transport logic via polymorphism, no branching - Future transports (e.g. SHM) only need a new subclass, zero adapter changes Signed-off-by: Tony Lin * restore unnecessary changes Signed-off-by: Tony Lin * Revert CPU registration payload from pickle bytes to scalar fields Signed-off-by: Tony Lin * update design doc Signed-off-by: Tony Lin * rebase dsv4: propagate vllm_logical_block_size through TransferContext.register() to restore DeepSeek V4 compress_ratio Signed-off-by: Tony Lin * rename to more general names: non_gpu_context & non_cuda_transfer_context Signed-off-by: Tony Lin * add todo note for deepseek v4 on non-cuda path Signed-off-by: Tony Lin * Consolidate MPCacheEngine context state into unified registry Signed-off-by: Tony Lin * use dataclass for payload Signed-off-by: Tony Lin * Auto-disable l1-use-lazy on non-CUDA backends Signed-off-by: Tony Lin * Lift layout hints to the caller layer to avoid redundant computation in transfer-context registration Signed-off-by: Tony Lin * [refactor] reserve zero-copy buffer allocation interface to NonGpuContext for shm solution Signed-off-by: Tony Lin * fix: update test to use new unified protocol methods Signed-off-by: Tony Lin * refactor: rename transfer context classes to handle/data semantics Signed-off-by: Tony Lin * update test Signed-off-by: Tony Lin * update docs Signed-off-by: Tony Lin * rename test file Signed-off-by: Tony Lin --------- Signed-off-by: Tony Lin --- .../v1/multiprocess/non_gpu_context_design.md | 217 +++++++++ .../vllm/vllm_multi_process_adapter.py | 84 ++-- lmcache/python_ops_fallback.py | 12 +- lmcache/v1/distributed/config.py | 13 + lmcache/v1/multiprocess/custom_types.py | 24 + lmcache/v1/multiprocess/non_gpu_context.py | 412 ++++++++++++++++ .../v1/multiprocess/non_gpu_context_pickle.py | 105 ++++ lmcache/v1/multiprocess/protocols/base.py | 5 + lmcache/v1/multiprocess/protocols/engine.py | 59 +++ lmcache/v1/multiprocess/server.py | 396 ++++++++++++--- lmcache/v1/multiprocess/transfer_context.py | 409 ++++++++++++++++ .../test_non_cuda_data_transfer.py | 461 ++++++++++++++++++ tests/v1/test_vllm_mp_adapter.py | 77 ++- 13 files changed, 2173 insertions(+), 101 deletions(-) create mode 100644 docs/design/v1/multiprocess/non_gpu_context_design.md create mode 100644 lmcache/v1/multiprocess/non_gpu_context.py create mode 100644 lmcache/v1/multiprocess/non_gpu_context_pickle.py create mode 100644 lmcache/v1/multiprocess/transfer_context.py create mode 100644 tests/v1/multiprocess/test_non_cuda_data_transfer.py diff --git a/docs/design/v1/multiprocess/non_gpu_context_design.md b/docs/design/v1/multiprocess/non_gpu_context_design.md new file mode 100644 index 00000000000..5ed541d30ed --- /dev/null +++ b/docs/design/v1/multiprocess/non_gpu_context_design.md @@ -0,0 +1,217 @@ +# Non-GPU Context Design (Multiprocess Mode) + +## 1. Motivation + +LMCache multiprocess mode originally depended on CUDA IPC: workers send IPC handles, +and the server reads/writes worker GPU memory directly. That path works well on +CUDA, but the required primitives are CUDA-specific (IPC memory handles, +interprocess CUDA events, CUDA stream semantics). + +For **CPU, XPU, HPU, and other non-CUDA devices**, those primitives do not exist. +The non-GPU context design introduces a device-agnostic path where workers move KV +data through CPU chunks instead of CUDA IPC handles. + +Goal: keep the existing CUDA path unchanged while adding a second path that works +across non-CUDA backends. + +## 2. Design + +### 2.1 Architecture Overview + +```text +Worker adapter (vLLM MP adapter) + └─ TransferContext + ├─ HandleTransferContext (CUDA IPC path) + └─ DataTransferContext (non-CUDA data path) + └─ NonGpuContext + ├─ NonGpuContextPickle + └─ NonGpuContextShm (TODO) +``` + +State machine overview (worker-side): + +```text + create_transfer_context() + | + +---------------+---------------+ + | | + v v + HandleTransferContext DataTransferContext + (device == CUDA) (device != CUDA) + | | + v v + register() register() + | | + +---------------+---------------+ + | + v + READY + | + +---------------+-------------------------------+ + | | + v v + submit_store (handle path) submit_store (data path) + -> STORE request (async) -> prepare_store -> gather -> commit_store + | | + +---------------+-------------------------------+ + | + v + READY + | + +---------------+-------------------------------+ + | | + v v + submit_retrieve (handle path) submit_retrieve (data path) + -> RETRIEVE request (async) -> prepare_retrieve -> scatter -> commit_retrieve + | | + +---------------+-------------------------------+ + | + v + READY + | + v + close() +``` + +Overall data flow: +- **CUDA path**: worker sends a handle, server pulls/pushes data directly. +- **Non-CUDA path**: worker gathers/scatters paged KV and exchanges CPU-side data + via a transport-specific `NonGpuContext` implementation. + +### 2.2 Worker Side: TransferContext + +`TransferContext` is the worker-side transport abstraction with four methods: +`register`, `submit_store`, `submit_retrieve`, and `close`. +The contract is intentionally minimal so worker adapters only depend on these +four lifecycle and transfer operations. + +- **HandleTransferContext** keeps the original CUDA IPC behavior: + worker sends a handle and server performs direct GPU-side transfer. +- **DataTransferContext** is the non-CUDA path: + worker transfers actual data chunks through `NonGpuContext`. + +`DataTransferContext` flows: +- **submit_store**: `prepare_store` → `gather_paged_kv_to_cpu` → `commit_store` +- **submit_retrieve**: `prepare_retrieve` → `scatter_cpu_to_paged_kv` → `commit_retrieve` + +Why `prepare → data operation → commit`: +- `prepare_*`: set up transport state (for SHM this allocates/returns shared buffers; + for pickle it is a protocol RPC that does not allocate transfer buffers). +- gather/scatter: worker-local data movement between paged KV and contiguous + CPU chunks, performed between protocol phases. +- `commit_*`: finalize and notify server to consume or release transfer state. + +`create_transfer_context()` selects the implementation once based on device type +(CUDA → `HandleTransferContext`, otherwise → `DataTransferContext`). +It also validates that all KV cache tensors share one device type and rejects +mixed-device configurations by raising an error. + +| Context | What is transferred | Who performs copy work | Completion style | +|---|---|---|---| +| HandleTransferContext | Device handle/reference | Server pulls/pushes via IPC | Async MQ future | +| DataTransferContext | Actual CPU chunk data | Worker gather/scatter + transport commit | Synchronous worker-side flow | + +### 2.3 Server Side: GPU Context vs Non-GPU Context + +- **GPU Context (existing path):** server uses CUDA IPC handles to access worker + device memory directly. +- **Non-GPU Context:** server participates in two separate two-phase protocols + exposed by `NonGpuContext`: `prepare_store/commit_store` for store, and + `prepare_retrieve/commit_retrieve` for retrieve, plus lifecycle cleanup via + `close`. + +`NonGpuContext` implementations: +- **NonGpuContextPickle**: serialize/deserialize chunk payloads with pickle. +- **NonGpuContextShm**: shared-memory transport (planned/TODO). + +This split keeps server protocol stable while allowing transport-specific behavior +behind one interface contract. + +### 2.4 Transport Comparison + +**Store (worker → server storage):** + +| Transport | Copies | Data flow | +|---|---|---| +| Handle (CUDA IPC) | 2 | GPU KV → GPU staging buffer → CPU memory object | +| Pickle | 4 | GPU KV → CPU chunk → serialize → deserialize → CPU memory object | +| SHM (TODO) | 1 | GPU KV → CPU memory object (SHM mapped) | + +**Retrieve (server storage → worker):** + +| Transport | Copies | Data flow | +|---|---|---| +| Handle (CUDA IPC) | 2 | CPU memory object → GPU staging buffer → GPU KV | +| Pickle | 4 | CPU memory object → serialize → deserialize → CPU chunk → GPU KV | +| SHM (TODO) | 1 | CPU memory object (SHM mapped) → GPU KV | + +| Transport | Pros | Cons | Best fit | +|---|---|---|---| +| Handle (CUDA IPC) | Mature path, good async overlap | CUDA-only | NVIDIA CUDA deployments | +| Pickle | Works everywhere, no SHM setup | Extra serialization + copy overhead | Universal fallback | +| SHM (TODO) | Lowest copy count, no serialization | Requires enough `/dev/shm` and synchronization | High-throughput non-CUDA setups | + +## 3. Protocol & Data Flow + +### 3.1 MQ Request Types Used by Non-GPU Path + +The non-GPU path uses five request types: + +1. `REGISTER_KV_CACHE_NON_GPU_CONTEXT` + Worker registers non-CUDA KV layout metadata so the server can reconstruct + the worker KV memory layout for store/retrieve operations. + +2. `PREPARE_STORE` + Worker asks server/transport to prepare store-side transfer state. + +3. `COMMIT_STORE` + Worker commits store data so server can persist it into storage. + +4. `PREPARE_RETRIEVE` + Worker asks server to prepare retrieval payload/state for a key. + +5. `COMMIT_RETRIEVE` + Worker acknowledges retrieval completion so transport state can be finalized. + +### 3.2 Data Flow: Pickle Path + +Store: +1. Worker `prepare_store` RPC. +2. Worker gathers paged KV into CPU chunks. +3. Worker `commit_store` sends serialized bytes. +4. Server deserializes and writes to storage. + +Retrieve: +1. Worker `prepare_retrieve` RPC. +2. Server reads from storage and returns serialized bytes. +3. Worker deserializes to CPU chunks. +4. Worker scatters chunks back to paged KV. +5. Worker `commit_retrieve` finalizes protocol state. + +```text +Store (pickle) +Worker: prepare_store --> Server +Worker: gather paged KV -> CPU chunks +Worker: commit_store(serialized bytes) --> Server +Server: deserialize -> storage write + +Retrieve (pickle) +Worker: prepare_retrieve --> Server +Server: read storage -> serialize bytes +Server: serialized bytes --> Worker +Worker: deserialize -> scatter to paged KV +Worker: commit_retrieve --> Server +``` + +### 3.3 Data Flow: SHM Path (TODO) + +Store: +1. Worker `prepare_store` obtains SHM slot/offset. +2. Worker gathers directly into SHM-backed buffers. +3. Worker `commit_store` notifies server to consume SHM data. + +Retrieve: +1. Worker `prepare_retrieve` asks server to populate SHM. +2. Server writes retrieved chunks into SHM. +3. Worker scatters from SHM-backed buffers into paged KV. +4. Worker `commit_retrieve` releases/read-completes SHM state. diff --git a/lmcache/integration/vllm/vllm_multi_process_adapter.py b/lmcache/integration/vllm/vllm_multi_process_adapter.py index b10ed5a4e56..5335cd259d1 100644 --- a/lmcache/integration/vllm/vllm_multi_process_adapter.py +++ b/lmcache/integration/vllm/vllm_multi_process_adapter.py @@ -13,7 +13,8 @@ # First Party from lmcache.integration.request_telemetry.factory import RequestTelemetryFactory -from lmcache.utils import EngineType, _lmcache_nvtx_annotate, init_logger +from lmcache.integration.vllm.utils import vllm_layout_hints +from lmcache.utils import _lmcache_nvtx_annotate, init_logger from lmcache.v1.multiprocess.custom_types import ( BlockAllocationRecord, CudaIPCWrapper, @@ -22,6 +23,10 @@ ) from lmcache.v1.multiprocess.mq import MessageQueueClient, MessagingFuture from lmcache.v1.multiprocess.protocol import RequestType, get_response_class +from lmcache.v1.multiprocess.transfer_context import ( + TransferContext, + create_transfer_context, +) from lmcache.v1.periodic_thread import PeriodicThread, ThreadLevel, ThreadRunSummary logger = init_logger(__name__) @@ -803,6 +808,9 @@ def __init__( # Registered kv caches from vLLM self.kv_caches: dict[str, torch.Tensor] = {} + # Transport context for transfer operations. + self.transfer_ctx: TransferContext | None = None + # Request futures self.store_futures: dict[str, MessagingFuture[StoreResult]] = {} # request_id -> (future, block_ids) @@ -939,27 +947,24 @@ def _send_register_kv_caches_request( ConnectionError: if the server does not respond within mq_timeout. """ - # First Party - from lmcache.integration.vllm.utils import vllm_layout_hints - + self.kv_caches = kv_caches + self.transfer_ctx = create_transfer_context(kv_caches) layout_hints = vllm_layout_hints() layout_hints["inference_engine_logical_block_size"] = ( self.vllm_logical_block_size ) - future = send_lmcache_request( - self.mq_client, - RequestType.REGISTER_KV_CACHE, - [ + try: + self.transfer_ctx.register( self.instance_id, - wrap_kv_caches(kv_caches), + kv_caches, self.model_name, self.world_size, - EngineType.VLLM, - layout_hints, - ], - ) - try: - future.result(timeout=self._mq_timeout) + self.blocks_in_chunk, + self.mq_client, + self._mq_timeout, + send_request=send_lmcache_request, + layout_hints=layout_hints, + ) except TimeoutError: raise ConnectionError( "LMCache server did not respond to " @@ -1049,11 +1054,20 @@ def submit_store_request( request_id=request_id, cache_salt=cache_salt, ) - future = send_lmcache_request( - self.mq_client, - RequestType.STORE, - [key, self.instance_id, op.block_ids, event.ipc_handle()], - ).to_cuda_future() + if self.transfer_ctx is None: + raise RuntimeError( + "Transfer context is not initialized. " + "Call register_kv_caches() before submitting store requests." + ) + future = self.transfer_ctx.submit_store( + request_id, + key, + self.instance_id, + self.kv_caches, + op.block_ids, + event, + self.blocks_in_chunk, + ) self.store_futures[request_id] = future @_lmcache_nvtx_annotate @@ -1088,17 +1102,21 @@ def submit_retrieve_request( request_id=request_id, cache_salt=cache_salt, ) - future = send_lmcache_request( - self.mq_client, - RequestType.RETRIEVE, - [ - key, - self.instance_id, - op.block_ids, - event.ipc_handle(), - op.skip_first_n_tokens, - ], - ).to_cuda_future() + if self.transfer_ctx is None: + raise RuntimeError( + "Transfer context is not initialized. " + "Call register_kv_caches() before submitting retrieve requests." + ) + future = self.transfer_ctx.submit_retrieve( + request_id, + key, + self.instance_id, + self.kv_caches, + op.block_ids, + event, + self.blocks_in_chunk, + skip_first_n_tokens=op.skip_first_n_tokens, + ) self.retrieve_futures[request_id] = (future, list(op.block_ids)) @_lmcache_nvtx_annotate @@ -1309,6 +1327,10 @@ def shutdown(self): self._mq_timeout, ) + if self.transfer_ctx is not None: + self.transfer_ctx.close() + self.transfer_ctx = None + self.mq_client.close() self.request_telemetry.close() diff --git a/lmcache/python_ops_fallback.py b/lmcache/python_ops_fallback.py index 2c92df067ff..10fce1d4afa 100644 --- a/lmcache/python_ops_fallback.py +++ b/lmcache/python_ops_fallback.py @@ -17,7 +17,7 @@ import torch # First Party -from lmcache import torch_dev, torch_device_type +from lmcache import torch_dev # Store the tensor objects in memory so that they can be accessed # outside the scope of this file @@ -327,10 +327,7 @@ def alloc_pinned_numa_ptr(size: int, numa_id: int = 0) -> int: Note: NUMA node selection is not supported on non-CUDA.""" # Create a 1D uint8 CPU tensor, as uint8 == 1 byte - # On XPU (Intel GPU), PyTorch 2.4+ supports pin_memory=True via SYCL USM - # host allocation, enabling fast DMA for XPU<->CPU transfers. - pin_memory = torch_device_type == "xpu" - tensor = torch.empty(size, dtype=torch.uint8, pin_memory=pin_memory) + tensor = torch.empty(size, dtype=torch.uint8, pin_memory=False) # First-touch initialization (forces physical allocation) tensor.fill_(0) @@ -358,10 +355,7 @@ def alloc_pinned_ptr(size: int, device_id: int = 0) -> int: fast DMA transfers. On other non-CUDA platforms, pinning is not supported.""" # Create a 1D uint8 CPU tensor, as uint8 == 1 byte - # On XPU (Intel GPU), PyTorch 2.4+ supports pin_memory=True via SYCL USM - # host allocation, enabling fast DMA for XPU<->CPU transfers. - pin_memory = torch_device_type == "xpu" - tensor = torch.empty(size, dtype=torch.uint8, pin_memory=pin_memory) + tensor = torch.empty(size, dtype=torch.uint8, pin_memory=False) # First-touch initialization (forces physical allocation) tensor.fill_(0) diff --git a/lmcache/v1/distributed/config.py b/lmcache/v1/distributed/config.py index 5bdf503ef4d..2690b043970 100644 --- a/lmcache/v1/distributed/config.py +++ b/lmcache/v1/distributed/config.py @@ -10,12 +10,16 @@ import argparse # First Party +from lmcache import torch_dev +from lmcache.logging import init_logger from lmcache.v1.distributed.l2_adapters.config import ( L2AdaptersConfig, add_l2_adapters_args, parse_args_to_l2_adapters_config, ) +logger = init_logger(__name__) + @dataclass class L1MemoryManagerConfig: @@ -38,6 +42,15 @@ class L1MemoryManagerConfig: def __post_init__(self): self.init_size_in_bytes = min(self.init_size_in_bytes, self.size_in_bytes) + # LazyMemoryAllocator requires cudart (CUDA host-pinned memory). + # Auto-disable on non-CUDA backends to avoid a RuntimeError. + if self.use_lazy and not hasattr(torch_dev, "cudart"): + logger.warning( + "LazyMemoryAllocator requires cudart which is not available " + "on the current backend. Disabling l1-use-lazy." + ) + self.use_lazy = False + @dataclass class L1ManagerConfig: diff --git a/lmcache/v1/multiprocess/custom_types.py b/lmcache/v1/multiprocess/custom_types.py index ec82b8bbd47..28cc2e3a85a 100644 --- a/lmcache/v1/multiprocess/custom_types.py +++ b/lmcache/v1/multiprocess/custom_types.py @@ -315,6 +315,30 @@ def no_worker_id_version(self) -> "IPCCacheEngineKey": KVCache = list[CudaIPCWrapper] +class RegisterNonGpuContextPayload(msgspec.Struct): + """Payload for the REGISTER_KV_CACHE_NON_GPU_CONTEXT protocol message. + + Attributes: + instance_id: Worker instance identifier (typically PID). + model_name: Model name associated with this worker. + world_size: Worker world size used in cache keys. + block_size: Tokens per paged block. + num_layers: Number of model layers. + hidden_dim_size: Flattened hidden dimension per token. + dtype_str: Torch dtype name (e.g. ``"float16"``). + use_mla: Whether the worker KV format is MLA. + """ + + instance_id: int + model_name: str + world_size: int + block_size: int + num_layers: int + hidden_dim_size: int + dtype_str: str + use_mla: bool + + @dataclass class CustomizedSerdeConfig: serializer: Callable[[Any], bytes] diff --git a/lmcache/v1/multiprocess/non_gpu_context.py b/lmcache/v1/multiprocess/non_gpu_context.py new file mode 100644 index 00000000000..e782c76d0c3 --- /dev/null +++ b/lmcache/v1/multiprocess/non_gpu_context.py @@ -0,0 +1,412 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Non-GPU context abstractions and utilities for multiprocess mode. + +This module provides: +- ``NonGpuContextMetadata``: layout metadata dataclass for non-CUDA workers. +- ``NonGpuContext``: abstract base class with a two-phase prepare/commit + interface for CPU-side KV data transfer. Concrete implementations (e.g. + ``NonGpuContextPickle``) each decide *how* data is serialised and transported. +- ``create_non_gpu_context()``: factory that returns the appropriate + ``NonGpuContext`` subclass (currently always ``NonGpuContextPickle``). +- ``compute_kv_layout``, ``gather_paged_kv_to_cpu``, ``scatter_cpu_to_paged_kv``: + shared gather/scatter utilities used by all concrete implementations. +""" + +# Standard +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, cast + +# Third Party +import torch + +# First Party +from lmcache.utils import EngineType +from lmcache.v1.distributed.api import MemoryLayoutDesc + + +@dataclass +class NonGpuContextMetadata: + """Non-GPU context layout metadata for non-CUDA workers. + + Attributes: + layout_desc: Memory layout descriptor used to interpret chunk payloads. + block_size: Number of tokens per paged block. + use_mla: Whether the worker KV format is MLA. + """ + + layout_desc: MemoryLayoutDesc + block_size: int + use_mla: bool + + +class NonGpuContext(ABC): + """Abstract base class for CPU-side KV data transfer contexts. + + All concrete implementations share a common message-queue client and + expose a uniform two-phase ``prepare/commit`` interface so that the + worker adapter is implementation-agnostic. + + Args: + metadata: Layout metadata describing the chunk format. + mq_client: Message-queue client used for server communication. + mq_timeout: Timeout in seconds for blocking MQ requests. + """ + + def __init__( + self, + metadata: NonGpuContextMetadata, + mq_client: Any, + mq_timeout: float, + ) -> None: + self.metadata = metadata + self.mq_client = mq_client + self.mq_timeout = mq_timeout + + @property + def layout_desc(self) -> MemoryLayoutDesc: + """The memory layout descriptor for this context.""" + return self.metadata.layout_desc + + @abstractmethod + def prepare_store(self, key: Any, instance_id: int) -> list[torch.Tensor] | None: + """Prepare store. Returns pre-allocated out buffers (shm) or None (pickle).""" + ... + + @abstractmethod + def commit_store( + self, key: Any, instance_id: int, chunks: list[torch.Tensor] + ) -> bool: + """Commit store. Pickle: serialize and send. Shm: notify server.""" + ... + + @abstractmethod + def prepare_retrieve(self, key: Any, instance_id: int) -> list[torch.Tensor] | None: + """Prepare retrieve. Returns chunks or shm views, or None on miss.""" + ... + + @abstractmethod + def commit_retrieve(self, key: Any, instance_id: int) -> bool: + """Commit retrieve. Pickle: no-op. Shm: release read locks.""" + ... + + @abstractmethod + def close(self) -> None: + """Release any resources held by this context.""" + ... + + +def create_non_gpu_context( + metadata: NonGpuContextMetadata, + mq_client: Any, + mq_timeout: float, +) -> NonGpuContext: + """Factory that returns the appropriate :class:`NonGpuContext` implementation. + + Currently always returns a pickle-based implementation + (``NonGpuContextPickle``). A future SHM-capable PR + may probe for shared-memory availability and fall back to pickle. + + Args: + metadata: Layout metadata for the non-GPU context. + mq_client: Message-queue client for server communication. + mq_timeout: Timeout in seconds for blocking MQ requests. + + Returns: + A concrete :class:`NonGpuContext` instance. + """ + # Local + from .non_gpu_context_pickle import NonGpuContextPickle + + return NonGpuContextPickle(metadata, mq_client, mq_timeout) + + +# --------------------------------------------------------------------------- +# Shared gather / scatter utilities +# --------------------------------------------------------------------------- + + +def compute_kv_layout( + kv_caches: dict[str, torch.Tensor], + layout_hints: Any | None = None, +) -> tuple[int, int, int, str, Any]: + """Compute KV layout metadata from KV tensors. + + Args: + kv_caches: Per-layer KV tensor mapping. + layout_hints: Optional engine layout hints. + + Returns: + Tuple of ``(block_size, num_layers, hidden_dim_size, dtype_str,`` + ``gpu_kv_format)``. + + Raises: + ValueError: If ``kv_caches`` is empty. + """ + # First Party + from lmcache.v1.gpu_connector.utils import ( + get_block_size, + get_hidden_dim_size, + get_num_layers, + normalize_kv_and_discover_format, + ) + + tensors = list(kv_caches.values()) + if not tensors: + raise ValueError("kv_caches is empty. Cannot compute KV layout.") + + gpu_kv_format, normalized = normalize_kv_and_discover_format( + tensors, EngineType.VLLM, layout_hints=layout_hints + ) + block_size = get_block_size(normalized, gpu_kv_format) + num_layers = get_num_layers(normalized, gpu_kv_format) + hidden_dim_size = get_hidden_dim_size(normalized, gpu_kv_format) + dtype_str = str(tensors[0].dtype).replace("torch.", "") + return block_size, num_layers, hidden_dim_size, dtype_str, gpu_kv_format + + +def gather_paged_kv_to_cpu( + kv_caches: dict[str, torch.Tensor], + block_ids: list[int], + blocks_per_chunk: int, + layout_hints: Any | None = None, + gpu_kv_format: Any | None = None, + out: list[torch.Tensor] | None = None, +) -> list[torch.Tensor]: + """Gather paged KV blocks into CPU chunk tensors. + + Args: + kv_caches: Per-layer KV tensor mapping. + block_ids: Flattened block IDs for all chunks. + blocks_per_chunk: Number of paged blocks in one LMCache chunk. + layout_hints: Optional engine layout hints. + gpu_kv_format: Optional pre-detected KV format. + + Returns: + List of CPU tensors, one per chunk. For non-MLA each chunk has shape + ``[2, num_layers, chunk_tokens, hidden_dim]`` where dimension ``0`` + stores ``(K, V)``. For MLA (multi-head latent attention) each chunk + has shape ``[num_layers, chunk_tokens, hidden_dim]``. + """ + # First Party + from lmcache.v1.gpu_connector.utils import ( + get_block_size, + is_mla, + normalize_kv_and_discover_format, + ) + import lmcache.c_ops as lmc_ops + + tensors = list(kv_caches.values()) + fmt, normalized = normalize_kv_and_discover_format( + tensors, EngineType.VLLM, layout_hints=layout_hints + ) + if gpu_kv_format is None: + gpu_kv_format = fmt + use_mla = is_mla(gpu_kv_format) + is_hnd = gpu_kv_format in ( + lmc_ops.GPUKVFormat.NL_X_TWO_NB_NH_BS_HS, + lmc_ops.GPUKVFormat.NL_X_NB_TWO_NH_BS_HS, + ) + + block_size = get_block_size(normalized, gpu_kv_format) + num_chunks = len(block_ids) // blocks_per_chunk + + # After normalization the structure is always a list of per-layer + # tensors. Cast once so all downstream indexing is typed correctly. + layer_tensors = cast(list[torch.Tensor], normalized) + + chunks: list[torch.Tensor] = [] if out is None else out + for chunk_idx in range(num_chunks): + chunk_block_ids = block_ids[ + chunk_idx * blocks_per_chunk : (chunk_idx + 1) * blocks_per_chunk + ] + if use_mla: + mla_layers: list[torch.Tensor] = [] + idx = torch.tensor(chunk_block_ids, dtype=torch.long) + for layer in layer_tensors: + layer_blocks = layer[idx] + mla_layers.append( + layer_blocks.reshape( + len(chunk_block_ids) * block_size, layer_blocks.shape[-1] + ) + ) + chunk_tensor = torch.stack(mla_layers, dim=0) + if out is not None: + out[chunk_idx].copy_(chunk_tensor, non_blocking=True) + else: + chunks.append(chunk_tensor.cpu()) + else: + k_layers: list[torch.Tensor] = [] + v_layers: list[torch.Tensor] = [] + for layer in layer_tensors: + if is_hnd: + if gpu_kv_format == lmc_ops.GPUKVFormat.NL_X_TWO_NB_NH_BS_HS: + k_t = layer[0] + v_t = layer[1] + else: + k_t = layer[:, 0] + v_t = layer[:, 1] + _num_blocks, num_heads, _block_size, head_size = k_t.shape + k_blocks = k_t[torch.tensor(chunk_block_ids, dtype=torch.long)] + v_blocks = v_t[torch.tensor(chunk_block_ids, dtype=torch.long)] + # HND blocks are [NB, NH, BS, HS]; convert to token-major + # [NB, BS, NH, HS] before flattening to [tokens, NH*HS]. + k_layers.append( + k_blocks.permute(0, 2, 1, 3).reshape( + len(chunk_block_ids) * block_size, num_heads * head_size + ) + ) + v_layers.append( + v_blocks.permute(0, 2, 1, 3).reshape( + len(chunk_block_ids) * block_size, num_heads * head_size + ) + ) + else: + if gpu_kv_format == lmc_ops.GPUKVFormat.NL_X_TWO_NB_BS_NH_HS: + k_t = layer[0] + v_t = layer[1] + else: + k_t = layer[:, 0] + v_t = layer[:, 1] + _num_blocks, _block_size, num_heads, head_size = k_t.shape + k_blocks = k_t[torch.tensor(chunk_block_ids, dtype=torch.long)] + v_blocks = v_t[torch.tensor(chunk_block_ids, dtype=torch.long)] + k_layers.append( + k_blocks.reshape( + len(chunk_block_ids) * block_size, num_heads * head_size + ) + ) + v_layers.append( + v_blocks.reshape( + len(chunk_block_ids) * block_size, num_heads * head_size + ) + ) + k_stacked = torch.stack(k_layers, dim=0) + v_stacked = torch.stack(v_layers, dim=0) + chunk_tensor = torch.stack([k_stacked, v_stacked], dim=0) + if out is not None: + out[chunk_idx].copy_(chunk_tensor, non_blocking=True) + else: + chunks.append(chunk_tensor.cpu()) + return chunks + + +def scatter_cpu_to_paged_kv( + kv_caches: dict[str, torch.Tensor], + block_ids: list[int], + chunks: list[torch.Tensor], + blocks_per_chunk: int, + skip_first_n_tokens: int = 0, + layout_hints: Any | None = None, + gpu_kv_format: Any | None = None, +) -> None: + """Scatter CPU chunk tensors back into paged KV tensors. + + Args: + kv_caches: Per-layer KV tensor mapping to write into. + block_ids: Flattened destination block IDs for all chunks. + chunks: List of CPU chunk tensors (as returned by + :func:`gather_paged_kv_to_cpu`). + blocks_per_chunk: Number of paged blocks in one LMCache chunk. + skip_first_n_tokens: Token prefix to skip when scattering. + layout_hints: Optional engine layout hints. + gpu_kv_format: Optional pre-detected KV format. + """ + # First Party + from lmcache.v1.gpu_connector.utils import ( + get_block_size, + is_mla, + normalize_kv_and_discover_format, + ) + import lmcache.c_ops as lmc_ops + + if not chunks: + return + + tensors = list(kv_caches.values()) + fmt, normalized = normalize_kv_and_discover_format( + tensors, EngineType.VLLM, layout_hints=layout_hints + ) + if gpu_kv_format is None: + gpu_kv_format = fmt + use_mla = is_mla(gpu_kv_format) + + block_size = get_block_size(normalized, gpu_kv_format) + device = tensors[0].device + is_hnd = gpu_kv_format in ( + lmc_ops.GPUKVFormat.NL_X_TWO_NB_NH_BS_HS, + lmc_ops.GPUKVFormat.NL_X_NB_TWO_NH_BS_HS, + ) + + # After normalization the structure is always a list of per-layer + # tensors. Cast once so all downstream indexing is typed correctly. + layer_tensors = cast(list[torch.Tensor], normalized) + + for chunk_idx, chunk_cpu in enumerate(chunks): + chunk_block_ids = block_ids[ + chunk_idx * blocks_per_chunk : (chunk_idx + 1) * blocks_per_chunk + ] + if not chunk_block_ids: + continue + + chunk_start_token = chunk_idx * blocks_per_chunk * block_size + chunk_end_token = chunk_start_token + len(chunk_block_ids) * block_size + effective_start = max(chunk_start_token, skip_first_n_tokens) + if effective_start >= chunk_end_token: + continue + + skip_blocks_in_chunk = (effective_start - chunk_start_token) // block_size + effective_block_ids = chunk_block_ids[skip_blocks_in_chunk:] + if not effective_block_ids: + continue + + skip_tokens = skip_blocks_in_chunk * block_size + chunk_device = chunk_cpu.to(device) + + if use_mla: + eff_idx = torch.tensor(effective_block_ids, dtype=torch.long) + for layer_idx, layer in enumerate(layer_tensors): + mla_src = chunk_device[layer_idx, skip_tokens:] + hidden_size = layer.shape[-1] + mla_src_3d = mla_src.reshape( + len(effective_block_ids), block_size, hidden_size + ) + layer[eff_idx] = mla_src_3d + elif is_hnd: + for layer_idx, layer in enumerate(layer_tensors): + k_src = chunk_device[0, layer_idx, skip_tokens:] + v_src = chunk_device[1, layer_idx, skip_tokens:] + if gpu_kv_format == lmc_ops.GPUKVFormat.NL_X_TWO_NB_NH_BS_HS: + k_t = layer[0] + v_t = layer[1] + else: + k_t = layer[:, 0] + v_t = layer[:, 1] + _nb, nh, _bs, hs = k_t.shape + k_blocks = k_src.reshape( + len(effective_block_ids), block_size, nh, hs + ).permute(0, 2, 1, 3) + v_blocks = v_src.reshape( + len(effective_block_ids), block_size, nh, hs + ).permute(0, 2, 1, 3) + k_t[effective_block_ids] = k_blocks + v_t[effective_block_ids] = v_blocks + else: + for layer_idx, layer in enumerate(layer_tensors): + k_src = chunk_device[0, layer_idx, skip_tokens:] + v_src = chunk_device[1, layer_idx, skip_tokens:] + if gpu_kv_format == lmc_ops.GPUKVFormat.NL_X_TWO_NB_BS_NH_HS: + k_t = layer[0] + v_t = layer[1] + else: + k_t = layer[:, 0] + v_t = layer[:, 1] + _num_blocks, _block_size, num_heads, head_size = k_t.shape + k_src_4d = k_src.reshape( + len(effective_block_ids), block_size, num_heads, head_size + ) + v_src_4d = v_src.reshape( + len(effective_block_ids), block_size, num_heads, head_size + ) + k_t[effective_block_ids] = k_src_4d + v_t[effective_block_ids] = v_src_4d diff --git a/lmcache/v1/multiprocess/non_gpu_context_pickle.py b/lmcache/v1/multiprocess/non_gpu_context_pickle.py new file mode 100644 index 00000000000..d310b9c65d1 --- /dev/null +++ b/lmcache/v1/multiprocess/non_gpu_context_pickle.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Pickle-based NonGpuContext implementation for multiprocess mode.""" + +# Standard +from typing import Any +import pickle + +# Third Party +import torch + +# First Party +from lmcache.v1.multiprocess.non_gpu_context import ( + NonGpuContext, + NonGpuContextMetadata, +) +from lmcache.v1.multiprocess.protocol import RequestType, get_response_class + + +class NonGpuContextPickle(NonGpuContext): + """Pickle-based implementation of :class:`NonGpuContext`. + + Transport mechanism: + - **Store**: ``prepare_store`` sends ``PREPARE_STORE`` (returns empty slots + for pickle mode); ``commit_store`` serialises chunks and sends + ``COMMIT_STORE``. + - **Retrieve**: ``prepare_retrieve`` sends ``PREPARE_RETRIEVE`` and + deserialises the returned bytes; ``commit_retrieve`` sends + ``COMMIT_RETRIEVE`` (no-op for pickle). + """ + + def __init__( + self, + metadata: NonGpuContextMetadata, + mq_client: Any, + mq_timeout: float, + ) -> None: + super().__init__(metadata, mq_client, mq_timeout) + + def prepare_store(self, key: Any, instance_id: int) -> list[torch.Tensor] | None: + """Send PREPARE_STORE RPC. For pickle, returns no pre-allocated buffers.""" + future = self.mq_client.submit_request( + RequestType.PREPARE_STORE, + [key, instance_id], + get_response_class(RequestType.PREPARE_STORE), + ) + try: + future.result(timeout=self.mq_timeout) + except TimeoutError: + pass + return None + + def commit_store( + self, key: Any, instance_id: int, chunks: list[torch.Tensor] + ) -> bool: + """Serialize chunks and send via COMMIT_STORE. + + Returns: + ``True`` on success, ``False`` on failure or timeout. + """ + serialised = pickle.dumps(chunks) + future = self.mq_client.submit_request( + RequestType.COMMIT_STORE, + [key, instance_id, serialised], + get_response_class(RequestType.COMMIT_STORE), + ) + try: + return bool(future.result(timeout=self.mq_timeout)) + except TimeoutError: + return False + + def prepare_retrieve(self, key: Any, instance_id: int) -> list[torch.Tensor] | None: + """Send PREPARE_RETRIEVE and deserialize the response data. + + Returns: + Chunks on hit, or None on miss/timeout. + """ + future = self.mq_client.submit_request( + RequestType.PREPARE_RETRIEVE, + [key, instance_id], + get_response_class(RequestType.PREPARE_RETRIEVE), + ) + try: + response = future.result(timeout=self.mq_timeout) + except TimeoutError: + return None + if not response.success or not response.data: + return None + chunks: list[torch.Tensor] = pickle.loads(response.data) + return chunks + + def commit_retrieve(self, key: Any, instance_id: int) -> bool: + """Send COMMIT_RETRIEVE (no-op for pickle path).""" + future = self.mq_client.submit_request( + RequestType.COMMIT_RETRIEVE, + [key, instance_id], + get_response_class(RequestType.COMMIT_RETRIEVE), + ) + try: + future.result(timeout=self.mq_timeout) + except TimeoutError: + pass + return True + + def close(self) -> None: + """No-op: the pickle path holds no persistent resources.""" diff --git a/lmcache/v1/multiprocess/protocols/base.py b/lmcache/v1/multiprocess/protocols/base.py index 383a41ff8c3..777ce029f29 100644 --- a/lmcache/v1/multiprocess/protocols/base.py +++ b/lmcache/v1/multiprocess/protocols/base.py @@ -48,6 +48,11 @@ class RequestType(enum.Enum): QUERY_PREFETCH_LOOKUP_HITS = enum.auto() FREE_LOOKUP_LOCKS = enum.auto() END_SESSION = enum.auto() + REGISTER_KV_CACHE_NON_GPU_CONTEXT = enum.auto() + PREPARE_STORE = enum.auto() + COMMIT_STORE = enum.auto() + PREPARE_RETRIEVE = enum.auto() + COMMIT_RETRIEVE = enum.auto() # Controller operations CLEAR = enum.auto() diff --git a/lmcache/v1/multiprocess/protocols/engine.py b/lmcache/v1/multiprocess/protocols/engine.py index e9f37fd422f..62ec4926cd8 100644 --- a/lmcache/v1/multiprocess/protocols/engine.py +++ b/lmcache/v1/multiprocess/protocols/engine.py @@ -12,15 +12,40 @@ - END_SESSION: End a session and clean up associated resources """ +# Standard +from dataclasses import dataclass, field + # First Party from lmcache.utils import EngineType from lmcache.v1.gpu_connector.utils import LayoutHints from lmcache.v1.multiprocess.custom_types import ( IPCCacheEngineKey, KVCache, + RegisterNonGpuContextPayload, ) from lmcache.v1.multiprocess.protocols.base import HandlerType, ProtocolDefinition + +@dataclass +class PrepareStoreResponse: + """Response for PREPARE_STORE.""" + + context: dict = field( + default_factory=dict + ) # pickle: {}, shm will put slot info here + + +@dataclass +class PrepareRetrieveResponse: + """Response for PREPARE_RETRIEVE.""" + + success: bool + data: bytes = b"" + context: dict = field( + default_factory=dict + ) # pickle: {}, shm will put slot info here + + # Define request names for this protocol group REQUEST_NAMES = [ "REGISTER_KV_CACHE", @@ -32,6 +57,11 @@ "QUERY_PREFETCH_LOOKUP_HITS", "FREE_LOOKUP_LOCKS", "END_SESSION", + "REGISTER_KV_CACHE_NON_GPU_CONTEXT", + "PREPARE_STORE", + "COMMIT_STORE", + "PREPARE_RETRIEVE", + "COMMIT_RETRIEVE", ] # Type alias for cache keys @@ -146,4 +176,33 @@ def get_protocol_definitions() -> dict[str, ProtocolDefinition]: response_class=None, handler_type=HandlerType.BLOCKING, ), + # Register non-GPU KV cache context + # Payload: + # - RegisterNonGpuContextPayload - all metadata fields in one struct + # Returns: None + "REGISTER_KV_CACHE_NON_GPU_CONTEXT": ProtocolDefinition( + payload_classes=[RegisterNonGpuContextPayload], + response_class=None, + handler_type=HandlerType.SYNC, + ), + "PREPARE_STORE": ProtocolDefinition( + payload_classes=[KeyType, int], + response_class=PrepareStoreResponse, + handler_type=HandlerType.BLOCKING, + ), + "COMMIT_STORE": ProtocolDefinition( + payload_classes=[KeyType, int, bytes], + response_class=bool, + handler_type=HandlerType.BLOCKING, + ), + "PREPARE_RETRIEVE": ProtocolDefinition( + payload_classes=[KeyType, int], + response_class=PrepareRetrieveResponse, + handler_type=HandlerType.BLOCKING, + ), + "COMMIT_RETRIEVE": ProtocolDefinition( + payload_classes=[KeyType, int], + response_class=bool, + handler_type=HandlerType.BLOCKING, + ), } diff --git a/lmcache/v1/multiprocess/server.py b/lmcache/v1/multiprocess/server.py index 1ff8f6ca173..bb67d07bf54 100644 --- a/lmcache/v1/multiprocess/server.py +++ b/lmcache/v1/multiprocess/server.py @@ -5,10 +5,12 @@ from itertools import islice from typing import Generator import argparse +import pickle import threading import time # Third Party +import torch import zmq # First Party @@ -55,16 +57,22 @@ BlockAllocationRecord, IPCCacheEngineKey, KVCache, + RegisterNonGpuContextPayload, ) from lmcache.v1.multiprocess.gpu_context import ( GPUCacheContext, ) from lmcache.v1.multiprocess.mq import MessageQueueServer +from lmcache.v1.multiprocess.non_gpu_context import NonGpuContextMetadata from lmcache.v1.multiprocess.protocol import ( RequestType, get_handler_type, get_payload_classes, ) +from lmcache.v1.multiprocess.protocols.engine import ( + PrepareRetrieveResponse, + PrepareStoreResponse, +) from lmcache.v1.multiprocess.session import SessionManager from lmcache.v1.multiprocess.token_hasher import TokenHasher import lmcache.c_ops as lmc_ops @@ -173,6 +181,45 @@ class _PrefetchJob: cache_salt: str = "" +@dataclass +class RegisteredContext: + """Registered context metadata for a single worker instance. + + At least one of ``gpu_context`` or ``non_cuda_metadata`` is expected to be + populated for valid registrations. + """ + + model_name: str + world_size: int + gpu_context: GPUCacheContext | None = None + non_cuda_metadata: NonGpuContextMetadata | None = None + + @property + def is_gpu(self) -> bool: + """Return whether this registration uses a GPU transfer context.""" + return self.gpu_context is not None + + def get_layout_desc(self, chunk_size: int) -> MemoryLayoutDesc: + """Return the layout descriptor for this registration. + + Args: + chunk_size: Chunk size in tokens used for GPU layout derivation. + + Returns: + The resolved memory layout descriptor. + + Raises: + ValueError: If no GPU context or non-CUDA metadata is configured. + """ + if self.gpu_context is not None: + return get_layout_desc(self.gpu_context, chunk_size) + if self.non_cuda_metadata is None: + raise ValueError( + "Invalid RegisteredContext: no GPU or non-CUDA metadata configured" + ) + return self.non_cuda_metadata.layout_desc + + # Main class for the mp cache engine class MPCacheEngine: def __init__( @@ -181,14 +228,8 @@ def __init__( chunk_size: int = 256, hash_algorithm: str = "blake3", ): - # GPU ID -> KV cache tensors - self.gpu_contexts: dict[int, GPUCacheContext] = {} - - # GPU ID -> (model name, world size) as metadata - # NOTE: This is mainly for determining the layout desc during prefetch - # We assume that if the (model name, world size) is the same, then - # the layout desc returned by the gpu context is the same. - self.gpu_context_meta: dict[int, tuple[str, int]] = {} + # Worker instance ID -> registered context metadata + self.contexts: dict[int, RegisteredContext] = {} # chunk size self.chunk_size = chunk_size @@ -216,6 +257,15 @@ def __init__( self._setup_metrics() + @property + def gpu_contexts(self) -> dict[int, GPUCacheContext]: + """Return GPU-only context mapping for backward compatibility.""" + return { + instance_id: ctx.gpu_context + for instance_id, ctx in self.contexts.items() + if ctx.gpu_context is not None + } + def register_kv_cache( self, instance_id: int, @@ -239,7 +289,7 @@ def register_kv_cache( layout_hints: See :class:`LayoutHints`. Forwarded to :class:`GPUCacheContext` for GPU KV format detection. """ - if instance_id in self.gpu_contexts: + if instance_id in self.contexts: logger.warning( "Instance %s's KV cache is already registered, " "skipping the new registration", @@ -253,8 +303,11 @@ def register_kv_cache( layout_hints=layout_hints or None, engine_type=engine_type, ) - self.gpu_contexts[instance_id] = gpu_context - self.gpu_context_meta[instance_id] = (model_name, world_size) + self.contexts[instance_id] = RegisteredContext( + model_name=model_name, + world_size=world_size, + gpu_context=gpu_context, + ) logger.info( "Registered KV cache for GPU ID %d with %d layers", instance_id, @@ -268,13 +321,216 @@ def unregister_kv_cache(self, instance_id: int) -> None: Args: instance_id (int): The GPU instance ID (such as PID). """ - if instance_id in self.gpu_contexts: - del self.gpu_contexts[instance_id] - del self.gpu_context_meta[instance_id] + context = self.contexts.pop(instance_id, None) + if context is None: + logger.warning( + "No registered context found for instance ID %d", instance_id + ) + return + + if context.is_gpu: logger.info("Unregistered KV cache for GPU ID %d", instance_id) torch_dev.empty_cache() else: - logger.warning("No KV cache found for GPU ID %d to unregister", instance_id) + logger.info("Unregistered non-CUDA context for instance ID %d", instance_id) + + def register_kv_cache_non_gpu_context( + self, + payload: RegisterNonGpuContextPayload, + ) -> None: + """Register non-CUDA KV layout metadata for non-GPU context mode. + + Args: + payload: Struct containing all registration fields + (instance_id, model_name, world_size, block_size, + num_layers, hidden_dim_size, dtype_str, use_mla). + + Raises: + ValueError: If ``payload.dtype_str`` is not a valid torch dtype name. + """ + if payload.instance_id in self.contexts: + logger.warning( + "Instance %s's KV cache is already registered, " + "skipping the new registration", + payload.instance_id, + ) + return + + dtype = getattr(torch, payload.dtype_str, None) + if dtype is None or not isinstance(dtype, torch.dtype): + raise ValueError( + f"Invalid dtype_str '{payload.dtype_str}': must be a valid torch dtype " + "attribute name (e.g. 'float16' for torch.float16, " + "'bfloat16' for torch.bfloat16, 'float32' for torch.float32)." + ) + + shape = ( + torch.Size([payload.num_layers, self.chunk_size, payload.hidden_dim_size]) + if payload.use_mla + else torch.Size( + [2, payload.num_layers, self.chunk_size, payload.hidden_dim_size] + ) + ) + layout_desc = MemoryLayoutDesc(shapes=[shape], dtypes=[dtype]) + self.contexts[payload.instance_id] = RegisteredContext( + model_name=payload.model_name, + world_size=payload.world_size, + non_cuda_metadata=NonGpuContextMetadata( + layout_desc=layout_desc, + block_size=payload.block_size, + use_mla=payload.use_mla, + ), + ) + + def _resolve_obj_keys(self, key: IPCCacheEngineKey) -> list[ObjectKey]: + """Resolve object keys from an IPC cache key. + + Args: + key: IPC cache key describing model/session/token range. + + Returns: + Resolved object keys for the requested token range. + + Raises: + ValueError: If ``key.worker_id`` is ``None``. + """ + session = self.session_manager.get_or_create(key.request_id) + session.set_tokens(list(key.token_ids)) + chunk_hashes = [ + TokenHasher.hash_to_bytes(h) for h in session.get_hashes(key.start, key.end) + ] + if key.worker_id is None: + raise ValueError("Must resolve keys with worker_id != None") + return ipc_key_to_object_keys(key, chunk_hashes) + + @_lmcache_nvtx_annotate + def prepare_store( + self, + key: IPCCacheEngineKey, + instance_id: int, + ) -> PrepareStoreResponse: + """Prepare a store operation. For pickle mode, returns empty slots. + + Args: + key: Cache key for the token range to store. + instance_id: Worker instance identifier. + + Returns: + PrepareStoreResponse with empty slots for pickle mode. + """ + + return PrepareStoreResponse(context={}) + + @_lmcache_nvtx_annotate + def commit_store( + self, + key: IPCCacheEngineKey, + instance_id: int, + cpu_data: bytes, + ) -> bool: + """Commit serialized CPU chunks to storage. + + Args: + key: Cache key for the token range to store. + instance_id: Worker instance identifier. + cpu_data: Pickled list of CPU tensors produced by the worker. + + Returns: + ``True`` when all reserved objects are written, otherwise ``False``. + """ + obj_keys = self._resolve_obj_keys(key) + + context = self.contexts.get(instance_id) + if context is None or context.non_cuda_metadata is None: + raise ValueError( + f"non-CUDA context not registered for instance ID {instance_id}" + ) + ctx = context.non_cuda_metadata + chunks: list[torch.Tensor] = pickle.loads(cpu_data) + reserved_dict = self.storage_manager.reserve_write( + obj_keys, ctx.layout_desc, "new" + ) + written_keys: list[ObjectKey] = [] + try: + for idx, obj_key in enumerate(obj_keys): + if obj_key not in reserved_dict: + continue + if idx >= len(chunks): + continue + memory_obj = reserved_dict[obj_key] + if memory_obj.tensor is None: + continue + chunk_cpu = chunks[idx] + if chunk_cpu.shape != memory_obj.tensor.shape: + continue + memory_obj.tensor.copy_(chunk_cpu) + written_keys.append(obj_key) + finally: + if written_keys: + self.storage_manager.finish_write(written_keys) + + return len(written_keys) == len(reserved_dict) + + @_lmcache_nvtx_annotate + def prepare_retrieve( + self, + key: IPCCacheEngineKey, + instance_id: int, + ) -> PrepareRetrieveResponse: + """Retrieve prefetched chunks and return serialized CPU tensors. + + Args: + key: Cache key for the token range to retrieve. + instance_id: Worker instance identifier. + + Returns: + PrepareRetrieveResponse with serialized data on hit. + """ + + obj_keys = self._resolve_obj_keys(key) + + context = self.contexts.get(instance_id) + if context is None or context.non_cuda_metadata is None: + raise ValueError( + f"non-CUDA context not registered for instance ID {instance_id}" + ) + + prefetched_keys: list[ObjectKey] = [] + try: + with self.storage_manager.read_prefetched_results(obj_keys) as memory_objs: + if not memory_objs or len(memory_objs) != len(obj_keys): + return PrepareRetrieveResponse(success=False, data=b"", context={}) + prefetched_keys = obj_keys[: len(memory_objs)] + chunks = [] + for memory_obj in memory_objs: + if memory_obj.tensor is None: + return PrepareRetrieveResponse( + success=False, data=b"", context={} + ) + chunks.append(memory_obj.tensor.cpu().clone()) + return PrepareRetrieveResponse( + success=True, data=pickle.dumps(chunks), context={} + ) + finally: + if prefetched_keys: + self.storage_manager.finish_read_prefetched(prefetched_keys) + + @_lmcache_nvtx_annotate + def commit_retrieve( + self, + key: IPCCacheEngineKey, + instance_id: int, + ) -> bool: + """Finalize a retrieve operation. No-op for pickle mode. + + Args: + key: Cache key (unused for pickle). + instance_id: Worker instance identifier (unused for pickle). + + Returns: + Always ``True``. + """ + return True @_lmcache_nvtx_annotate def store( @@ -299,22 +555,18 @@ def store( that signals the completion of the store operation. The second element indicates whether the store operation was successful. """ - session = self.session_manager.get_or_create(key.request_id) - session.set_tokens(list(key.token_ids)) - chunk_hashes = [ - TokenHasher.hash_to_bytes(h) for h in session.get_hashes(key.start, key.end) - ] - st = time.perf_counter() + obj_keys = self._resolve_obj_keys(key) - assert key.worker_id is not None, "Must store with worker_id != None" - obj_keys = ipc_key_to_object_keys(key, chunk_hashes) - - assert instance_id in self.gpu_contexts, ( - f"KV cache not registered for GPU ID {instance_id}" + context = self.contexts.get(instance_id) + assert context is not None, ( + f"No context registered for instance ID {instance_id}" + ) + assert context.gpu_context is not None, ( + f"GPU context not registered for instance ID {instance_id}" ) - gpu_context = self.gpu_contexts[instance_id] - model_name = self.gpu_context_meta[instance_id][0] + gpu_context = context.gpu_context + model_name = context.model_name # ``blocks_per_chunk`` is counted in inference-engine-side # blocks (each block addresses @@ -490,22 +742,18 @@ def retrieve( that signals the completion of the retrieve operation. The second element indicates whether the key was successfully retrieved. """ - session = self.session_manager.get_or_create(key.request_id) - session.set_tokens(list(key.token_ids)) - chunk_hashes = [ - TokenHasher.hash_to_bytes(h) for h in session.get_hashes(key.start, key.end) - ] - st = time.perf_counter() + obj_keys = self._resolve_obj_keys(key) - assert key.worker_id is not None, "Must retrieve with worker_id != None" - obj_keys = ipc_key_to_object_keys(key, chunk_hashes) - - assert instance_id in self.gpu_contexts, ( - f"KV cache not registered for GPU ID {instance_id}" + context = self.contexts.get(instance_id) + assert context is not None, ( + f"No context registered for instance ID {instance_id}" + ) + assert context.gpu_context is not None, ( + f"GPU context not registered for instance ID {instance_id}" ) - gpu_context = self.gpu_contexts[instance_id] - model_name = self.gpu_context_meta[instance_id][0] + gpu_context = context.gpu_context + model_name = context.model_name # CPU-synchronous sentinel: a GPU retrieve is about to be enqueued. # Must be published via publish() (not publish_on_stream) so the @@ -675,18 +923,16 @@ def _find_layout_desc( model_name: str, world_size: int, ) -> MemoryLayoutDesc | None: - """Find layout desc from a matching GPU context. + """Find layout desc from a matching GPU or CPU context. Returns: - The layout descriptor, or None if no context - matches (model_name, world_size). + The layout descriptor, or None if no context matches + ``(model_name, world_size)``. GPU contexts are checked first, + then CPU contexts. """ - for gpu_id, (m, w) in self.gpu_context_meta.items(): - if m == model_name and w == world_size: - return get_layout_desc( - self.gpu_contexts[gpu_id], - self.chunk_size, - ) + for context in self.contexts.values(): + if context.model_name == model_name and context.world_size == world_size: + return context.get_layout_desc(self.chunk_size) return None def lookup( @@ -1002,13 +1248,18 @@ def report_status(self) -> dict: sm = self.storage_manager.report_status() gpu_context_meta: dict[str, dict] = {} - for gpu_id, meta in self.gpu_context_meta.items(): + non_cuda_context_meta: dict[str, dict] = {} + registered_gpu_ids: list[int] = [] + registered_non_cuda_ids: list[int] = [] + + for instance_id, context in self.contexts.items(): entry: dict = { - "model_name": meta[0], - "world_size": meta[1], + "model_name": context.model_name, + "world_size": context.world_size, } - ctx = self.gpu_contexts.get(gpu_id) - if ctx is not None: + if context.gpu_context is not None: + registered_gpu_ids.append(instance_id) + ctx = context.gpu_context entry["kv_cache_layout"] = { "num_layers": ctx.num_layers, "inference_engine_logical_block_size": ( @@ -1026,15 +1277,26 @@ def report_status(self) -> dict: "attention_backend": ctx.attention_backend, "cache_size_per_token": ctx.cache_size_per_token(), } - gpu_context_meta[str(gpu_id)] = entry + gpu_context_meta[str(instance_id)] = entry + continue + + if context.non_cuda_metadata is not None: + registered_non_cuda_ids.append(instance_id) + non_cuda_context_meta[str(instance_id)] = { + **entry, + "block_size": context.non_cuda_metadata.block_size, + "use_mla": context.non_cuda_metadata.use_mla, + } return { "is_healthy": sm["is_healthy"], "engine_type": self.__class__.__name__, "chunk_size": self.chunk_size, "hash_algorithm": self.token_hasher.hash_algorithm_name, - "registered_gpu_ids": list(self.gpu_contexts.keys()), + "registered_gpu_ids": registered_gpu_ids, "gpu_context_meta": gpu_context_meta, + "registered_non_cuda_instance_ids": registered_non_cuda_ids, + "non_cuda_context_meta": non_cuda_context_meta, "active_sessions": self.session_manager.active_count(), "active_prefetch_jobs": self._active_prefetch_count(), "storage_manager": sm, @@ -1086,7 +1348,7 @@ def close(self) -> None: logger.info("MPCacheEngine closed") # Release GPU contexts - self.gpu_contexts.clear() + self.contexts.clear() def _active_prefetch_count(self) -> int: """Return the number of active prefetch jobs (thread-safe).""" @@ -1170,6 +1432,12 @@ def run_cache_server( server, RequestType.UNREGISTER_KV_CACHE, engine.unregister_kv_cache ) add_handler_helper(server, RequestType.STORE, engine.store) + add_handler_helper( + server, + RequestType.REGISTER_KV_CACHE_NON_GPU_CONTEXT, + engine.register_kv_cache_non_gpu_context, + ) + add_handler_helper(server, RequestType.PREPARE_STORE, engine.prepare_store) add_handler_helper(server, RequestType.LOOKUP, engine.lookup) add_handler_helper( server, RequestType.QUERY_PREFETCH_STATUS, engine.query_prefetch_status @@ -1181,6 +1449,9 @@ def run_cache_server( ) add_handler_helper(server, RequestType.FREE_LOOKUP_LOCKS, engine.free_lookup_locks) add_handler_helper(server, RequestType.RETRIEVE, engine.retrieve) + add_handler_helper(server, RequestType.COMMIT_STORE, engine.commit_store) + add_handler_helper(server, RequestType.PREPARE_RETRIEVE, engine.prepare_retrieve) + add_handler_helper(server, RequestType.COMMIT_RETRIEVE, engine.commit_retrieve) add_handler_helper(server, RequestType.CLEAR, engine.clear) add_handler_helper(server, RequestType.GET_CHUNK_SIZE, engine.get_chunk_size) add_handler_helper(server, RequestType.PING, engine.ping) @@ -1194,7 +1465,14 @@ def run_cache_server( # Assign thread pools server.add_affinity_thread_pool( - [RequestType.STORE, RequestType.RETRIEVE], + [ + RequestType.STORE, + RequestType.RETRIEVE, + RequestType.PREPARE_STORE, + RequestType.COMMIT_STORE, + RequestType.PREPARE_RETRIEVE, + RequestType.COMMIT_RETRIEVE, + ], max_workers=mp_config.max_gpu_workers, ) server.add_normal_thread_pool( diff --git a/lmcache/v1/multiprocess/transfer_context.py b/lmcache/v1/multiprocess/transfer_context.py new file mode 100644 index 00000000000..2a598791bb6 --- /dev/null +++ b/lmcache/v1/multiprocess/transfer_context.py @@ -0,0 +1,409 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Transfer context abstractions for LMCache multiprocess worker adapters.""" + +# Standard +from abc import ABC, abstractmethod +from typing import Any, Callable, Protocol + +# Third Party +import torch + +# First Party +from lmcache import torch_dev +from lmcache.utils import EngineType, init_logger +from lmcache.v1.distributed.api import MemoryLayoutDesc +from lmcache.v1.gpu_connector.utils import LayoutHints, is_mla +from lmcache.v1.multiprocess.custom_types import RegisterNonGpuContextPayload +from lmcache.v1.multiprocess.futures import MessagingFuture +from lmcache.v1.multiprocess.mq import MessageQueueClient +from lmcache.v1.multiprocess.non_gpu_context import ( + NonGpuContext, + NonGpuContextMetadata, + compute_kv_layout, + create_non_gpu_context, + gather_paged_kv_to_cpu, + scatter_cpu_to_paged_kv, +) +from lmcache.v1.multiprocess.protocol import RequestType + +logger = init_logger(__name__) + + +class IPCEvent(Protocol): + """Protocol for IPC-capable CUDA events used by transport operations.""" + + def ipc_handle(self) -> object: + """Return an IPC handle consumable by the multiprocess server.""" + + +SendRequest = Callable[[MessageQueueClient, RequestType, list[object]], MessagingFuture] + + +class TransferContext(ABC): + """Abstract transport layer for worker-side KV transfer. + + Concrete implementations encapsulate how worker-side store/retrieve + operations are transmitted to the multiprocess server. CUDA paths return + CUDA-aware futures backed by MQ requests, while CPU paths may perform + gather/scatter synchronously and return already-resolved futures. + """ + + @abstractmethod + def register( + self, + instance_id: int, + kv_caches: dict[str, torch.Tensor], + model_name: str, + world_size: int, + blocks_in_chunk: int, + mq_client: MessageQueueClient, + mq_timeout: float, + send_request: SendRequest, + layout_hints: LayoutHints | None = None, + ) -> None: + """Register KV caches with the server and wait for ACK. + + Args: + instance_id: Worker process instance identifier. + kv_caches: Worker KV cache tensors keyed by layer name. + model_name: Model name used by cache keys. + world_size: KV world size. + blocks_in_chunk: Number of vLLM blocks per LMCache chunk. + mq_client: Message queue client used to communicate with server. + mq_timeout: Timeout in seconds for synchronous request wait. + send_request: Request sender callable used to issue MQ requests. + layout_hints: Optional inference-engine-provided layout hints. + + Raises: + TimeoutError: If server registration does not complete before + ``mq_timeout``. + RuntimeError: If a concrete context cannot initialize. + """ + + @abstractmethod + def submit_store( + self, + request_id: str, + key: Any, + instance_id: int, + kv_caches: dict[str, torch.Tensor], + block_ids: list[int], + event: IPCEvent, + blocks_in_chunk: int, + ) -> MessagingFuture: + """Submit a store request and return a completion future. + + Args: + request_id: External request identifier. + key: LMCache key object for the store range. + instance_id: Worker process instance identifier. + kv_caches: Worker KV cache tensors keyed by layer name. + block_ids: vLLM block IDs to store. + event: Synchronization event object. + blocks_in_chunk: Number of vLLM blocks per LMCache chunk. + + Returns: + A future compatible with adapter-side ``query()``/``result()`` flow. + + Raises: + RuntimeError: If register() was not called first. + """ + + @abstractmethod + def submit_retrieve( + self, + request_id: str, + key: Any, + instance_id: int, + kv_caches: dict[str, torch.Tensor], + block_ids: list[int], + event: IPCEvent, + blocks_in_chunk: int, + skip_first_n_tokens: int = 0, + ) -> MessagingFuture: + """Submit a retrieve request and return a completion future. + + Args: + request_id: External request identifier. + key: LMCache key object for the retrieve range. + instance_id: Worker process instance identifier. + kv_caches: Worker KV cache tensors keyed by layer name. + block_ids: vLLM block IDs to retrieve into. + event: Synchronization event object. + blocks_in_chunk: Number of vLLM blocks per LMCache chunk. + skip_first_n_tokens: Number of initial tokens to skip when writing. + + Returns: + A future compatible with adapter-side ``query()``/``result()`` flow. + + Raises: + RuntimeError: If register() was not called first. + """ + + @abstractmethod + def close(self) -> None: + """Release resources held by this context.""" + + +class HandleTransferContext(TransferContext): + """Handle-based IPC + MQ future transport context.""" + + def __init__(self) -> None: + self._mq_client: MessageQueueClient | None = None + self._send_request: SendRequest | None = None + + def register( + self, + instance_id: int, + kv_caches: dict[str, torch.Tensor], + model_name: str, + world_size: int, + _blocks_in_chunk: int, + mq_client: MessageQueueClient, + mq_timeout: float, + send_request: SendRequest, + layout_hints: LayoutHints | None = None, + ) -> None: + # First Party + from lmcache.integration.vllm.vllm_multi_process_adapter import wrap_kv_caches + + self._mq_client = mq_client + self._send_request = send_request + future = send_request( + mq_client, + RequestType.REGISTER_KV_CACHE, + [ + instance_id, + wrap_kv_caches(kv_caches), + model_name, + world_size, + EngineType.VLLM, + layout_hints, + ], + ) + future.result(timeout=mq_timeout) + + def submit_store( + self, + _request_id: str, + key: Any, + instance_id: int, + _kv_caches: dict[str, torch.Tensor], + block_ids: list[int], + event: IPCEvent, + _blocks_in_chunk: int, + ) -> MessagingFuture: + if self._mq_client is None or self._send_request is None: + raise RuntimeError( + "Handle transfer context is not registered. " + "Call register() before submit_store()." + ) + return self._send_request( + self._mq_client, + RequestType.STORE, + [key, instance_id, block_ids, event.ipc_handle()], + ).to_cuda_future() + + def submit_retrieve( + self, + _request_id: str, + key: Any, + instance_id: int, + _kv_caches: dict[str, torch.Tensor], + block_ids: list[int], + event: IPCEvent, + _blocks_in_chunk: int, + skip_first_n_tokens: int = 0, + ) -> MessagingFuture: + if self._mq_client is None or self._send_request is None: + raise RuntimeError( + "Handle transfer context is not registered. " + "Call register() before submit_retrieve()." + ) + return self._send_request( + self._mq_client, + RequestType.RETRIEVE, + [key, instance_id, block_ids, event.ipc_handle(), skip_first_n_tokens], + ).to_cuda_future() + + def close(self) -> None: + self._mq_client = None + self._send_request = None + + +class DataTransferContext(TransferContext): + """Data transfer context for non-CUDA workers.""" + + def __init__(self) -> None: + self._non_gpu_context: NonGpuContext | None = None + self._layout_hints: LayoutHints | None = None + self._gpu_kv_format: Any = None + + def register( + self, + instance_id: int, + kv_caches: dict[str, torch.Tensor], + model_name: str, + world_size: int, + blocks_in_chunk: int, + mq_client: MessageQueueClient, + mq_timeout: float, + send_request: SendRequest, + layout_hints: LayoutHints | None = None, + ) -> None: + # TODO: inference_engine_logical_block_size is currently used by + # DeepSeek V4 on the CUDA path. The non-CUDA path is yet to be + # implemented. + ( + block_size, + num_layers, + hidden_dim_size, + dtype_str, + gpu_kv_format, + ) = compute_kv_layout(kv_caches, layout_hints=layout_hints) + self._layout_hints = layout_hints + self._gpu_kv_format = gpu_kv_format + + use_mla_flag = is_mla(gpu_kv_format) + shape = ( + torch.Size([num_layers, blocks_in_chunk * block_size, hidden_dim_size]) + if use_mla_flag + else torch.Size( + [2, num_layers, blocks_in_chunk * block_size, hidden_dim_size] + ) + ) + dtype = getattr(torch, dtype_str) + layout_desc = MemoryLayoutDesc(shapes=[shape], dtypes=[dtype]) + + future = send_request( + mq_client, + RequestType.REGISTER_KV_CACHE_NON_GPU_CONTEXT, + [ + RegisterNonGpuContextPayload( + instance_id=instance_id, + model_name=model_name, + world_size=world_size, + block_size=block_size, + num_layers=num_layers, + hidden_dim_size=hidden_dim_size, + dtype_str=dtype_str, + use_mla=use_mla_flag, + ) + ], + ) + + metadata = NonGpuContextMetadata( + layout_desc=layout_desc, + block_size=block_size, + use_mla=use_mla_flag, + ) + self._non_gpu_context = create_non_gpu_context(metadata, mq_client, mq_timeout) + future.result(timeout=mq_timeout) + + def submit_store( + self, + _request_id: str, + key: Any, + instance_id: int, + kv_caches: dict[str, torch.Tensor], + block_ids: list[int], + _event: IPCEvent, + blocks_in_chunk: int, + ) -> MessagingFuture: + if self._non_gpu_context is None: + raise RuntimeError( + "Data transfer context is not registered. " + "Call register() before submit_store()." + ) + + torch_dev.synchronize() + out_buffers = self._non_gpu_context.prepare_store(key, instance_id) + cpu_chunks = gather_paged_kv_to_cpu( + kv_caches, + block_ids, + blocks_in_chunk, + layout_hints=self._layout_hints, + gpu_kv_format=self._gpu_kv_format, + out=out_buffers, + ) + ok = self._non_gpu_context.commit_store(key, instance_id, cpu_chunks) + + future: MessagingFuture[bool] = MessagingFuture() + future.set_result(ok) + return future + + def submit_retrieve( + self, + _request_id: str, + key: Any, + instance_id: int, + kv_caches: dict[str, torch.Tensor], + block_ids: list[int], + _event: IPCEvent, + blocks_in_chunk: int, + skip_first_n_tokens: int = 0, + ) -> MessagingFuture: + if self._non_gpu_context is None: + raise RuntimeError( + "Data transfer context is not registered. " + "Call register() before submit_retrieve()." + ) + + src_buffers = self._non_gpu_context.prepare_retrieve(key, instance_id) + ok = src_buffers is not None + if src_buffers is not None: + try: + scatter_cpu_to_paged_kv( + kv_caches, + block_ids, + src_buffers, + blocks_in_chunk, + skip_first_n_tokens=skip_first_n_tokens, + layout_hints=self._layout_hints, + gpu_kv_format=self._gpu_kv_format, + ) + except (RuntimeError, ValueError, TypeError, IndexError): + logger.exception("Failed to scatter retrieved CPU context chunks") + ok = False + self._non_gpu_context.commit_retrieve(key, instance_id) + + future: MessagingFuture[bool] = MessagingFuture() + future.set_result(ok) + return future + + def close(self) -> None: + if self._non_gpu_context is not None: + self._non_gpu_context.close() + self._non_gpu_context = None + + +def create_transfer_context( + kv_caches: dict[str, torch.Tensor], + **_kwargs: Any, +) -> TransferContext: + """Create a transfer context from KV cache device type. + + The device check is intentionally centralized here. + + Args: + kv_caches: Worker KV cache tensors keyed by layer name. + **kwargs: Unused placeholder for forward-compatible factory extension. + + Returns: + A concrete :class:`TransferContext` implementation. + + Raises: + ValueError: If ``kv_caches`` is empty or has mixed device types. + """ + if not kv_caches: + raise ValueError("kv_caches is empty") + device_types = {tensor.device.type for tensor in kv_caches.values()} + if len(device_types) != 1: + raise ValueError( + f"All KV cache tensors must share one device type, got {device_types}" + ) + device_type = next(iter(device_types)) + logger.info("Creating transfer context (device_type=%s)", device_type) + if device_type == "cuda": + return HandleTransferContext() + return DataTransferContext() diff --git a/tests/v1/multiprocess/test_non_cuda_data_transfer.py b/tests/v1/multiprocess/test_non_cuda_data_transfer.py new file mode 100644 index 00000000000..f8a281a8785 --- /dev/null +++ b/tests/v1/multiprocess/test_non_cuda_data_transfer.py @@ -0,0 +1,461 @@ +# SPDX-License-Identifier: Apache-2.0 +# Standard +from contextlib import contextmanager +from typing import Any, Callable +from unittest.mock import MagicMock, patch +import pickle +import sys + +# Third Party +import pytest +import torch + + +def _make_kv_caches( + num_layers: int = 2, + num_blocks: int = 6, + block_size: int = 4, + num_heads: int = 2, + head_size: int = 8, +) -> dict[str, torch.Tensor]: + """Build per-layer NHD KV tensors for non-CUDA data transfer tests.""" + kv_caches = {} + for i in range(num_layers): + kv_caches[f"layer_{i}"] = torch.randn( + 2, num_blocks, block_size, num_heads, head_size + ) + return kv_caches + + +def _make_mla_kv_caches( + num_layers: int = 2, + num_blocks: int = 6, + block_size: int = 4, + hidden_size: int = 16, +) -> dict[str, torch.Tensor]: + """Build per-layer MLA KV tensors for non-CUDA data transfer tests. + + Args: + num_layers: Number of KV layers to generate. + num_blocks: Number of paged blocks per layer. + block_size: Number of tokens per block. + hidden_size: Hidden size per token. + + Returns: + Mapping from layer name to MLA KV tensor with shape + ``[num_blocks, block_size, hidden_size]``. + """ + kv_caches = {} + for i in range(num_layers): + kv_caches[f"layer_{i}"] = torch.randn(num_blocks, block_size, hidden_size) + return kv_caches + + +def _make_hnd_kv_caches( + num_layers: int = 2, + num_blocks: int = 6, + block_size: int = 4, + num_heads: int = 2, + head_size: int = 8, +) -> dict[str, torch.Tensor]: + """Build per-layer HND KV tensors for non-CUDA data transfer tests.""" + kv_caches = {} + for i in range(num_layers): + kv_caches[f"layer_{i}"] = torch.randn( + 2, num_blocks, num_heads, block_size, head_size + ) + return kv_caches + + +def _make_hnd_flashinfer_kv_caches( + num_layers: int = 2, + num_blocks: int = 6, + block_size: int = 4, + num_heads: int = 2, + head_size: int = 8, +) -> dict[str, torch.Tensor]: + """Build per-layer HND flash-infer KV tensors for non-CUDA data transfer tests.""" + kv_caches = {} + for i in range(num_layers): + kv_caches[f"layer_{i}"] = torch.randn( + num_blocks, 2, num_heads, block_size, head_size + ) + return kv_caches + + +def test_wrap_kv_caches_wraps_all_tensors(monkeypatch: Any) -> None: + """Verify wrap_kv_caches wraps all provided KV tensors.""" + # First Party + from lmcache.integration.vllm import vllm_multi_process_adapter as adapter_mod + + kv_caches = _make_kv_caches() + monkeypatch.setattr( + adapter_mod, + "CudaIPCWrapper", + lambda tensor: ("wrapped", tensor), + ) + + wrapped = adapter_mod.wrap_kv_caches(kv_caches) + assert len(wrapped) == len(kv_caches) + + +def test_create_transfer_context_uses_non_cuda_context_on_cpu() -> None: + """Ensure transfer context factory returns DataTransferContext for CPU KV.""" + # First Party + from lmcache.v1.multiprocess.transfer_context import ( + DataTransferContext, + create_transfer_context, + ) + + context = create_transfer_context({"layer_0": torch.randn(2, 2)}) + assert isinstance(context, DataTransferContext) + + +def test_compute_kv_layout_and_gather_scatter_roundtrip() -> None: + """Validate layout extraction and gather/scatter round-trip on CPU tensors.""" + # First Party + from lmcache.v1.multiprocess.non_gpu_context import ( + compute_kv_layout, + gather_paged_kv_to_cpu, + scatter_cpu_to_paged_kv, + ) + + source = _make_kv_caches(num_layers=2, num_blocks=8, block_size=4) + ( + block_size, + num_layers, + hidden_dim, + dtype_str, + detected_kv_format, + ) = compute_kv_layout(source) + assert block_size == 4 + assert num_layers == 2 + assert hidden_dim == 16 + assert dtype_str == "float32" + assert detected_kv_format is not None + + blocks_per_chunk = 2 + gathered = gather_paged_kv_to_cpu(source, [0, 1], blocks_per_chunk) + destination = {name: torch.zeros_like(tensor) for name, tensor in source.items()} + scatter_cpu_to_paged_kv(destination, [4, 5], gathered, blocks_per_chunk) + + for name in source: + assert torch.allclose(source[name][:, 0], destination[name][:, 4]) + assert torch.allclose(source[name][:, 1], destination[name][:, 5]) + + +@pytest.mark.parametrize( + ("hnd_builder", "expected_format"), + [ + (_make_hnd_kv_caches, "NL_X_TWO_NB_NH_BS_HS"), + (_make_hnd_flashinfer_kv_caches, "NL_X_NB_TWO_NH_BS_HS"), + ], +) +def test_gather_scatter_roundtrip_hnd_layout( + hnd_builder: Callable[[int, int, int, int, int], dict[str, torch.Tensor]], + expected_format: str, +) -> None: + """Validate gather/scatter round-trip for HND vLLM KV layout.""" + # First Party + from lmcache.v1.multiprocess.non_gpu_context import ( + compute_kv_layout, + gather_paged_kv_to_cpu, + scatter_cpu_to_paged_kv, + ) + import lmcache.c_ops as lmc_ops + + source = hnd_builder(2, 8, 4, 2, 8) + layout_hints = {"kv_layout": "HND"} + ( + block_size, + num_layers, + hidden_dim, + dtype_str, + detected_kv_format, + ) = compute_kv_layout(source, layout_hints=layout_hints) + assert block_size == 4 + assert num_layers == 2 + assert hidden_dim == 16 + assert dtype_str == "float32" + assert detected_kv_format == getattr(lmc_ops.GPUKVFormat, expected_format) + + blocks_per_chunk = 2 + gathered = gather_paged_kv_to_cpu( + source, + [0, 1], + blocks_per_chunk, + layout_hints=layout_hints, + gpu_kv_format=detected_kv_format, + ) + destination = {name: torch.zeros_like(tensor) for name, tensor in source.items()} + scatter_cpu_to_paged_kv( + destination, + [4, 5], + gathered, + blocks_per_chunk, + layout_hints=layout_hints, + gpu_kv_format=detected_kv_format, + ) + + for name in source: + if detected_kv_format == lmc_ops.GPUKVFormat.NL_X_TWO_NB_NH_BS_HS: + assert torch.allclose(source[name][:, 0], destination[name][:, 4]) + assert torch.allclose(source[name][:, 1], destination[name][:, 5]) + else: + assert torch.allclose(source[name][0], destination[name][4]) + assert torch.allclose(source[name][1], destination[name][5]) + + +def test_scatter_respects_skip_first_n_tokens() -> None: + """Ensure scatter honors skip_first_n_tokens and preserves skipped blocks.""" + # First Party + from lmcache.v1.multiprocess.non_gpu_context import ( + gather_paged_kv_to_cpu, + scatter_cpu_to_paged_kv, + ) + + source = _make_kv_caches(num_layers=2, num_blocks=8, block_size=4) + destination = { + name: torch.full_like(tensor, 999.0) for name, tensor in source.items() + } + gathered = gather_paged_kv_to_cpu(source, [0, 1, 2, 3], blocks_per_chunk=4) + scatter_cpu_to_paged_kv( + destination, + [0, 1, 2, 3], + gathered, + blocks_per_chunk=4, + skip_first_n_tokens=8, + ) + + for name in destination: + assert torch.all(destination[name][:, 0] == 999.0) + assert torch.all(destination[name][:, 1] == 999.0) + assert torch.allclose(destination[name][:, 2], source[name][:, 2]) + assert torch.allclose(destination[name][:, 3], source[name][:, 3]) + + +def test_compute_kv_layout_and_gather_scatter_roundtrip_mla() -> None: + """Validate gather/scatter round-trip for MLA KV tensors.""" + # First Party + from lmcache.v1.multiprocess.non_gpu_context import ( + compute_kv_layout, + gather_paged_kv_to_cpu, + scatter_cpu_to_paged_kv, + ) + + source = _make_mla_kv_caches( + num_layers=2, num_blocks=8, block_size=4, hidden_size=16 + ) + ( + block_size, + num_layers, + hidden_dim, + dtype_str, + detected_kv_format, + ) = compute_kv_layout(source) + assert block_size == 4 + assert num_layers == 2 + assert hidden_dim == 16 + assert dtype_str == "float32" + assert detected_kv_format is not None + + blocks_per_chunk = 2 + gathered = gather_paged_kv_to_cpu(source, [0, 1], blocks_per_chunk) + destination = {name: torch.zeros_like(tensor) for name, tensor in source.items()} + scatter_cpu_to_paged_kv(destination, [4, 5], gathered, blocks_per_chunk) + + for name in source: + assert torch.allclose(source[name][0], destination[name][4]) + assert torch.allclose(source[name][1], destination[name][5]) + + +def test_compute_kv_layout_empty_raises_value_error() -> None: + """Ensure compute_kv_layout rejects empty KV cache input.""" + # First Party + from lmcache.v1.multiprocess.non_gpu_context import compute_kv_layout + + with pytest.raises(ValueError, match="kv_caches is empty"): + compute_kv_layout({}) + + +def test_scatter_mla_respects_skip_first_n_tokens() -> None: + """Ensure MLA scatter honors skip_first_n_tokens and preserves skipped blocks.""" + # First Party + from lmcache.v1.multiprocess.non_gpu_context import ( + gather_paged_kv_to_cpu, + scatter_cpu_to_paged_kv, + ) + + source = _make_mla_kv_caches( + num_layers=2, num_blocks=8, block_size=4, hidden_size=16 + ) + destination = { + name: torch.full_like(tensor, 999.0) for name, tensor in source.items() + } + gathered = gather_paged_kv_to_cpu(source, [0, 1, 2, 3], blocks_per_chunk=4) + scatter_cpu_to_paged_kv( + destination, + [0, 1, 2, 3], + gathered, + blocks_per_chunk=4, + skip_first_n_tokens=8, + ) + + for name in destination: + assert torch.all(destination[name][0] == 999.0) + assert torch.all(destination[name][1] == 999.0) + assert torch.allclose(destination[name][2], source[name][2]) + assert torch.allclose(destination[name][3], source[name][3]) + + +def test_scatter_mla_skip_past_chunk_keeps_destination_unchanged() -> None: + """Ensure MLA scatter is a no-op when skip_first_n_tokens exceeds chunk tokens.""" + # First Party + from lmcache.v1.multiprocess.non_gpu_context import ( + gather_paged_kv_to_cpu, + scatter_cpu_to_paged_kv, + ) + + source = _make_mla_kv_caches( + num_layers=2, num_blocks=8, block_size=4, hidden_size=16 + ) + destination = { + name: torch.full_like(tensor, 123.0) for name, tensor in source.items() + } + gathered = gather_paged_kv_to_cpu(source, [0, 1, 2, 3], blocks_per_chunk=4) + scatter_cpu_to_paged_kv( + destination, + [0, 1, 2, 3], + gathered, + blocks_per_chunk=4, + skip_first_n_tokens=40, + ) + + for name in destination: + assert torch.all(destination[name] == 123.0) + + +@pytest.fixture +def stub_native_storage_ops() -> Any: + """Stub native modules so server imports work in source-only test runs.""" + module = type(sys)("lmcache.native_storage_ops") + module.TTLLock = type("TTLLock", (), {}) # type: ignore[attr-defined] + module.Bitmap = type("Bitmap", (), {}) # type: ignore[attr-defined] + with patch.dict( + sys.modules, + { + "lmcache.native_storage_ops": module, + "cupy": MagicMock(), + }, + ): + yield + + +def test_server_register_and_find_non_cuda_context_layout( + stub_native_storage_ops: Any, +) -> None: + """Ensure non-CUDA registration stores metadata and lookup finds layout.""" + # First Party + from lmcache.v1.multiprocess.custom_types import RegisterNonGpuContextPayload + from lmcache.v1.multiprocess.server import MPCacheEngine + + with ( + patch("lmcache.v1.multiprocess.server.StorageManager"), + patch("lmcache.v1.multiprocess.server.TokenHasher"), + patch("lmcache.v1.multiprocess.server.SessionManager"), + patch("lmcache.v1.multiprocess.server.get_event_bus"), + ): + engine = MPCacheEngine(storage_manager_config=MagicMock(), chunk_size=16) + engine.register_kv_cache_non_gpu_context( + RegisterNonGpuContextPayload( + instance_id=1, + model_name="m", + world_size=1, + block_size=4, + num_layers=2, + hidden_dim_size=16, + dtype_str="float32", + use_mla=False, + ) + ) + + layout = engine._find_layout_desc("m", 1) + assert layout is not None + assert layout.shapes[0] == torch.Size([2, 2, 16, 16]) + + +def test_server_store_and_retrieve_cpu_chunks(stub_native_storage_ops: Any) -> None: + """Validate mocked server-side CPU chunk store and retrieve behavior.""" + # First Party + from lmcache.v1.multiprocess.custom_types import ( + IPCCacheEngineKey, + RegisterNonGpuContextPayload, + ) + from lmcache.v1.multiprocess.server import MPCacheEngine + + mock_storage = MagicMock() + target_tensor = torch.zeros(2, 2, 8, 16) + mock_memory_obj = MagicMock() + mock_memory_obj.tensor = target_tensor + mock_storage.reserve_write.return_value = {"obj": mock_memory_obj} + + @contextmanager + def _read_prefetched_results(_keys: Any) -> Any: + yield [mock_memory_obj] + + mock_storage.read_prefetched_results.side_effect = _read_prefetched_results + mock_session = MagicMock() + mock_session.get_hashes.return_value = [b"h"] + with ( + patch( + "lmcache.v1.multiprocess.server.StorageManager", + return_value=mock_storage, + ), + patch("lmcache.v1.multiprocess.server.TokenHasher"), + patch("lmcache.v1.multiprocess.server.SessionManager") as session_cls, + patch("lmcache.v1.multiprocess.server.get_event_bus"), + patch( + "lmcache.v1.multiprocess.server.ipc_key_to_object_keys", + return_value=["obj"], + ), + ): + session_cls.return_value.get_or_create.return_value = mock_session + engine = MPCacheEngine(storage_manager_config=MagicMock(), chunk_size=8) + + engine.register_kv_cache_non_gpu_context( + RegisterNonGpuContextPayload( + instance_id=2, + model_name="m", + world_size=1, + block_size=4, + num_layers=2, + hidden_dim_size=16, + dtype_str="float32", + use_mla=False, + ) + ) + payload = torch.ones(2, 2, 8, 16) + key = IPCCacheEngineKey.from_token_ids( + "m", + 1, + 0, + [1] * 8, + start=0, + end=8, + request_id="req", + ) + with patch( + "lmcache.v1.multiprocess.server.ipc_key_to_object_keys", + return_value=["obj"], + ): + store_ok = engine.commit_store(key, 2, pickle.dumps([payload])) + response = engine.prepare_retrieve(key, 2) + success = response.success + cpu_data = response.data + assert isinstance(store_ok, bool) + assert torch.allclose(mock_memory_obj.tensor, payload) + + assert success is True + recovered_chunks: list[torch.Tensor] = pickle.loads(cpu_data) + assert len(recovered_chunks) == 1 + assert torch.allclose(recovered_chunks[0], payload) diff --git a/tests/v1/test_vllm_mp_adapter.py b/tests/v1/test_vllm_mp_adapter.py index 91d0b1d120a..ef30d25327b 100644 --- a/tests/v1/test_vllm_mp_adapter.py +++ b/tests/v1/test_vllm_mp_adapter.py @@ -14,11 +14,13 @@ # Third Party import pytest +import torch # First Party from lmcache.integration.vllm import vllm_multi_process_adapter as adapter_mod from lmcache.integration.vllm.vllm_multi_process_adapter import ( LMCacheMPWorkerAdapter, + LoadStoreOp, ParallelStrategy, ) from lmcache.v1.multiprocess.protocol import RequestType @@ -81,7 +83,9 @@ def fake_adapter(monkeypatch): def test_register_kv_caches_updates_kv_caches_and_submits(fake_adapter): """Public register_kv_caches stores the dict and submits one request.""" adapter, send_mock, _ = fake_adapter - new_caches = {"layer.0": object(), "layer.1": object()} + fake_tensor = MagicMock() + fake_tensor.device.type = "cuda" + new_caches = {"layer.0": fake_tensor, "layer.1": fake_tensor} adapter.register_kv_caches(new_caches) @@ -97,4 +101,73 @@ def test_register_kv_caches_raises_connection_error_on_timeout(fake_adapter): future.result.side_effect = TimeoutError("server down") with pytest.raises(ConnectionError, match="did not respond"): - adapter.register_kv_caches({"layer.0": object()}) + fake_tensor = MagicMock() + fake_tensor.device.type = "cuda" + adapter.register_kv_caches({"layer.0": fake_tensor}) + + +def test_register_kv_caches_cpu_submits_non_gpu_context_registration( + fake_adapter, monkeypatch +): + """CPU KV cache registration routes to REGISTER_KV_CACHE_NON_GPU_CONTEXT.""" + adapter, send_mock, _ = fake_adapter + monkeypatch.setattr( + "lmcache.integration.vllm.utils.vllm_layout_hints", + lambda: {}, + raising=False, + ) + cpu_kv = {"layer.0": torch.randn(2, 8, 4, 2, 8)} + + adapter.register_kv_caches(cpu_kv) + + assert adapter.kv_caches is cpu_kv + assert send_mock.call_count == 1 + args, _kwargs = send_mock.call_args + assert args[1] == RequestType.REGISTER_KV_CACHE_NON_GPU_CONTEXT + assert len(args[2]) == 1 + + +def test_submit_store_request_tracks_returned_future(fake_adapter, monkeypatch): + """submit_store_request stores the returned future in store_futures.""" + adapter, _send_mock, _ = fake_adapter + monkeypatch.setattr(adapter, "_ensure_heartbeat_started", lambda: None) + fake_tensor = MagicMock() + fake_tensor.device.type = "cuda" + adapter.kv_caches = {"layer.0": fake_tensor} + transfer_ctx = MagicMock() + fake_future = MagicMock() + transfer_ctx.submit_store.return_value = fake_future + adapter.transfer_ctx = transfer_ctx + op = LoadStoreOp(token_ids=[1, 2, 3, 4], block_ids=[0], start=0, end=4) + + adapter.submit_store_request("req-1", op, event=MagicMock()) + + assert transfer_ctx.submit_store.called + assert transfer_ctx.submit_store.call_args.kwargs == {} + assert adapter.store_futures["req-1"] is fake_future + + +def test_submit_retrieve_request_tracks_returned_future(fake_adapter, monkeypatch): + """submit_retrieve_request stores returned future and block IDs.""" + adapter, _send_mock, _ = fake_adapter + monkeypatch.setattr(adapter, "_ensure_heartbeat_started", lambda: None) + fake_tensor = MagicMock() + fake_tensor.device.type = "cuda" + adapter.kv_caches = {"layer.0": fake_tensor} + transfer_ctx = MagicMock() + fake_future = MagicMock() + transfer_ctx.submit_retrieve.return_value = fake_future + adapter.transfer_ctx = transfer_ctx + op = LoadStoreOp( + token_ids=[1, 2, 3, 4], + block_ids=[0], + start=0, + end=4, + skip_first_n_tokens=1, + ) + + adapter.submit_retrieve_request("req-1", op, event=MagicMock()) + + assert transfer_ctx.submit_retrieve.called + assert transfer_ctx.submit_retrieve.call_args.kwargs == {"skip_first_n_tokens": 1} + assert adapter.retrieve_futures["req-1"] == (fake_future, [0]) From 65bf93f8f38e4b8d800bf71f1285b771beb5482c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 May 2026 15:34:47 -0700 Subject: [PATCH 45/69] Update Chinese documentation translations (#3335) Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: deng451e <57919305+deng451e@users.noreply.github.com> --- .../api_reference/configurations.po | 126 +++- .../LC_MESSAGES/getting_started/quickstart.po | 112 ++-- .../getting_started/troubleshoot.po | 4 +- .../internal_api_server/common_apis.po | 162 +++--- .../internal_api_server/controller_apis.po | 126 ++-- .../dynamic_backend_management.po | 54 +- .../internal_api_server.po | 40 +- .../internal_api_server/vllm_apis.po | 390 ++++++------- .../LC_MESSAGES/kv_cache/async_loading.po | 76 +-- .../LC_MESSAGES/kv_cache/caching_policies.po | 8 +- .../LC_MESSAGES/kv_cache/multiprocess_mode.po | 6 +- .../zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po | 72 +-- .../kv_cache/storage_backends/3fs.po | 56 +- .../kv_cache/storage_backends/cpu_ram.po | 78 +-- .../storage_backends/custom_backend.po | 6 +- .../kv_cache/storage_backends/dax.po | 92 +-- .../kv_cache/storage_backends/eic.po | 20 +- .../kv_cache/storage_backends/gds.po | 114 ++-- .../kv_cache/storage_backends/hfbucket.po | 56 +- .../kv_cache/storage_backends/index.po | 6 +- .../kv_cache/storage_backends/infinistore.po | 88 +-- .../storage_backends/local_storage.po | 126 ++-- .../kv_cache/storage_backends/maru.po | 84 +-- .../kv_cache/storage_backends/mock.po | 34 +- .../kv_cache/storage_backends/mooncake.po | 206 +++---- .../kv_cache/storage_backends/nixl.po | 70 +-- .../kv_cache/storage_backends/redis.po | 134 ++--- .../kv_cache/storage_backends/resp.po | 288 ++++----- .../kv_cache/storage_backends/s3.po | 78 +-- .../storage_backends/sagemaker_hyperpod.po | 46 +- .../kv_cache/storage_backends/valkey.po | 104 ++-- .../kv_cache/storage_backends/weka.po | 46 +- .../kv_cache_management/check_finish.po | 4 +- .../LC_MESSAGES/kv_cache_management/clear.po | 24 +- .../kv_cache_management/compress.po | 30 +- .../LC_MESSAGES/kv_cache_management/health.po | 16 +- .../LC_MESSAGES/kv_cache_management/index.po | 80 +-- .../LC_MESSAGES/kv_cache_management/lookup.po | 26 +- .../LC_MESSAGES/kv_cache_management/move.po | 26 +- .../LC_MESSAGES/kv_cache_management/pin.po | 24 +- .../kv_cache_management/query_worker_info.po | 20 +- .../kv_cache_optimizations/blending.po | 18 +- .../compression/cachegen.po | 12 +- .../compression/index.po | 6 +- .../kv_cache_optimizations/layerwise.po | 120 ++-- .../zh_CN/LC_MESSAGES/mp/architecture.po | 320 +++++----- .../zh_CN/LC_MESSAGES/mp/configuration.po | 324 +++++------ .../locale/zh_CN/LC_MESSAGES/mp/deployment.po | 90 +-- .../locale/zh_CN/LC_MESSAGES/mp/http_api.po | 298 +++++----- .../locale/zh_CN/LC_MESSAGES/mp/index.po | 46 +- .../locale/zh_CN/LC_MESSAGES/mp/l2_storage.po | 478 +++++++-------- .../zh_CN/LC_MESSAGES/mp/observability.po | 548 +++++++++--------- .../locale/zh_CN/LC_MESSAGES/mp/operator.po | 434 +++++++------- .../locale/zh_CN/LC_MESSAGES/mp/quickstart.po | 76 +-- .../locale/zh_CN/LC_MESSAGES/mp/serde.po | 50 +- .../LC_MESSAGES/mp/tracing_and_debugging.po | 162 +++--- .../LC_MESSAGES/non_kv_cache/encoder_cache.po | 92 +-- .../production/docker_deployment.po | 37 +- .../production/kubernetes_deployment.po | 12 +- .../LC_MESSAGES/production/kv_cache_events.po | 58 +- .../observability/chunk_statistics.po | 256 ++++---- .../production/observability/frontend.po | 88 +-- .../observability/health_monitor.po | 214 +++---- .../production/observability/index.po | 2 +- .../observability/internal_api_server.po | 54 +- .../production/observability/metrics.po | 374 ++++++------ .../observability/periodic_thread_api.po | 96 +-- .../production/observability/vllm_endpoint.po | 194 +++---- .../production/performance_tuning.po | 44 +- .../zh_CN/LC_MESSAGES/recipes/devstral.po | 46 +- .../zh_CN/LC_MESSAGES/recipes/gemma4.po | 46 +- .../zh_CN/LC_MESSAGES/recipes/gpt_oss.po | 50 +- .../locale/zh_CN/LC_MESSAGES/recipes/index.po | 82 +-- .../zh_CN/LC_MESSAGES/recipes/minimax_m2.po | 58 +- .../locale/zh_CN/LC_MESSAGES/recipes/qwen3.po | 58 +- 75 files changed, 4042 insertions(+), 3959 deletions(-) diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/configurations.po b/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/configurations.po index 88df708b347..5609a191a88 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/configurations.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/api_reference/configurations.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: LMCache \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"POT-Creation-Date: 2026-05-19 20:13+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language: zh_CN\n" @@ -151,7 +151,10 @@ msgid "" "``\"file:///path/to/cache\"`` or ``\"/path/a,/path/b\"`` for multi-device" " I/O. See ``local_disk_path_sharding`` for how paths are assigned to " "GPUs." -msgstr "本地磁盘缓存目录的路径（或以逗号分隔的路径）。格式：``\"file:///path/to/cache\"`` 或 ``\"/path/a,/path/b\"`` 用于多设备 I/O。有关路径如何分配给 GPU，请参见 ``local_disk_path_sharding``。" +msgstr "" +"本地磁盘缓存目录的路径（或以逗号分隔的路径）。格式：``\"file:///path/to/cache\"`` 或 " +"``\"/path/a,/path/b\"`` 用于多设备 I/O。有关路径如何分配给 GPU，请参见 " +"``local_disk_path_sharding``。" #: ../../source/api_reference/configurations.rst:40 msgid "local_disk_path_sharding" @@ -167,7 +170,9 @@ msgid "" "Strategy for selecting a path when multiple paths are provided. Currently" " only ``\"by_gpu\"`` is supported, which selects paths based on GPU " "device ID (default: \"by_gpu\")." -msgstr "当提供多个路径时选择路径的策略。目前仅支持 ``\"by_gpu\"``，该策略根据 GPU 设备 ID 选择路径（默认值：\\\"by_gpu\\\"）。" +msgstr "" +"当提供多个路径时选择路径的策略。目前仅支持 ``\"by_gpu\"``，该策略根据 GPU 设备 ID " +"选择路径（默认值：\\\"by_gpu\\\"）。" #: ../../source/api_reference/configurations.rst:43 msgid "max_local_disk_size" @@ -307,7 +312,9 @@ msgid "" " \"manual\" (use extra_config mapping), null (disabled). When enabled, " "allocates pinned memory on specific NUMA nodes for better GPU-CPU memory " "bandwidth. Default: null" -msgstr "NUMA 感知内存分配模式。值：\\\"auto\\\"（从系统检测）、\\\"manual\\\"（使用 extra_config 映射）、null（禁用）。启用时，在特定的 NUMA 节点上分配固定内存，以提高 GPU-CPU 内存带宽。默认值：null" +msgstr "" +"NUMA 感知内存分配模式。值：\\\"auto\\\"（从系统检测）、\\\"manual\\\"（使用 extra_config " +"映射）、null（禁用）。启用时，在特定的 NUMA 节点上分配固定内存，以提高 GPU-CPU 内存带宽。默认值：null" #: ../../source/api_reference/configurations.rst:76 msgid "external_lookup_client" @@ -321,7 +328,9 @@ msgstr "LMCACHE_EXTERNAL_LOOKUP_CLIENT" msgid "" "External KV lookup service URI (e.g., \"mooncakestore://address\"). If " "null, defaults to LMCache's internal lookup client. Default: null" -msgstr "外部 KV 查找服务 URI（例如，\\\"mooncakestore://address\\\"）。如果为 null，则默认为 LMCache 的内部查找客户端。默认值：null" +msgstr "" +"外部 KV 查找服务 URI（例如，\\\"mooncakestore://address\\\"）。如果为 null，则默认为 LMCache " +"的内部查找客户端。默认值：null" #: ../../source/api_reference/configurations.rst:79 msgid "priority_limit" @@ -351,7 +360,9 @@ msgid "" "< this value, skip retrieve but still record the hits to avoid re-storing" " existing chunks. See :ref:`performance_tuning` for a working example. " "Default: 0 (disabled)" -msgstr "执行检索所需的最小命中令牌数。如果命中令牌少于此值，则跳过检索，但仍记录命中以避免重新存储现有块。有关工作示例，请参见 :ref:`performance_tuning`。默认值：0（禁用）" +msgstr "" +"执行检索所需的最小命中令牌数。如果命中令牌少于此值，则跳过检索，但仍记录命中以避免重新存储现有块。有关工作示例，请参见 " +":ref:`performance_tuning`。默认值：0（禁用）" #: ../../source/api_reference/configurations.rst:85 msgid "store_location" @@ -372,7 +383,11 @@ msgid "" "store location for a decoder instance in a PD setup, since PDBackend is " "one-way from prefiller to decoder only. Default: null (store to all " "active backends)" -msgstr "一个存储后端名称，用于存储 KV 缓存。当指定时，只有匹配的后端接收存储操作。有效值是存储管理器中注册的后端类名称，包括：``\\\"LocalCPUBackend\\\"``、``\\\"LocalDiskBackend\\\"``、``\\\"RemoteBackend\\\"``、``\\\"PDBackend\\\"``、``\\\"P2PBackend\\\"``、``\\\"GdsBackend\\\"`` 等，以及任何存储插件后端。注意：``\\\"PDBackend\\\"`` 不能用作 PD 设置中解码器实例的存储位置，因为 PDBackend 仅从预填充器到解码器是单向的。默认值：null（存储到所有活动后端）" +msgstr "" +"一个存储后端名称，用于存储 KV " +"缓存。当指定时，只有匹配的后端接收存储操作。有效值是存储管理器中注册的后端类名称，包括：``\\\"LocalCPUBackend\\\"``、``\\\"LocalDiskBackend\\\"``、``\\\"RemoteBackend\\\"``、``\\\"PDBackend\\\"``、``\\\"P2PBackend\\\"``、``\\\"GdsBackend\\\"``" +" 等，以及任何存储插件后端。注意：``\\\"PDBackend\\\"`` 不能用作 PD 设置中解码器实例的存储位置，因为 PDBackend" +" 仅从预填充器到解码器是单向的。默认值：null（存储到所有活动后端）" #: ../../source/api_reference/configurations.rst:88 msgid "retrieve_locations" @@ -391,7 +406,10 @@ msgid "" "``\"RemoteBackend\"``, ``\"PDBackend\"``, ``\"P2PBackend\"``, " "``\"GdsBackend\"``, etc, and any storage plugin backends. Default: null " "(search all active backends)" -msgstr "检索或查找 KV Cache 时要搜索的存储后端名称列表。当指定时，仅搜索列出的后端。有效值为在存储管理器中注册的后端类名称，包括：``\\\"LocalCPUBackend\\\"``、``\\\"LocalDiskBackend\\\"``、``\\\"RemoteBackend\\\"``、``\\\"PDBackend\\\"``、``\\\"P2PBackend\\\"``、``\\\"GdsBackend\\\"`` 等，以及任何存储插件后端。默认值：null（搜索所有活动后端）" +msgstr "" +"检索或查找 KV Cache " +"时要搜索的存储后端名称列表。当指定时，仅搜索列出的后端。有效值为在存储管理器中注册的后端类名称，包括：``\\\"LocalCPUBackend\\\"``、``\\\"LocalDiskBackend\\\"``、``\\\"RemoteBackend\\\"``、``\\\"PDBackend\\\"``、``\\\"P2PBackend\\\"``、``\\\"GdsBackend\\\"``" +" 等，以及任何存储插件后端。默认值：null（搜索所有活动后端）" #: ../../source/api_reference/configurations.rst:91 msgid "extra_config" @@ -407,7 +425,9 @@ msgstr "LMCACHE_EXTRA_CONFIG={\\\"key\\\": value, ...}" msgid "" "Additional configuration as JSON dict. For NUMA manual mode, include " "\"gpu_to_numa_mapping\": {gpu_id: numa_node, ...}. Default: {}" -msgstr "额外的配置作为 JSON 字典。对于 NUMA 手动模式，包含 \\\"gpu_to_numa_mapping\\\": {gpu_id: numa_node, ...}。默认值：{}" +msgstr "" +"额外的配置作为 JSON 字典。对于 NUMA 手动模式，包含 \\\"gpu_to_numa_mapping\\\": {gpu_id: " +"numa_node, ...}。默认值：{}" #: ../../source/api_reference/configurations.rst:96 msgid "Lazy Memory Allocator Configurations" @@ -425,7 +445,9 @@ msgid "" " configurations. It starts with a small initial allocation and gradually " "expands as needed, reducing startup wait time and avoiding unnecessary " "memory consumption when the full capacity is not required." -msgstr "懒惰内存分配器旨在处理大 CPU 内存配置的场景。它从小的初始分配开始，并根据需要逐渐扩展，从而减少启动等待时间，并避免在不需要全部容量时的不必要内存消耗。" +msgstr "" +"懒惰内存分配器旨在处理大 CPU " +"内存配置的场景。它从小的初始分配开始，并根据需要逐渐扩展，从而减少启动等待时间，并避免在不需要全部容量时的不必要内存消耗。" #: ../../source/api_reference/configurations.rst:104 msgid "**Key characteristics:**" @@ -519,7 +541,9 @@ msgid "" "Threshold in GB above which lazy allocator activates. If " "max_local_cpu_size ≤ this value, lazy allocator is disabled regardless of" " enable_lazy_memory_allocator setting. Default: 0.0" -msgstr "在 GB 中的阈值，超过该值后懒惰分配器激活。如果 max_local_cpu_size ≤ 此值，则无论 enable_lazy_memory_allocator 设置如何，懒惰分配器都将被禁用。默认值：0.0" +msgstr "" +"在 GB 中的阈值，超过该值后懒惰分配器激活。如果 max_local_cpu_size ≤ 此值，则无论 " +"enable_lazy_memory_allocator 设置如何，懒惰分配器都将被禁用。默认值：0.0" #: ../../source/api_reference/configurations.rst:132 msgid "reserve_local_cpu_size" @@ -549,7 +573,10 @@ msgid "" "`_. We " "also have more :doc:`detailed documentation " "<../kv_cache_optimizations/blending>`." -msgstr "我们有一个端到端的 `示例 `_。我们还有更多的 :doc:`详细文档 <../kv_cache_optimizations/blending>`。" +msgstr "" +"我们有一个端到端的 `示例 " +"`_。我们还有更多的" +" :doc:`详细文档 <../kv_cache_optimizations/blending>`。" #: ../../source/api_reference/configurations.rst:153 msgid "enable_blending" @@ -735,7 +762,9 @@ msgstr "分离式 Prefill 配置" msgid "" "Settings for disaggregated prefill functionality. The latest/default PD " "is implemented inside of `lmcache/v1/storage_backend/pd_backend.py`." -msgstr "分离式 Prefill 功能的设置。最新/默认的 PD 实现在 `lmcache/v1/storage_backend/pd_backend.py` 中。" +msgstr "" +"分离式 Prefill 功能的设置。最新/默认的 PD 实现在 " +"`lmcache/v1/storage_backend/pd_backend.py` 中。" #: ../../source/api_reference/configurations.rst:226 msgid "" @@ -946,7 +975,9 @@ msgid "" "Maximum prefill token length that the PD buffer must be able to hold. If " "> 0, initialization raises ValueError when the buffer capacity (in " "tokens) is smaller than this value. Set to 0 (default) to skip the check." -msgstr "PD 缓冲区必须能够容纳的最大 Prefill 令牌长度。如果大于 0，当缓冲区容量（以令牌为单位）小于此值时，初始化将引发 ValueError。设置为 0（默认值）以跳过检查。" +msgstr "" +"PD 缓冲区必须能够容纳的最大 Prefill 令牌长度。如果大于 0，当缓冲区容量（以令牌为单位）小于此值时，初始化将引发 " +"ValueError。设置为 0（默认值）以跳过检查。" #: ../../source/api_reference/configurations.rst:284 msgid "pd_backend_mode" @@ -961,7 +992,9 @@ msgid "" "Select the PD backend implementation: 'async' (default) uses the asyncio-" "based implementation; 'sync' uses the original thread-based synchronous " "implementation. Default: \"async\"" -msgstr "选择 PD 后端实现：'async'（默认）使用基于 asyncio 的实现；'sync' 使用原始的基于线程的同步实现。默认值：\\\"async\\\"" +msgstr "" +"选择 PD 后端实现：'async'（默认）使用基于 asyncio 的实现；'sync' " +"使用原始的基于线程的同步实现。默认值：\\\"async\\\"" #: ../../source/api_reference/configurations.rst:287 msgid "pd_skip_proxy_notification" @@ -981,7 +1014,11 @@ msgid "" " proxy (``disagg_proxy_server.py``), which depends on ZMQ notifications " "to know when KV transfer is complete before forwarding the decode " "request. Values: true/false. Default: false" -msgstr "当为真时，发送方在 KV 传输后跳过 ZMQ 代理通知，并且不需要 pd_proxy_host/pd_proxy_port。此选项仅适用于管理通过 HTTP 的分离式解码请求流的外部调度器（例如，vLLM 生产堆栈路由器），并且不依赖于 ZMQ 通知。它不得与 LMCache 的内置分离代理（``disagg_proxy_server.py``）一起使用，因为该代理依赖于 ZMQ 通知来知道 KV 传输何时完成，然后再转发解码请求。值：true/false。默认值：false" +msgstr "" +"当为真时，发送方在 KV 传输后跳过 ZMQ 代理通知，并且不需要 pd_proxy_host/pd_proxy_port。此选项仅适用于管理通过" +" HTTP 的分离式解码请求流的外部调度器（例如，vLLM 生产堆栈路由器），并且不依赖于 ZMQ 通知。它不得与 LMCache " +"的内置分离代理（``disagg_proxy_server.py``）一起使用，因为该代理依赖于 ZMQ 通知来知道 KV " +"传输何时完成，然后再转发解码请求。值：true/false。默认值：false" #: ../../source/api_reference/configurations.rst:290 msgid "pd_bidirectional" @@ -996,7 +1033,9 @@ msgid "" "When true, enables bidirectional NIXL cache probe. The prefiller queries " "the decoder for cached KV blocks before transfer, and reads cached blocks" " via NIXL RDMA instead of recomputing. Values: true/false. Default: false" -msgstr "当为真时，启用双向 NIXL 缓存探测。Prefiller 在传输之前查询解码器以获取缓存的 KV 块，并通过 NIXL RDMA 读取缓存的块，而不是重计算。值：true/false。默认值：false" +msgstr "" +"当为真时，启用双向 NIXL 缓存探测。Prefiller 在传输之前查询解码器以获取缓存的 KV 块，并通过 NIXL RDMA " +"读取缓存的块，而不是重计算。值：true/false。默认值：false" #: ../../source/api_reference/configurations.rst:293 msgid "pd_peer_query_port" @@ -1011,7 +1050,9 @@ msgid "" "ZMQ ports for the bidirectional cache query channel (one per TP rank). " "Required on both prefiller and decoder when pd_bidirectional=true. " "Example: [7500, 7501, 7502, 7503]" -msgstr "用于双向缓存查询通道的 ZMQ 端口（每个 TP 排名一个）。当 pd_bidirectional=true 时，预填充器和解码器都需要此配置。示例：[7500, 7501, 7502, 7503]" +msgstr "" +"用于双向缓存查询通道的 ZMQ 端口（每个 TP 排名一个）。当 pd_bidirectional=true " +"时，预填充器和解码器都需要此配置。示例：[7500, 7501, 7502, 7503]" #: ../../source/api_reference/configurations.rst:298 msgid "P2P Backend Configurations" @@ -1113,9 +1154,13 @@ msgstr "nixl_endpoint_list" #: ../../source/api_reference/configurations.rst:356 msgid "" -"List of object-storage endpoint URLs for per-worker distribution. " -"Overrides ``nixl_backend_params.endpoint_override`` when set." -msgstr "每个工作节点分发的对象存储端点 URL 列表。当设置时，会覆盖 ``nixl_backend_params.endpoint_override``。" +"List of object-storage endpoint URLs for per-worker distribution. Each TP" +" worker selects an entry round-robin by ``local_worker_id``, overriding " +"``nixl_backend_params.endpoint_override``. Only applied when " +"``nixl_backend`` is ``\"OBJ\"`` (silently ignored otherwise). Each entry " +"must start with ``http://`` or ``https://``; an empty list raises " +"``ValueError`` at engine init." +msgstr "每个工作节点的对象存储端点 URL 列表。每个 TP 工作节点通过 ``local_worker_id`` 轮询选择一个条目，覆盖 ``nixl_backend_params.endpoint_override``。仅在 ``nixl_backend`` 为 ``\"OBJ\"`` 时应用（否则静默忽略）。每个条目必须以 ``http://`` 或 ``https://`` 开头；空列表在引擎初始化时会引发 ``ValueError``。" #: ../../source/api_reference/configurations.rst:360 msgid "Additional Storage Configurations" @@ -1138,7 +1183,10 @@ msgid "" "Path for GDS backend. Supports comma-separated paths for multi-device I/O" " (e.g. ``/mnt/nvme0/cache,/mnt/nvme1/cache``). See ``gds_path_sharding`` " "for how paths are assigned to GPUs." -msgstr "GDS 后端的路径。支持以逗号分隔的多设备 I/O 路径（例如 ``/mnt/nvme0/cache,/mnt/nvme1/cache``）。有关路径如何分配给 GPU，请参见 ``gds_path_sharding``。" +msgstr "" +"GDS 后端的路径。支持以逗号分隔的多设备 I/O 路径（例如 " +"``/mnt/nvme0/cache,/mnt/nvme1/cache``）。有关路径如何分配给 GPU，请参见 " +"``gds_path_sharding``。" #: ../../source/api_reference/configurations.rst:374 msgid "gds_path_sharding" @@ -1194,7 +1242,9 @@ msgid "" "histogram metric by adding a key ``histogram_bucket_`` to " "``extra_config``, where ```` is the metric name **without** the " "``lmcache:`` prefix." -msgstr "您可以通过向 ``extra_config`` 添加一个键 ``histogram_bucket_`` 来覆盖任何 Prometheus 直方图度量的默认桶边界，其中 ```` 是度量名称 **不带** ``lmcache:`` 前缀。" +msgstr "" +"您可以通过向 ``extra_config`` 添加一个键 ``histogram_bucket_`` 来覆盖任何 " +"Prometheus 直方图度量的默认桶边界，其中 ```` 是度量名称 **不带** ``lmcache:`` 前缀。" #: ../../source/api_reference/configurations.rst:394 msgid "The value must be a list of numeric boundaries (floats or ints)." @@ -1374,7 +1424,9 @@ msgid "" "debugging APIs for LMCache engines. The API server runs on each worker " "and scheduler, allowing you to inspect and control LMCache behavior at " "runtime." -msgstr "用于内部 API 服务器的设置，该服务器为 LMCache 引擎提供管理和调试 API。API 服务器在每个工作节点和调度器上运行，允许您在运行时检查和控制 LMCache 的行为。" +msgstr "" +"用于内部 API 服务器的设置，该服务器为 LMCache 引擎提供管理和调试 API。API " +"服务器在每个工作节点和调度器上运行，允许您在运行时检查和控制 LMCache 的行为。" #: ../../source/api_reference/configurations.rst:456 msgid "The internal API server provides endpoints for:" @@ -1448,7 +1500,9 @@ msgid "" "port_start + 0, Worker i = port_start + i + 1. Example: If " "port_start=6999, then Scheduler=6999, Worker 0=7000, Worker 1=7001, etc. " "Default: 6999" -msgstr "内部 API 服务器的起始端口。端口分配：调度器 = port_start + 0，工作者 i = port_start + i + 1。示例：如果 port_start=6999，则调度器=6999，工作者 0=7000，工作者 1=7001，等等。默认值：6999" +msgstr "" +"内部 API 服务器的起始端口。端口分配：调度器 = port_start + 0，工作者 i = port_start + i + " +"1。示例：如果 port_start=6999，则调度器=6999，工作者 0=7000，工作者 1=7001，等等。默认值：6999" #: ../../source/api_reference/configurations.rst:484 msgid "internal_api_server_include_index_list" @@ -1464,7 +1518,9 @@ msgid "" "scheduler, 1 for worker 0, 2 for worker 1, etc. If null, enables on all " "workers/scheduler. Example: [0, 1] enables only on scheduler and worker " "0. Default: null" -msgstr "启用 API 服务器的工作者/调度器索引列表。使用 0 表示调度器，1 表示工作者 0，2 表示工作者 1，依此类推。如果为 null，则在所有工作者/调度器上启用。示例：[0, 1] 仅在调度器和工作者 0 上启用。默认值：null" +msgstr "" +"启用 API 服务器的工作者/调度器索引列表。使用 0 表示调度器，1 表示工作者 0，2 表示工作者 1，依此类推。如果为 " +"null，则在所有工作者/调度器上启用。示例：[0, 1] 仅在调度器和工作者 0 上启用。默认值：null" #: ../../source/api_reference/configurations.rst:487 msgid "internal_api_server_socket_path_prefix" @@ -1481,7 +1537,11 @@ msgid "" "will be \"{prefix}_{port}\". Example: \"/tmp/lmcache_api_socket\" creates" " \"/tmp/lmcache_api_socket_6999\", \"/tmp/lmcache_api_socket_7000\", etc." " Default: null" -msgstr "如果指定，则使用 Unix 域套接字而不是 TCP 端口。套接字路径将为 \\\"{prefix}_{port}\\\"。示例：\\\"/tmp/lmcache_api_socket\\\" 创建 \\\"/tmp/lmcache_api_socket_6999\\\"、\\\"/tmp/lmcache_api_socket_7000\\\" 等。默认值：null" +msgstr "" +"如果指定，则使用 Unix 域套接字而不是 TCP 端口。套接字路径将为 " +"\\\"{prefix}_{port}\\\"。示例：\\\"/tmp/lmcache_api_socket\\\" 创建 " +"\\\"/tmp/lmcache_api_socket_6999\\\"、\\\"/tmp/lmcache_api_socket_7000\\\"" +" 等。默认值：null" #: ../../source/api_reference/configurations.rst:492 msgid "Plugin Configurations" @@ -1523,5 +1583,15 @@ msgstr "LMCACHE_AUDIT_ACTUAL_REMOTE_URL" msgid "" "(Deprecated) URL of actual remote LMCache instance for auditing. Use " "extra_config['audit_actual_remote_url'] instead" -msgstr "(已弃用) 用于审计的实际远程 LMCache 实例的 URL。请改用 extra_config['audit_actual_remote_url']。" +msgstr "" +"(已弃用) 用于审计的实际远程 LMCache 实例的 URL。请改用 " +"extra_config['audit_actual_remote_url']。" + +#~ msgid "" +#~ "List of object-storage endpoint URLs " +#~ "for per-worker distribution. Overrides " +#~ "``nixl_backend_params.endpoint_override`` when set." +#~ msgstr "" +#~ "每个工作节点分发的对象存储端点 URL 列表。当设置时，会覆盖 " +#~ "``nixl_backend_params.endpoint_override``。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart.po index 2dac320fc07..d8a886c532d 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart.po @@ -21,26 +21,26 @@ msgstr "" #: ../../source/getting_started/quickstart.rst:4 msgid "Quickstart" -msgstr "" +msgstr "快速入门" #: ../../source/getting_started/quickstart.rst:6 msgid "" "This guide helps you get LMCache running end-to-end in a couple of " "minutes. Use the tabs below to switch the engine. Steps are the same; " "only the libraries and launch commands change." -msgstr "" +msgstr "本指南帮助您在几分钟内完成 LMCache 的端到端运行。使用下面的选项卡切换引擎。步骤是相同的；只有库和启动命令有所不同。" #: ../../source/getting_started/quickstart.rst msgid "vLLM" -msgstr "" +msgstr "vLLM" #: ../../source/getting_started/quickstart.rst:13 msgid "**Install LMCache**" -msgstr "" +msgstr "**安装 LMCache**" #: ../../source/getting_started/quickstart.rst:21 msgid "LMCache supports two deployment modes with vLLM:" -msgstr "" +msgstr "LMCache 支持与 vLLM 的两种部署模式：" #: ../../source/getting_started/quickstart.rst:23 msgid "" @@ -48,23 +48,23 @@ msgid "" "standalone service and vLLM attaches via ``LMCacheMPConnector``. Scales " "better, exposes management/observability endpoints, and supports sharing " "one cache across multiple engine instances." -msgstr "" +msgstr "**多进程 (MP) 模式** -- **推荐。** LMCache 作为独立服务运行，vLLM 通过 ``LMCacheMPConnector`` 连接。扩展性更好，暴露管理/可观察性端点，并支持在多个引擎实例之间共享一个缓存。" #: ../../source/getting_started/quickstart.rst:27 msgid "" "**In-process mode** -- LMCache runs inside the vLLM process via " "``LMCacheConnectorV1``. Single command, convenient for quick single-node " "experiments." -msgstr "" +msgstr "**进程内模式** -- LMCache 通过 ``LMCacheConnectorV1`` 在 vLLM 进程内部运行。单个命令，方便进行快速的单节点实验。" #: ../../source/getting_started/quickstart.rst msgid "MP mode (recommended)" -msgstr "" +msgstr "MP 模式（推荐）" #: ../../source/getting_started/quickstart.rst:37 #: ../../source/getting_started/quickstart.rst:390 msgid "Start the LMCache server:" -msgstr "" +msgstr "启动 LMCache 服务器：" #: ../../source/getting_started/quickstart.rst:47 msgid "" @@ -72,17 +72,17 @@ msgid "" "frontend (default **8080**) serves the management and metrics endpoints. " "See :doc:`../mp/quickstart` and :doc:`../mp/configuration` for the full " "list of ``lmcache server`` and connector options." -msgstr "" +msgstr "ZMQ 端口（默认 **5555**）接受来自 vLLM 的连接；HTTP 前端（默认 **8080**）提供管理和指标端点。有关 ``lmcache server`` 和连接器选项的完整列表，请参见 :doc:`../mp/quickstart` 和 :doc:`../mp/configuration`。" #: ../../source/getting_started/quickstart.rst:53 msgid "Start vLLM with the MP connector in a separate terminal:" -msgstr "" +msgstr "在单独的终端中启动 vLLM 与 MP 连接器：" #: ../../source/getting_started/quickstart.rst:62 msgid "" "**Where does** ``LMCacheMPConnector`` **resolve to?** This depends on " "your vLLM version:" -msgstr "" +msgstr "**``LMCacheMPConnector`` **解析到哪里？** 这取决于您的 vLLM 版本：" #: ../../source/getting_started/quickstart.rst:64 msgid "" @@ -90,7 +90,7 @@ msgid "" "resolves to vLLM's built-in " "``vllm.distributed.kv_transfer.kv_connector.v1.LMCacheMPConnector``; " "there is no way to redirect it to the LMCache-shipped implementation." -msgstr "" +msgstr "**vLLM < 0.20.0** -- ``\"kv_connector\":\"LMCacheMPConnector\"`` 始终解析为 vLLM 内置的 ``vllm.distributed.kv_transfer.kv_connector.v1.LMCacheMPConnector``；无法将其重定向到 LMCache 提供的实现。" #: ../../source/getting_started/quickstart.rst:69 msgid "" @@ -99,14 +99,14 @@ msgid "" "shipped implementation " "(:mod:`lmcache.integration.vllm.lmcache_mp_connector`) by adding " "``kv_connector_module_path``:" -msgstr "" +msgstr "**vLLM >= 0.20.0** -- ``\"kv_connector\":\"LMCacheMPConnector\"`` 仍然默认为 vLLM 的内置连接器，但您可以通过添加 ``kv_connector_module_path`` 选择使用 LMCache 提供的实现 (:mod:`lmcache.integration.vllm.lmcache_mp_connector`):" #: ../../source/getting_started/quickstart.rst:81 msgid "" "The LMCache-shipped connector tracks the latest LMCache server protocol " "and ships fixes/features ahead of the version vendored into vLLM, so " "prefer it whenever you are on vLLM 0.20.0 or newer." -msgstr "" +msgstr "LMCache 提供的连接器跟踪最新的 LMCache 服务器协议，并在集成到 vLLM 的版本之前发布修复和功能，因此在使用 vLLM 0.20.0 或更高版本时，建议优先使用它。" #: ../../source/getting_started/quickstart.rst:85 #: ../../source/getting_started/quickstart.rst:171 @@ -114,154 +114,154 @@ msgstr "" msgid "" "**Test** -- open a new terminal and send two requests whose prompts share" " a prefix:" -msgstr "" +msgstr "**测试** -- 打开一个新终端并发送两个共享前缀的请求：" #: ../../source/getting_started/quickstart.rst:88 #: ../../source/getting_started/quickstart.rst:174 #: ../../source/getting_started/quickstart.rst:259 msgid "**First request**" -msgstr "" +msgstr "**第一次请求**" #: ../../source/getting_started/quickstart.rst:101 #: ../../source/getting_started/quickstart.rst:187 #: ../../source/getting_started/quickstart.rst:272 msgid "**Second request**" -msgstr "" +msgstr "**第二个请求**" #: ../../source/getting_started/quickstart.rst:114 msgid "" "**You should see LMCache logs like this** -- in MP mode the " "store/retrieve logs come from the standalone ``lmcache server`` process, " "one entry per chunk." -msgstr "" +msgstr "**您应该看到类似于这样的 LMCache 日志** -- 在 MP 模式下，存储/检索日志来自独立的 ``lmcache server`` 进程，每个条目对应一个块。" #: ../../source/getting_started/quickstart.rst:118 msgid "**First request** -- cache is empty, so every aligned chunk is offloaded:" -msgstr "" +msgstr "**第一次请求** -- 缓存为空，因此每个对齐的块都被卸载：" #: ../../source/getting_started/quickstart.rst:128 msgid "" "**Second request** -- the shared prefix is retrieved from CPU RAM; only " "the new tail is stored:" -msgstr "" +msgstr "**第二个请求** -- 共享前缀从 CPU 内存中检索；只有新的尾部被存储：" #: ../../source/getting_started/quickstart.rst:138 msgid "" "For request-level statistics (hit ratio, bytes transferred) see " ":doc:`../mp/observability`." -msgstr "" +msgstr "有关请求级统计信息（命中率、传输字节数），请参见 :doc:`../mp/observability`。" #: ../../source/getting_started/quickstart.rst msgid "In-process mode" -msgstr "" +msgstr "进程内模式" #: ../../source/getting_started/quickstart.rst:144 msgid "Start vLLM with LMCache embedded in the engine process:" -msgstr "" +msgstr "在引擎进程中启动嵌入了 LMCache 的 vLLM：" #: ../../source/getting_started/quickstart.rst:155 msgid "" "To customize further, create a config file. See " ":doc:`../api_reference/configurations` for all options." -msgstr "" +msgstr "要进一步自定义，请创建一个配置文件。有关所有选项，请参见 :doc:`../api_reference/configurations`。" #: ../../source/getting_started/quickstart.rst:158 msgid "**Alternative simpler command:**" -msgstr "" +msgstr "**替代的简单命令：**" #: ../../source/getting_started/quickstart.rst:167 msgid "" "The ``--disable-hybrid-kv-cache-manager`` flag is mandatory. All " "configuration options from the :doc:`../api_reference/configurations` " "page still apply." -msgstr "" +msgstr "``--disable-hybrid-kv-cache-manager`` 标志是必需的。来自 :doc:`../api_reference/configurations` 页面的所有配置选项仍然适用。" #: ../../source/getting_started/quickstart.rst:200 msgid "" "**You should see LMCache logs like this** -- in-process mode emits the " "logs inline with the vLLM engine core." -msgstr "" +msgstr "**您应该看到类似于 LMCache 的日志** -- 进程模式会将日志与 vLLM 引擎核心一起内联输出。" #: ../../source/getting_started/quickstart.rst:203 msgid "**First request** -- prompt is offloaded to LMCache:" -msgstr "" +msgstr "**第一次请求** -- 提示被卸载到 LMCache：" #: ../../source/getting_started/quickstart.rst:209 msgid "**Second request** -- hits the cache and stores the new tail:" -msgstr "" +msgstr "**第二个请求** -- 命中缓存并存储新的尾部：" #: ../../source/getting_started/quickstart.rst:218 msgid "**Total tokens 32**: The new prompt has 32 tokens after tokenization." -msgstr "" +msgstr "**总令牌 32**：新提示在分词后有 32 个令牌。" #: ../../source/getting_started/quickstart.rst:219 msgid "" "**LMCache hit tokens: 24**: 24 tokens (full 8-token chunks) were found in" " the cache from the first request that stored 31 tokens." -msgstr "" +msgstr "**LMCache 命中令牌: 24**: 从第一个请求中找到 24 个令牌（完整的 8 令牌块），该请求存储了 31 个令牌。" #: ../../source/getting_started/quickstart.rst:220 msgid "" "**Need to load: 8**: vLLM auto prefix caching uses block size 16; 16 " "tokens already sit in GPU RAM, so LMCache only loads 24-16=8." -msgstr "" +msgstr "**需要加载: 8**: vLLM 自动前缀缓存使用块大小 16；16 个令牌已经存储在显存中，因此 LMCache 只加载 24-16=8。" #: ../../source/getting_started/quickstart.rst:221 msgid "" "**Why 24 hit tokens instead of 31?** LMCache hashes every 8 tokens (8, " "16, 24, 31). It matches page-aligned chunks, so it uses the 24-token " "hash." -msgstr "" +msgstr "**为什么是 24 个命中令牌而不是 31 个？** LMCache 每 8 个令牌（8、16、24、31）进行哈希。它匹配页面对齐的块，因此使用 24 令牌哈希。" #: ../../source/getting_started/quickstart.rst:222 msgid "" "**Stored another 8 tokens**: The new 8 tokens form a full chunk and are " "stored for future reuse." -msgstr "" +msgstr "**存储另外 8 个令牌**：这 8 个新令牌形成一个完整的块，并被存储以供将来重用。" #: ../../source/getting_started/quickstart.rst msgid "SGLang" -msgstr "" +msgstr "SGLang" #: ../../source/getting_started/quickstart.rst:226 msgid "**Install SGLang**" -msgstr "" +msgstr "**安装 SGLang**" #: ../../source/getting_started/quickstart.rst:234 msgid "**Start SGLang with LMCache**" -msgstr "" +msgstr "**使用 LMCache 启动 SGLang**" #: ../../source/getting_started/quickstart.rst:254 msgid "" "Configure LMCache via the config file. See " ":doc:`../api_reference/configurations` for the full list." -msgstr "" +msgstr "通过配置文件配置 LMCache。有关完整列表，请参见 :doc:`../api_reference/configurations`。" #: ../../source/getting_started/quickstart.rst:285 msgid "**You should see LMCache logs like this:**" -msgstr "" +msgstr "**您应该看到类似于以下的 LMCache 日志：**" #: ../../source/getting_started/quickstart.rst:287 msgid "**First request** -- prompt plus generated tokens are stored:" -msgstr "" +msgstr "**第一次请求** -- 提示词和生成的令牌被存储：" #: ../../source/getting_started/quickstart.rst:296 msgid "" "**Second request** -- Radix Cache and LMCache share the prefix; only the " "new portion is stored:" -msgstr "" +msgstr "**第二个请求** -- Radix Cache 和 LMCache 共享前缀；仅存储新部分：" #: ../../source/getting_started/quickstart.rst:306 msgid "" "**Total tokens 140**: SGLang stores KV cache for both prefill and decode " "tokens together, so total = 40 prompt + 100 generated = 140 tokens." -msgstr "" +msgstr "**总令牌 140**：SGLang 将 KV Cache 存储用于 Prefill 和解码令牌，因此总计 = 40 提示 + 100 生成 = 140 令牌。" #: ../../source/getting_started/quickstart.rst:307 msgid "" "**Cached tokens: 30**: SGLang's Radix Attention Cache reused 30 tokens " "from the first request." -msgstr "" +msgstr "**缓存的令牌: 30**: SGLang 的 Radix Attention Cache 重用了来自第一次请求的 30 个令牌。" #: ../../source/getting_started/quickstart.rst:308 msgid "" @@ -269,24 +269,24 @@ msgid "" "chunks) stored from the first request. Since Radix Cache already provides" " 30 tokens in GPU memory, these 24 tokens don't need to be loaded from " "LMCache or stored again." -msgstr "" +msgstr "**LMCache 命中令牌: 24**: LMCache 检测到从第一次请求中存储的 24 个令牌（3 个完整的 8 令牌块）。由于 Radix Cache 已经在显存中提供了 30 个令牌，因此这 24 个令牌不需要从 LMCache 加载或再次存储。" #: ../../source/getting_started/quickstart.rst:309 msgid "" "**New tokens: 10**: Only 10 prompt tokens need prefill computation (40 " "prompt - 30 cached = 10)." -msgstr "" +msgstr "**新令牌：10**：只有 10 个提示令牌需要进行 Prefill 计算（40 个提示 - 30 个缓存 = 10）。" #: ../../source/getting_started/quickstart.rst:310 msgid "" "**Stored 112 out of 140**: 24 tokens (3 full chunks) are already in " "LMCache and skipped. Of the remaining 116 tokens, 112 (14 full 8-token " "chunks) are stored." -msgstr "" +msgstr "**存储 112 个中的 140 个**：24 个令牌（3 个完整块）已经在 LMCache 中并被跳过。在剩余的 116 个令牌中，112 个（14 个完整的 8 令牌块）被存储。" #: ../../source/getting_started/quickstart.rst msgid "TensorRT-LLM" -msgstr "" +msgstr "TensorRT-LLM" #: ../../source/getting_started/quickstart.rst:315 msgid "" @@ -294,43 +294,43 @@ msgid "" "/TensorRT-LLM PR #12626 `_ and the matching LMCache adapter, neither of which has " "shipped in a stable release yet. Until they do, install both from source:" -msgstr "" +msgstr "此集成依赖于 `NVIDIA/TensorRT-LLM PR #12626 `_ 中的连接器预设注册表和匹配的 LMCache 适配器，这两者尚未在稳定版本中发布。在它们发布之前，请从源代码安装这两者：" #: ../../source/getting_started/quickstart.rst:332 msgid "Once both ship in a stable release, the install command will be:" -msgstr "" +msgstr "一旦两者在稳定版本中发布，安装命令将是：" #: ../../source/getting_started/quickstart.rst:339 msgid "" "LMCache integrates with TensorRT-LLM via TRT-LLM's **KV Cache Connector**" " API and supports two deployment modes:" -msgstr "" +msgstr "LMCache 通过 TRT-LLM 的 **KV Cache Connector** API 与 TensorRT-LLM 集成，并支持两种部署模式：" #: ../../source/getting_started/quickstart.rst:342 msgid "" "**In-process mode** (``connector: lmcache``) -- LMCache runs as a " "singleton inside the TRT-LLM process. Simplest setup; no extra service to" " manage." -msgstr "" +msgstr "**进程内模式** (``connector: lmcache``) -- LMCache 作为单例在 TRT-LLM 进程内运行。最简单的设置；无需管理额外的服务。" #: ../../source/getting_started/quickstart.rst:345 msgid "" "**MP mode** (``connector: lmcache-mp``) -- LMCache runs as a standalone " "server. Multiple TRT-LLM workers on the same node can share the cache, " "and the cache survives a TRT-LLM crash." -msgstr "" +msgstr "**MP 模式** (``connector: lmcache-mp``) -- LMCache 作为独立服务器运行。多个 TRT-LLM 工作进程可以共享缓存，并且缓存在 TRT-LLM 崩溃后仍然存在。" #: ../../source/getting_started/quickstart.rst:355 msgid "Configure LMCache via env vars:" -msgstr "" +msgstr "通过环境变量配置 LMCache：" #: ../../source/getting_started/quickstart.rst:364 msgid "Build the TRT-LLM ``LLM`` with ``connector: lmcache``:" -msgstr "" +msgstr "构建 TRT-LLM ``LLM`` 与 ``connector: lmcache``：" #: ../../source/getting_started/quickstart.rst msgid "MP mode" -msgstr "" +msgstr "MP 模式" #: ../../source/getting_started/quickstart.rst:386 msgid "" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/troubleshoot.po b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/troubleshoot.po index 253541a58db..be658b9a9d6 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/troubleshoot.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/getting_started/troubleshoot.po @@ -21,9 +21,9 @@ msgstr "" #: ../../source/getting_started/troubleshoot.rst:2 msgid "TroubleShoot" -msgstr "" +msgstr "故障排除" #: ../../source/getting_started/troubleshoot.rst:4 msgid "Coming soon..." -msgstr "" +msgstr "敬请期待..." diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/common_apis.po b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/common_apis.po index 530a34daf4c..a2dcc6b8649 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/common_apis.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/common_apis.po @@ -21,25 +21,25 @@ msgstr "" #: ../../source/internal_api_server/common_apis.rst:4 msgid "Common APIs" -msgstr "" +msgstr "通用 API" #: ../../source/internal_api_server/common_apis.rst:6 msgid "" "Common APIs are available across all components (scheduler, worker, " "controller)." -msgstr "" +msgstr "所有组件（调度器、工作者、控制器）都提供公共 API。" #: ../../source/internal_api_server/common_apis.rst:11 msgid "Endpoints" -msgstr "" +msgstr "端点" #: ../../source/internal_api_server/common_apis.rst:14 msgid "``GET /env`` — Environment Variables" -msgstr "" +msgstr "``GET /env`` — 环境变量" #: ../../source/internal_api_server/common_apis.rst:16 msgid "Get all environment variables of the running process." -msgstr "" +msgstr "获取运行进程的所有环境变量。" #: ../../source/internal_api_server/common_apis.rst:18 #: ../../source/internal_api_server/common_apis.rst:47 @@ -49,65 +49,65 @@ msgstr "" #: ../../source/internal_api_server/common_apis.rst:228 #: ../../source/internal_api_server/common_apis.rst:259 msgid "**Method**: ``GET``" -msgstr "" +msgstr "**方法**: ``GET``" #: ../../source/internal_api_server/common_apis.rst:19 msgid "**Path**: ``/env``" -msgstr "" +msgstr "**路径**: ``/env``" #: ../../source/internal_api_server/common_apis.rst:20 #: ../../source/internal_api_server/common_apis.rst:105 #: ../../source/internal_api_server/common_apis.rst:120 #: ../../source/internal_api_server/common_apis.rst:261 msgid "**Parameters**: None" -msgstr "" +msgstr "**参数**: 无" #: ../../source/internal_api_server/common_apis.rst:21 msgid "" "**Response**: ``application/json`` — JSON object of all environment " "variables (sorted by key)." -msgstr "" +msgstr "**响应**: ``application/json`` — 所有环境变量的 JSON 对象（按键排序）。" #: ../../source/internal_api_server/common_apis.rst:27 #: ../../source/internal_api_server/common_apis.rst:188 #: ../../source/internal_api_server/common_apis.rst:334 msgid "**Example Response**:" -msgstr "" +msgstr "**示例响应**：" #: ../../source/internal_api_server/common_apis.rst:39 msgid "``GET /loglevel`` — Log Level Management" -msgstr "" +msgstr "``GET /loglevel`` — 日志级别管理" #: ../../source/internal_api_server/common_apis.rst:41 msgid "" "Get or set the log level for Python loggers. Behavior depends on query " "parameters:" -msgstr "" +msgstr "获取或设置 Python 日志记录器的日志级别。行为取决于查询参数：" #: ../../source/internal_api_server/common_apis.rst:43 msgid "**No parameters**: List all loggers and their levels." -msgstr "" +msgstr "**无参数**：列出所有记录器及其级别。" #: ../../source/internal_api_server/common_apis.rst:44 msgid "**``logger_name`` only**: Get the level of the specified logger." -msgstr "" +msgstr "**``logger_name`` 仅**: 获取指定记录器的级别。" #: ../../source/internal_api_server/common_apis.rst:45 msgid "" "**``logger_name`` and ``level``**: Set the level of the specified logger " "(including all its handlers)." -msgstr "" +msgstr "**``logger_name`` 和 ``level``**: 设置指定记录器（包括其所有处理程序）的级别。" #: ../../source/internal_api_server/common_apis.rst:48 msgid "**Path**: ``/loglevel``" -msgstr "" +msgstr "**路径**: ``/loglevel``" #: ../../source/internal_api_server/common_apis.rst:49 #: ../../source/internal_api_server/common_apis.rst:135 #: ../../source/internal_api_server/common_apis.rst:165 #: ../../source/internal_api_server/common_apis.rst:307 msgid "**Parameters**:" -msgstr "" +msgstr "**参数**:" #: ../../source/internal_api_server/common_apis.rst:52 #: ../../source/internal_api_server/common_apis.rst:138 @@ -115,7 +115,7 @@ msgstr "" #: ../../source/internal_api_server/common_apis.rst:233 #: ../../source/internal_api_server/common_apis.rst:310 msgid "Name" -msgstr "" +msgstr "名称" #: ../../source/internal_api_server/common_apis.rst:52 #: ../../source/internal_api_server/common_apis.rst:138 @@ -123,7 +123,7 @@ msgstr "" #: ../../source/internal_api_server/common_apis.rst:233 #: ../../source/internal_api_server/common_apis.rst:310 msgid "Type" -msgstr "" +msgstr "类型" #: ../../source/internal_api_server/common_apis.rst:52 #: ../../source/internal_api_server/common_apis.rst:138 @@ -131,11 +131,11 @@ msgstr "" #: ../../source/internal_api_server/common_apis.rst:233 #: ../../source/internal_api_server/common_apis.rst:310 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/internal_api_server/common_apis.rst:54 msgid "``logger_name``" -msgstr "" +msgstr "``logger_name``" #: ../../source/internal_api_server/common_apis.rst:54 #: ../../source/internal_api_server/common_apis.rst:55 @@ -143,217 +143,217 @@ msgstr "" #: ../../source/internal_api_server/common_apis.rst:170 #: ../../source/internal_api_server/common_apis.rst:235 msgid "str" -msgstr "" +msgstr "字符串" #: ../../source/internal_api_server/common_apis.rst:54 msgid "(Optional) Logger name to query or set" -msgstr "" +msgstr "(可选) 查询或设置的日志记录器名称" #: ../../source/internal_api_server/common_apis.rst:55 #: ../../source/internal_api_server/common_apis.rst:170 msgid "``level``" -msgstr "" +msgstr "``level``" #: ../../source/internal_api_server/common_apis.rst:55 msgid "" "(Optional) Log level to set (e.g. ``DEBUG``, ``INFO``, ``WARNING``, " "``ERROR``)" -msgstr "" +msgstr "(可选) 设置的日志级别 (例如 ``DEBUG``, ``INFO``, ``WARNING``, ``ERROR``)" #: ../../source/internal_api_server/common_apis.rst:58 msgid "**Response**: ``text/plain``" -msgstr "" +msgstr "**响应**: ``text/plain``" #: ../../source/internal_api_server/common_apis.rst:71 msgid "**Example Response** (list all):" -msgstr "" +msgstr "**示例响应**（列出所有）：" #: ../../source/internal_api_server/common_apis.rst:79 msgid "**Example Response** (get):" -msgstr "" +msgstr "**示例响应**（获取）：" #: ../../source/internal_api_server/common_apis.rst:85 msgid "**Example Response** (set):" -msgstr "" +msgstr "**示例响应** (设置):" #: ../../source/internal_api_server/common_apis.rst:91 #: ../../source/internal_api_server/common_apis.rst:214 msgid "**Error Response** (invalid level, HTTP 400):" -msgstr "" +msgstr "**错误响应**（无效级别，HTTP 400）：" #: ../../source/internal_api_server/common_apis.rst:99 msgid "``GET /metrics`` — Prometheus Metrics" -msgstr "" +msgstr "``GET /metrics`` — Prometheus 指标" #: ../../source/internal_api_server/common_apis.rst:101 msgid "Get Prometheus metrics data in the standard exposition format." -msgstr "" +msgstr "以标准展示格式获取 Prometheus 指标数据。" #: ../../source/internal_api_server/common_apis.rst:104 msgid "**Path**: ``/metrics``" -msgstr "" +msgstr "**路径**: ``/metrics``" #: ../../source/internal_api_server/common_apis.rst:106 msgid "**Response**: ``text/plain`` — Prometheus text-based exposition format." -msgstr "" +msgstr "**响应**: ``text/plain`` — Prometheus 文本格式的展示。" #: ../../source/internal_api_server/common_apis.rst:114 msgid "``POST /metrics/reset`` — Reset Prometheus Metrics" -msgstr "" +msgstr "``POST /metrics/reset`` — 重置 Prometheus 指标" #: ../../source/internal_api_server/common_apis.rst:116 msgid "Reset all Prometheus metrics to their initial state." -msgstr "" +msgstr "重置所有 Prometheus 指标为其初始状态。" #: ../../source/internal_api_server/common_apis.rst:118 #: ../../source/internal_api_server/common_apis.rst:304 msgid "**Method**: ``POST``" -msgstr "" +msgstr "**方法**: ``POST``" #: ../../source/internal_api_server/common_apis.rst:119 msgid "**Path**: ``/metrics/reset``" -msgstr "" +msgstr "**路径**: ``/metrics/reset``" #: ../../source/internal_api_server/common_apis.rst:121 msgid "**Response**: ``text/plain`` — ``\"ok\"`` on success." -msgstr "" +msgstr "**响应**: ``text/plain`` — ``\"ok\"`` 表示成功。" #: ../../source/internal_api_server/common_apis.rst:129 msgid "``GET /threads`` — Thread Information" -msgstr "" +msgstr "``GET /threads`` — 线程信息" #: ../../source/internal_api_server/common_apis.rst:131 msgid "Get information about active threads with optional filtering." -msgstr "" +msgstr "获取有关活动线程的信息，并可选择性地进行过滤。" #: ../../source/internal_api_server/common_apis.rst:134 msgid "**Path**: ``/threads``" -msgstr "" +msgstr "**路径**: ``/threads``" #: ../../source/internal_api_server/common_apis.rst:140 msgid "``name``" -msgstr "" +msgstr "``name``" #: ../../source/internal_api_server/common_apis.rst:140 msgid "(Optional) Filter by thread name (fuzzy match, case-insensitive)" -msgstr "" +msgstr "(可选) 按线程名称过滤（模糊匹配，大小写不敏感）" #: ../../source/internal_api_server/common_apis.rst:141 msgid "``thread_id``" -msgstr "" +msgstr "``thread_id``" #: ../../source/internal_api_server/common_apis.rst:141 msgid "int" -msgstr "" +msgstr "整数" #: ../../source/internal_api_server/common_apis.rst:141 msgid "(Optional) Filter by thread ID" -msgstr "" +msgstr "（可选）按线程 ID 过滤" #: ../../source/internal_api_server/common_apis.rst:144 msgid "**Response**: ``text/plain`` — Thread info with stack traces and summary." -msgstr "" +msgstr "**响应**: ``text/plain`` — 包含堆栈跟踪和摘要的线程信息。" #: ../../source/internal_api_server/common_apis.rst:159 msgid "``GET /periodic-threads`` — Periodic Thread Status" -msgstr "" +msgstr "``GET /periodic-threads`` — 周期性线程状态" #: ../../source/internal_api_server/common_apis.rst:161 msgid "Get information about registered periodic threads." -msgstr "" +msgstr "获取有关注册的周期性线程的信息。" #: ../../source/internal_api_server/common_apis.rst:164 msgid "**Path**: ``/periodic-threads``" -msgstr "" +msgstr "**路径**: ``/periodic-threads``" #: ../../source/internal_api_server/common_apis.rst:170 msgid "" "(Optional) Filter by thread level: ``critical``, ``high``, ``medium``, " "``low``" -msgstr "" +msgstr "(可选) 按线程级别过滤：``critical``、``high``、``medium``、``low``" #: ../../source/internal_api_server/common_apis.rst:171 msgid "``running_only``" -msgstr "" +msgstr "``running_only``" #: ../../source/internal_api_server/common_apis.rst:171 #: ../../source/internal_api_server/common_apis.rst:172 msgid "bool" -msgstr "" +msgstr "布尔值" #: ../../source/internal_api_server/common_apis.rst:171 msgid "(Optional) Only show running threads (default: ``false``)" -msgstr "" +msgstr "(可选) 仅显示正在运行的线程（默认值：``false``）" #: ../../source/internal_api_server/common_apis.rst:172 msgid "``active_only``" -msgstr "" +msgstr "``active_only``" #: ../../source/internal_api_server/common_apis.rst:172 msgid "(Optional) Only show active threads (default: ``false``)" -msgstr "" +msgstr "(可选) 仅显示活动线程（默认值：``false``）" #: ../../source/internal_api_server/common_apis.rst:175 #: ../../source/internal_api_server/common_apis.rst:238 #: ../../source/internal_api_server/common_apis.rst:262 msgid "**Response**: ``application/json``" -msgstr "" +msgstr "**响应**: ``application/json``" #: ../../source/internal_api_server/common_apis.rst:224 #, python-brace-format msgid "``GET /periodic-threads/{thread_name}`` — Single Periodic Thread" -msgstr "" +msgstr "``GET /periodic-threads/{thread_name}`` — 单个周期线程" #: ../../source/internal_api_server/common_apis.rst:226 msgid "Get detailed information about a specific periodic thread by name." -msgstr "" +msgstr "通过名称获取特定周期线程的详细信息。" #: ../../source/internal_api_server/common_apis.rst:229 #, python-brace-format msgid "**Path**: ``/periodic-threads/{thread_name}``" -msgstr "" +msgstr "**路径**: ``/periodic-threads/{thread_name}``" #: ../../source/internal_api_server/common_apis.rst:230 msgid "**Path Parameters**:" -msgstr "" +msgstr "**路径参数**：" #: ../../source/internal_api_server/common_apis.rst:235 msgid "``thread_name``" -msgstr "" +msgstr "``thread_name``" #: ../../source/internal_api_server/common_apis.rst:235 msgid "Name of the periodic thread" -msgstr "" +msgstr "周期性线程的名称" #: ../../source/internal_api_server/common_apis.rst:244 msgid "**Error Response** (not found, HTTP 404):" -msgstr "" +msgstr "**错误响应**（未找到，HTTP 404）：" #: ../../source/internal_api_server/common_apis.rst:254 msgid "``GET /periodic-threads-health`` — Periodic Thread Health Check" -msgstr "" +msgstr "``GET /periodic-threads-health`` — 周期性线程健康检查" #: ../../source/internal_api_server/common_apis.rst:256 msgid "" "Quick health check for periodic threads. Returns healthy status if all " "``critical`` and ``high`` level threads are active." -msgstr "" +msgstr "对周期性线程的快速健康检查。如果所有 ``critical`` 和 ``high`` 级别的线程都处于活动状态，则返回健康状态。" #: ../../source/internal_api_server/common_apis.rst:260 msgid "**Path**: ``/periodic-threads-health``" -msgstr "" +msgstr "**路径**: ``/periodic-threads-health``" #: ../../source/internal_api_server/common_apis.rst:268 msgid "**Example Response** (healthy):" -msgstr "" +msgstr "**示例响应**（健康）：" #: ../../source/internal_api_server/common_apis.rst:278 msgid "**Example Response** (unhealthy):" -msgstr "" +msgstr "**示例响应**（不健康）：" #: ../../source/internal_api_server/common_apis.rst:297 msgid "``POST /run_script`` — Run Script" -msgstr "" +msgstr "``POST /run_script`` — 运行脚本" #: ../../source/internal_api_server/common_apis.rst:299 msgid "" @@ -361,43 +361,43 @@ msgid "" "The script has access to ``app`` (the FastAPI application instance) and a" " limited set of builtins. Import is restricted to modules configured in " "``script_allowed_imports``." -msgstr "" +msgstr "在受限的沙箱环境中上传并执行 Python 脚本。该脚本可以访问 ``app``（FastAPI 应用实例）和有限的内置函数。导入仅限于在 ``script_allowed_imports`` 中配置的模块。" #: ../../source/internal_api_server/common_apis.rst:305 msgid "**Path**: ``/run_script``" -msgstr "" +msgstr "**路径**: ``/run_script``" #: ../../source/internal_api_server/common_apis.rst:306 msgid "**Content-Type**: ``multipart/form-data``" -msgstr "" +msgstr "**内容类型**: ``multipart/form-data``" #: ../../source/internal_api_server/common_apis.rst:312 msgid "``script``" -msgstr "" +msgstr "``script``" #: ../../source/internal_api_server/common_apis.rst:312 msgid "file" -msgstr "" +msgstr "文件" #: ../../source/internal_api_server/common_apis.rst:312 msgid "Python script file to execute" -msgstr "" +msgstr "要执行的 Python 脚本文件" #: ../../source/internal_api_server/common_apis.rst:315 msgid "" "**Response**: ``text/plain`` — The ``result`` variable from the script, " "or ``\"Script executed successfully\"`` if no ``result`` is set." -msgstr "" +msgstr "**响应**: ``text/plain`` — 脚本中的 ``result`` 变量，或者如果没有设置 ``result`` 则为 ``\"脚本执行成功\"``。" #: ../../source/internal_api_server/common_apis.rst:323 msgid "**Example Script** (``scratch.py``):" -msgstr "" +msgstr "**示例脚本** (``scratch.py``):" #: ../../source/internal_api_server/common_apis.rst:340 msgid "**Error Response** (no script, HTTP 400):" -msgstr "" +msgstr "**错误响应**（无脚本，HTTP 400）：" #: ../../source/internal_api_server/common_apis.rst:346 msgid "**Error Response** (execution error, HTTP 500):" -msgstr "" +msgstr "**错误响应**（执行错误，HTTP 500）：" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/controller_apis.po b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/controller_apis.po index 21803212726..9b0a87dd53c 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/controller_apis.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/controller_apis.po @@ -21,80 +21,80 @@ msgstr "" #: ../../source/internal_api_server/controller_apis.rst:4 msgid "Controller APIs" -msgstr "" +msgstr "控制器 API" #: ../../source/internal_api_server/controller_apis.rst:6 msgid "" "These APIs are specific to the LMCache Controller component. They provide" " visibility into registered instances, workers, and key statistics." -msgstr "" +msgstr "这些 API 是特定于 LMCache Controller 组件的。它们提供对注册实例、工作者和关键统计信息的可见性。" #: ../../source/internal_api_server/controller_apis.rst:12 msgid "Endpoints" -msgstr "" +msgstr "端点" #: ../../source/internal_api_server/controller_apis.rst:15 msgid "``GET /controller/key-stats`` — Key Statistics" -msgstr "" +msgstr "``GET /controller/key-stats`` — 关键统计信息" #: ../../source/internal_api_server/controller_apis.rst:17 msgid "Get key statistics across all instances and workers." -msgstr "" +msgstr "获取所有实例和工作节点的关键统计信息。" #: ../../source/internal_api_server/controller_apis.rst:19 #: ../../source/internal_api_server/controller_apis.rst:90 msgid "**Method**: ``GET``" -msgstr "" +msgstr "**方法**: ``GET``" #: ../../source/internal_api_server/controller_apis.rst:20 msgid "**Path**: ``/controller/key-stats``" -msgstr "" +msgstr "**路径**: ``/controller/key-stats``" #: ../../source/internal_api_server/controller_apis.rst:21 msgid "**Parameters**: None" -msgstr "" +msgstr "**参数**: 无" #: ../../source/internal_api_server/controller_apis.rst:22 #: ../../source/internal_api_server/controller_apis.rst:101 msgid "**Response**: ``application/json``" -msgstr "" +msgstr "**响应**: ``application/json``" #: ../../source/internal_api_server/controller_apis.rst:28 msgid "**Example Response** (HTTP 200):" -msgstr "" +msgstr "**示例响应** (HTTP 200):" #: ../../source/internal_api_server/controller_apis.rst:50 #: ../../source/internal_api_server/controller_apis.rst:175 msgid "**Error Response** (controller not available, HTTP 503):" -msgstr "" +msgstr "**错误响应**（控制器不可用，HTTP 503）：" #: ../../source/internal_api_server/controller_apis.rst:58 msgid "**Response Schema**:" -msgstr "" +msgstr "**响应模式**:" #: ../../source/internal_api_server/controller_apis.rst:61 #: ../../source/internal_api_server/controller_apis.rst:72 #: ../../source/internal_api_server/controller_apis.rst:186 msgid "Field" -msgstr "" +msgstr "字段" #: ../../source/internal_api_server/controller_apis.rst:61 #: ../../source/internal_api_server/controller_apis.rst:72 #: ../../source/internal_api_server/controller_apis.rst:95 #: ../../source/internal_api_server/controller_apis.rst:186 msgid "Type" -msgstr "" +msgstr "类型" #: ../../source/internal_api_server/controller_apis.rst:61 #: ../../source/internal_api_server/controller_apis.rst:72 #: ../../source/internal_api_server/controller_apis.rst:95 #: ../../source/internal_api_server/controller_apis.rst:186 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/internal_api_server/controller_apis.rst:63 msgid "``total_key_count``" -msgstr "" +msgstr "``total_key_count``" #: ../../source/internal_api_server/controller_apis.rst:63 #: ../../source/internal_api_server/controller_apis.rst:64 @@ -106,49 +106,49 @@ msgstr "" #: ../../source/internal_api_server/controller_apis.rst:191 #: ../../source/internal_api_server/controller_apis.rst:195 msgid "int" -msgstr "" +msgstr "整数" #: ../../source/internal_api_server/controller_apis.rst:63 msgid "Total number of KV keys across all instances" -msgstr "" +msgstr "所有实例中 KV 键的总数" #: ../../source/internal_api_server/controller_apis.rst:64 msgid "``total_instance_count``" -msgstr "" +msgstr "``total_instance_count``" #: ../../source/internal_api_server/controller_apis.rst:64 msgid "Total number of registered instances" -msgstr "" +msgstr "注册实例的总数" #: ../../source/internal_api_server/controller_apis.rst:65 msgid "``total_worker_count``" -msgstr "" +msgstr "``total_worker_count``" #: ../../source/internal_api_server/controller_apis.rst:65 msgid "Total number of workers across all instances" -msgstr "" +msgstr "所有实例的工作线程总数" #: ../../source/internal_api_server/controller_apis.rst:66 msgid "``instances``" -msgstr "" +msgstr "``instances``" #: ../../source/internal_api_server/controller_apis.rst:66 msgid "list" -msgstr "" +msgstr "列表" #: ../../source/internal_api_server/controller_apis.rst:66 msgid "Per-instance breakdown (see below)" -msgstr "" +msgstr "每个实例的详细信息（见下文）" #: ../../source/internal_api_server/controller_apis.rst:69 msgid "**Instance Schema**:" -msgstr "" +msgstr "**实例架构**:" #: ../../source/internal_api_server/controller_apis.rst:74 #: ../../source/internal_api_server/controller_apis.rst:97 #: ../../source/internal_api_server/controller_apis.rst:188 msgid "``instance_id``" -msgstr "" +msgstr "``instance_id``" #: ../../source/internal_api_server/controller_apis.rst:74 #: ../../source/internal_api_server/controller_apis.rst:97 @@ -156,154 +156,154 @@ msgstr "" #: ../../source/internal_api_server/controller_apis.rst:190 #: ../../source/internal_api_server/controller_apis.rst:192 msgid "str" -msgstr "" +msgstr "字符串" #: ../../source/internal_api_server/controller_apis.rst:74 msgid "Unique identifier of the instance" -msgstr "" +msgstr "实例的唯一标识符" #: ../../source/internal_api_server/controller_apis.rst:75 #: ../../source/internal_api_server/controller_apis.rst:195 msgid "``key_count``" -msgstr "" +msgstr "``key_count``" #: ../../source/internal_api_server/controller_apis.rst:75 msgid "Number of KV keys held by this instance" -msgstr "" +msgstr "此实例持有的 KV 键数量" #: ../../source/internal_api_server/controller_apis.rst:76 msgid "``worker_count``" -msgstr "" +msgstr "``worker_count``" #: ../../source/internal_api_server/controller_apis.rst:76 msgid "Number of workers in this instance" -msgstr "" +msgstr "此实例中的工作线程数量" #: ../../source/internal_api_server/controller_apis.rst:81 msgid "``GET /controller/workers`` — Worker Information" -msgstr "" +msgstr "``GET /controller/workers`` — 工作信息" #: ../../source/internal_api_server/controller_apis.rst:83 msgid "" "Get worker information with flexible query parameters. Behavior depends " "on the combination of parameters:" -msgstr "" +msgstr "使用灵活的查询参数获取工作信息。行为取决于参数的组合：" #: ../../source/internal_api_server/controller_apis.rst:86 msgid "**No parameters**: List all registered workers across all instances." -msgstr "" +msgstr "**无参数**：列出所有实例中注册的所有工作者。" #: ../../source/internal_api_server/controller_apis.rst:87 msgid "**``instance_id`` only**: List all workers for a specific instance." -msgstr "" +msgstr "**``instance_id`` 仅**: 列出特定实例的所有工作者。" #: ../../source/internal_api_server/controller_apis.rst:88 msgid "" "**``instance_id`` and ``worker_id``**: Get detailed info about a specific" " worker." -msgstr "" +msgstr "**``instance_id`` 和 ``worker_id``**: 获取特定工作者的详细信息。" #: ../../source/internal_api_server/controller_apis.rst:91 msgid "**Path**: ``/controller/workers``" -msgstr "" +msgstr "**路径**: ``/controller/workers``" #: ../../source/internal_api_server/controller_apis.rst:92 msgid "**Parameters**:" -msgstr "" +msgstr "**参数**:" #: ../../source/internal_api_server/controller_apis.rst:95 msgid "Name" -msgstr "" +msgstr "名称" #: ../../source/internal_api_server/controller_apis.rst:97 msgid "(Optional) Instance ID to filter workers" -msgstr "" +msgstr "（可选）用于过滤工作节点的实例 ID" #: ../../source/internal_api_server/controller_apis.rst:98 #: ../../source/internal_api_server/controller_apis.rst:189 msgid "``worker_id``" -msgstr "" +msgstr "``worker_id``" #: ../../source/internal_api_server/controller_apis.rst:98 msgid "" "(Optional) Worker ID for specific worker details (requires " "``instance_id``)" -msgstr "" +msgstr "（可选）特定工作者详细信息的工作者 ID（需要 ``instance_id``）" #: ../../source/internal_api_server/controller_apis.rst:114 msgid "**Example Response** (list workers):" -msgstr "" +msgstr "**示例响应**（列出工作者）：" #: ../../source/internal_api_server/controller_apis.rst:144 msgid "**Example Response** (single worker):" -msgstr "" +msgstr "**示例响应**（单个工作者）：" #: ../../source/internal_api_server/controller_apis.rst:159 msgid "**Error Response** (worker not found, HTTP 404):" -msgstr "" +msgstr "**错误响应**（未找到工作者，HTTP 404）：" #: ../../source/internal_api_server/controller_apis.rst:167 msgid "**Error Response** (instance not found, HTTP 404):" -msgstr "" +msgstr "**错误响应**（实例未找到，HTTP 404）：" #: ../../source/internal_api_server/controller_apis.rst:183 msgid "**Worker Response Schema**:" -msgstr "" +msgstr "**工作者响应模式**:" #: ../../source/internal_api_server/controller_apis.rst:188 msgid "Instance this worker belongs to" -msgstr "" +msgstr "该工作者所属的实例" #: ../../source/internal_api_server/controller_apis.rst:189 msgid "Worker index within the instance" -msgstr "" +msgstr "实例内的工作线程索引" #: ../../source/internal_api_server/controller_apis.rst:190 msgid "``ip``" -msgstr "" +msgstr "``ip``" #: ../../source/internal_api_server/controller_apis.rst:190 msgid "Worker IP address" -msgstr "" +msgstr "工作节点 IP 地址" #: ../../source/internal_api_server/controller_apis.rst:191 msgid "``port``" -msgstr "" +msgstr "``port``" #: ../../source/internal_api_server/controller_apis.rst:191 msgid "Worker port number" -msgstr "" +msgstr "工作端口号" #: ../../source/internal_api_server/controller_apis.rst:192 msgid "``peer_init_url``" -msgstr "" +msgstr "``peer_init_url``" #: ../../source/internal_api_server/controller_apis.rst:192 msgid "(Optional) Peer initialization URL" -msgstr "" +msgstr "(可选) 对等初始化 URL" #: ../../source/internal_api_server/controller_apis.rst:193 msgid "``registration_time``" -msgstr "" +msgstr "``registration_time``" #: ../../source/internal_api_server/controller_apis.rst:193 #: ../../source/internal_api_server/controller_apis.rst:194 msgid "float" -msgstr "" +msgstr "浮点数" #: ../../source/internal_api_server/controller_apis.rst:193 msgid "Unix timestamp of worker registration" -msgstr "" +msgstr "工作节点注册的 Unix 时间戳" #: ../../source/internal_api_server/controller_apis.rst:194 msgid "``last_heartbeat_time``" -msgstr "" +msgstr "``last_heartbeat_time``" #: ../../source/internal_api_server/controller_apis.rst:194 msgid "Unix timestamp of last heartbeat" -msgstr "" +msgstr "最后心跳的 Unix 时间戳" #: ../../source/internal_api_server/controller_apis.rst:195 msgid "Number of KV keys held by this worker" -msgstr "" +msgstr "此工作线程持有的 KV 键数量" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/dynamic_backend_management.po b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/dynamic_backend_management.po index e91faa85192..332ea84ea31 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/dynamic_backend_management.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/dynamic_backend_management.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/internal_api_server/dynamic_backend_management.rst:4 msgid "Dynamic Backend Management" -msgstr "" +msgstr "动态后端管理" #: ../../source/internal_api_server/dynamic_backend_management.rst:6 msgid "" @@ -31,129 +31,129 @@ msgid "" "between different storage configurations on the fly — for example, " "migrating from a ``LocalDiskBackend`` to a ``GdsBackend``, or changing " "the remote connector from a filesystem connector to Redis." -msgstr "" +msgstr "LMCache 提供了一组内部 API 端点，允许您在运行时 **列出**、**关闭** 和 **创建** 存储后端，而无需重启服务引擎。当您需要动态切换不同的存储配置时，这非常有用——例如，从 ``LocalDiskBackend`` 迁移到 ``GdsBackend``，或将远程连接器从文件系统连接器更改为 Redis。" #: ../../source/internal_api_server/dynamic_backend_management.rst:14 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/internal_api_server/dynamic_backend_management.rst:16 msgid "The workflow for dynamically switching a storage backend is:" -msgstr "" +msgstr "动态切换存储后端的工作流程是：" #: ../../source/internal_api_server/dynamic_backend_management.rst:18 msgid "**Close** the backend you want to replace." -msgstr "" +msgstr "**关闭**您想要替换的后端。" #: ../../source/internal_api_server/dynamic_backend_management.rst:19 msgid "**Update** the relevant configuration via the ``POST /conf`` API." -msgstr "" +msgstr "**更新**相关配置通过 ``POST /conf`` API。" #: ../../source/internal_api_server/dynamic_backend_management.rst:20 msgid "" "**Create** new backends — only backends that are not already present will" " be created." -msgstr "" +msgstr "**创建** 新的后端 — 只有尚不存在的后端才会被创建。" #: ../../source/internal_api_server/dynamic_backend_management.rst:23 msgid "" "Any backend that was **not** closed will be skipped during creation, so " "the operation is safe and idempotent." -msgstr "" +msgstr "任何**未**关闭的后端在创建过程中将被跳过，因此该操作是安全且幂等的。" #: ../../source/internal_api_server/dynamic_backend_management.rst:27 msgid "API Endpoints" -msgstr "" +msgstr "API 端点" #: ../../source/internal_api_server/dynamic_backend_management.rst:30 msgid "``GET /backends``" -msgstr "" +msgstr "``GET /backends``" #: ../../source/internal_api_server/dynamic_backend_management.rst:32 msgid "List all active storage backends." -msgstr "" +msgstr "列出所有活动的存储后端。" #: ../../source/internal_api_server/dynamic_backend_management.rst:38 #: ../../source/internal_api_server/dynamic_backend_management.rst:58 #: ../../source/internal_api_server/dynamic_backend_management.rst:80 msgid "Response:" -msgstr "" +msgstr "响应：" #: ../../source/internal_api_server/dynamic_backend_management.rst:48 #, python-brace-format msgid "``DELETE /backends/{backend_name}``" -msgstr "" +msgstr "``DELETE /backends/{backend_name}``" #: ../../source/internal_api_server/dynamic_backend_management.rst:50 msgid "" "Close and remove a specific storage backend. After this call the backend" " is fully shut down and removed from the internal dictionary, ensuring no" " stale references remain." -msgstr "" +msgstr "关闭并移除特定的存储后端。此调用后，后端将完全关闭并从内部字典中移除，确保没有过时的引用残留。" #: ../../source/internal_api_server/dynamic_backend_management.rst:71 msgid "``POST /backends``" -msgstr "" +msgstr "``POST /backends``" #: ../../source/internal_api_server/dynamic_backend_management.rst:73 msgid "" "Create new storage backends based on the current ``LMCacheEngineConfig``." " Existing backends are skipped." -msgstr "" +msgstr "根据当前的 ``LMCacheEngineConfig`` 创建新的存储后端。现有的后端将被跳过。" #: ../../source/internal_api_server/dynamic_backend_management.rst:96 msgid "Use-Case Examples" -msgstr "" +msgstr "用例示例" #: ../../source/internal_api_server/dynamic_backend_management.rst:99 msgid "Switching from ``LocalDiskBackend`` to ``GdsBackend``" -msgstr "" +msgstr "从 ``LocalDiskBackend`` 切换到 ``GdsBackend``" #: ../../source/internal_api_server/dynamic_backend_management.rst:101 msgid "" "If you originally configured a local-disk backend and want to migrate to " "NVIDIA GPUDirect Storage (GDS) at runtime:" -msgstr "" +msgstr "如果您最初配置了本地磁盘后端，并希望在运行时迁移到 NVIDIA GPUDirect Storage (GDS)：" #: ../../source/internal_api_server/dynamic_backend_management.rst:124 msgid "Switching ``RemoteBackend`` connector (FS → Redis)" -msgstr "" +msgstr "切换 ``RemoteBackend`` 连接器 (FS → Redis)" #: ../../source/internal_api_server/dynamic_backend_management.rst:126 msgid "To change the remote connector type without restarting:" -msgstr "" +msgstr "要在不重启的情况下更改远程连接器类型：" #: ../../source/internal_api_server/dynamic_backend_management.rst:147 msgid "Replacing only one backend while keeping others" -msgstr "" +msgstr "仅替换一个后端而保持其他后端不变" #: ../../source/internal_api_server/dynamic_backend_management.rst:149 msgid "" "If only the ``RemoteBackend`` needs to be updated but the " "``LocalCPUBackend`` should stay untouched:" -msgstr "" +msgstr "如果只需要更新 ``RemoteBackend`` 而 ``LocalCPUBackend`` 应保持不变：" #: ../../source/internal_api_server/dynamic_backend_management.rst:166 msgid "Notes" -msgstr "" +msgstr "注意事项" #: ../../source/internal_api_server/dynamic_backend_management.rst:168 msgid "" "Closing the ``LocalCPUBackend`` is possible but should be done with " "caution since many other backends rely on it as an intermediate buffer." -msgstr "" +msgstr "关闭 ``LocalCPUBackend`` 是可能的，但应谨慎进行，因为许多其他后端依赖它作为中间缓冲区。" #: ../../source/internal_api_server/dynamic_backend_management.rst:170 msgid "" "The ``POST /backends`` endpoint calls the same ``CreateStorageBackends`` " "factory that is used during engine initialization, so all backend types " "(local CPU, local disk, GDS, remote, P2P, plugins, etc.) are supported." -msgstr "" +msgstr "``POST /backends`` 端点调用与引擎初始化期间使用的相同 ``CreateStorageBackends`` 工厂，因此支持所有后端类型（本地 CPU、本地磁盘、GDS、远程、P2P、插件等）。" #: ../../source/internal_api_server/dynamic_backend_management.rst:174 msgid "" "After creating backends, the ``StorageManager`` automatically refreshes " "its internal references (``non_allocator_backends``, " "``local_cpu_backend``, etc.)." -msgstr "" +msgstr "创建后端后，``StorageManager`` 会自动刷新其内部引用（``non_allocator_backends``、``local_cpu_backend`` 等）。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/internal_api_server.po b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/internal_api_server.po index 58f651f24b0..00b354ae2d7 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/internal_api_server.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/internal_api_server.po @@ -21,62 +21,62 @@ msgstr "" #: ../../source/internal_api_server/internal_api_server.rst:4 msgid "Internal API Server" -msgstr "" +msgstr "内部 API 服务器" #: ../../source/internal_api_server/internal_api_server.rst:6 msgid "" "The ``internal_api_server`` provides HTTP APIs for managing and " "inspecting the LMCache engine at runtime. APIs are organized into three " "categories:" -msgstr "" +msgstr "``internal_api_server`` 提供用于在运行时管理和检查 LMCache 引擎的 HTTP API。API 被组织为三个类别：" #: ../../source/internal_api_server/internal_api_server.rst:9 msgid "" "**Common APIs** — Available across all components (scheduler, worker, " "controller)." -msgstr "" +msgstr "**通用 API** — 在所有组件（调度器、工作线程、控制器）中可用。" #: ../../source/internal_api_server/internal_api_server.rst:10 msgid "**vLLM / Inference APIs** — Specific to vLLM inference workers." -msgstr "" +msgstr "**vLLM / 推理 API** — 特定于 vLLM 推理工作节点。" #: ../../source/internal_api_server/internal_api_server.rst:11 msgid "**Controller APIs** — Specific to the LMCache Controller." -msgstr "" +msgstr "**控制器 API** — 特定于 LMCache 控制器。" #: ../../source/internal_api_server/internal_api_server.rst:22 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/internal_api_server/internal_api_server.rst:24 msgid "The following parameters can be configured in the YAML file:" -msgstr "" +msgstr "以下参数可以在 YAML 文件中配置：" #: ../../source/internal_api_server/internal_api_server.rst:46 msgid "Port Assignment" -msgstr "" +msgstr "端口分配" #: ../../source/internal_api_server/internal_api_server.rst:48 msgid "The port for each component is computed as:" -msgstr "" +msgstr "每个组件的端口计算公式为：" #: ../../source/internal_api_server/internal_api_server.rst:54 msgid "Where ``port_offset`` is:" -msgstr "" +msgstr "``port_offset`` 是：" #: ../../source/internal_api_server/internal_api_server.rst:56 msgid "``0`` for the Scheduler" -msgstr "" +msgstr "``0`` 用于调度器" #: ../../source/internal_api_server/internal_api_server.rst:57 msgid "" "``1 + worker_id`` for Workers (e.g., Worker 0 → offset 1, Worker 1 → " "offset 2)" -msgstr "" +msgstr "``1 + worker_id`` 用于工作进程（例如，工作进程 0 → 偏移 1，工作进程 1 → 偏移 2）" #: ../../source/internal_api_server/internal_api_server.rst:61 msgid "API Category & Route Discovery" -msgstr "" +msgstr "API 类别与路由发现" #: ../../source/internal_api_server/internal_api_server.rst:63 #, python-brace-format @@ -85,33 +85,33 @@ msgid "" "API endpoint modules. Any file named ``*_api.py`` under " "``lmcache/v1/internal_api_server/{common,vllm,controller}/`` that exports" " a ``router = APIRouter()`` will be automatically included." -msgstr "" +msgstr "服务器使用 ``APIRegistry`` 自动发现和注册 API 端点模块。任何位于 ``lmcache/v1/internal_api_server/{common,vllm,controller}/`` 下并导出 ``router = APIRouter()`` 的文件，名称以 ``*_api.py`` 结尾，将会被自动包含。" #: ../../source/internal_api_server/internal_api_server.rst:70 msgid "Extending the Server" -msgstr "" +msgstr "扩展服务器" #: ../../source/internal_api_server/internal_api_server.rst:72 msgid "To add a new API endpoint:" -msgstr "" +msgstr "要添加一个新的 API 端点：" #: ../../source/internal_api_server/internal_api_server.rst:74 msgid "" "Create a new file in the appropriate category directory (``common/``, " "``vllm/``, or ``controller/``)." -msgstr "" +msgstr "在适当的类别目录中创建一个新文件（``common/``、``vllm/`` 或 ``controller/``）。" #: ../../source/internal_api_server/internal_api_server.rst:76 msgid "Name the file with ``_api.py`` suffix (e.g., ``my_feature_api.py``)." -msgstr "" +msgstr "将文件命名为 ``_api.py`` 后缀（例如，``my_feature_api.py``）。" #: ../../source/internal_api_server/internal_api_server.rst:77 msgid "Define ``router = APIRouter()`` and add your endpoints." -msgstr "" +msgstr "定义 ``router = APIRouter()`` 并添加你的端点。" #: ../../source/internal_api_server/internal_api_server.rst:79 msgid "" "The endpoint will be automatically discovered and registered on the next " "server startup." -msgstr "" +msgstr "该端点将在下次服务器启动时自动发现并注册。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/vllm_apis.po b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/vllm_apis.po index 43ac811b251..37a854d2e7d 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/vllm_apis.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/vllm_apis.po @@ -21,30 +21,30 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:4 msgid "vLLM / Inference APIs" -msgstr "" +msgstr "vLLM / 推理 API" #: ../../source/internal_api_server/vllm_apis.rst:6 msgid "" "These APIs are specific to vLLM inference workers and provide cache " "management, configuration, freeze control, chunk statistics, and version " "information." -msgstr "" +msgstr "这些 API 特定于 vLLM 推理工作者，提供缓存管理、配置、冻结控制、块统计和版本信息。" #: ../../source/internal_api_server/vllm_apis.rst:12 msgid "Endpoints" -msgstr "" +msgstr "端点" #: ../../source/internal_api_server/vllm_apis.rst:15 msgid "Version & Info" -msgstr "" +msgstr "版本与信息" #: ../../source/internal_api_server/vllm_apis.rst:18 msgid "``GET /lmc_version`` — LMCache Version" -msgstr "" +msgstr "``GET /lmc_version`` — LMCache 版本" #: ../../source/internal_api_server/vllm_apis.rst:20 msgid "Get the LMCache library version string." -msgstr "" +msgstr "获取 LMCache 库版本字符串。" #: ../../source/internal_api_server/vllm_apis.rst:22 #: ../../source/internal_api_server/vllm_apis.rst:37 @@ -61,11 +61,11 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:963 #: ../../source/internal_api_server/vllm_apis.rst:1003 msgid "**Method**: ``GET``" -msgstr "" +msgstr "**方法**: ``GET``" #: ../../source/internal_api_server/vllm_apis.rst:23 msgid "**Path**: ``/lmc_version``" -msgstr "" +msgstr "**路径**: ``/lmc_version``" #: ../../source/internal_api_server/vllm_apis.rst:24 #: ../../source/internal_api_server/vllm_apis.rst:39 @@ -87,55 +87,55 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:965 #: ../../source/internal_api_server/vllm_apis.rst:1005 msgid "**Parameters**: None" -msgstr "" +msgstr "**参数**: 无" #: ../../source/internal_api_server/vllm_apis.rst:25 msgid "**Response**: Plain version string." -msgstr "" +msgstr "**响应**：纯文本版本字符串。" #: ../../source/internal_api_server/vllm_apis.rst:33 msgid "``GET /commit_id`` — LMCache Commit ID" -msgstr "" +msgstr "``GET /commit_id`` — LMCache 提交 ID" #: ../../source/internal_api_server/vllm_apis.rst:35 msgid "Get the LMCache git commit ID." -msgstr "" +msgstr "获取 LMCache git 提交 ID。" #: ../../source/internal_api_server/vllm_apis.rst:38 msgid "**Path**: ``/commit_id``" -msgstr "" +msgstr "**路径**: ``/commit_id``" #: ../../source/internal_api_server/vllm_apis.rst:40 msgid "**Response**: Plain commit ID string." -msgstr "" +msgstr "**响应**: 纯文本提交 ID 字符串。" #: ../../source/internal_api_server/vllm_apis.rst:48 msgid "``GET /version`` — Full Version Info" -msgstr "" +msgstr "``GET /version`` — 完整版本信息" #: ../../source/internal_api_server/vllm_apis.rst:50 msgid "Get full version info (version + commit ID)." -msgstr "" +msgstr "获取完整版本信息（版本 + 提交 ID）。" #: ../../source/internal_api_server/vllm_apis.rst:53 msgid "**Path**: ``/version``" -msgstr "" +msgstr "**路径**: ``/version``" #: ../../source/internal_api_server/vllm_apis.rst:55 msgid "**Response**: Combined version string." -msgstr "" +msgstr "**响应**: 组合版本字符串。" #: ../../source/internal_api_server/vllm_apis.rst:63 msgid "``GET /inference_info`` — Inference Information" -msgstr "" +msgstr "``GET /inference_info`` — 推理信息" #: ../../source/internal_api_server/vllm_apis.rst:65 msgid "Get inference information including vLLM config and LMCache details." -msgstr "" +msgstr "获取推理信息，包括 vLLM 配置和 LMCache 详细信息。" #: ../../source/internal_api_server/vllm_apis.rst:68 msgid "**Path**: ``/inference_info``" -msgstr "" +msgstr "**路径**: ``/inference_info``" #: ../../source/internal_api_server/vllm_apis.rst:69 #: ../../source/internal_api_server/vllm_apis.rst:126 @@ -149,7 +149,7 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:1031 #: ../../source/internal_api_server/vllm_apis.rst:1074 msgid "**Parameters**:" -msgstr "" +msgstr "**参数**:" #: ../../source/internal_api_server/vllm_apis.rst:72 #: ../../source/internal_api_server/vllm_apis.rst:129 @@ -163,7 +163,7 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:1034 #: ../../source/internal_api_server/vllm_apis.rst:1077 msgid "Name" -msgstr "" +msgstr "名称" #: ../../source/internal_api_server/vllm_apis.rst:72 #: ../../source/internal_api_server/vllm_apis.rst:129 @@ -178,7 +178,7 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:1034 #: ../../source/internal_api_server/vllm_apis.rst:1077 msgid "Type" -msgstr "" +msgstr "类型" #: ../../source/internal_api_server/vllm_apis.rst:72 #: ../../source/internal_api_server/vllm_apis.rst:129 @@ -193,11 +193,11 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:1034 #: ../../source/internal_api_server/vllm_apis.rst:1077 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/internal_api_server/vllm_apis.rst:74 msgid "``format``" -msgstr "" +msgstr "``format``" #: ../../source/internal_api_server/vllm_apis.rst:74 #: ../../source/internal_api_server/vllm_apis.rst:131 @@ -210,11 +210,11 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:1036 #: ../../source/internal_api_server/vllm_apis.rst:1079 msgid "str" -msgstr "" +msgstr "字符串" #: ../../source/internal_api_server/vllm_apis.rst:74 msgid "(Optional) Reserved for future use" -msgstr "" +msgstr "(可选) 保留以备将来使用" #: ../../source/internal_api_server/vllm_apis.rst:77 #: ../../source/internal_api_server/vllm_apis.rst:101 @@ -244,23 +244,23 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:1039 #: ../../source/internal_api_server/vllm_apis.rst:1082 msgid "**Response**: ``application/json``" -msgstr "" +msgstr "**响应**: ``application/json``" #: ../../source/internal_api_server/vllm_apis.rst:83 msgid "**Error Response** (HTTP 500):" -msgstr "" +msgstr "**错误响应** (HTTP 500):" #: ../../source/internal_api_server/vllm_apis.rst:94 msgid "``GET /inference_version`` — vLLM Version" -msgstr "" +msgstr "``GET /inference_version`` — vLLM 版本" #: ../../source/internal_api_server/vllm_apis.rst:96 msgid "Get the vLLM version information." -msgstr "" +msgstr "获取 vLLM 版本信息。" #: ../../source/internal_api_server/vllm_apis.rst:99 msgid "**Path**: ``/inference_version``" -msgstr "" +msgstr "**路径**: ``/inference_version``" #: ../../source/internal_api_server/vllm_apis.rst:107 #: ../../source/internal_api_server/vllm_apis.rst:254 @@ -283,47 +283,47 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:1045 #: ../../source/internal_api_server/vllm_apis.rst:1088 msgid "**Example Response**:" -msgstr "" +msgstr "**示例响应**:" #: ../../source/internal_api_server/vllm_apis.rst:117 msgid "Configuration & Metadata" -msgstr "" +msgstr "配置与元数据" #: ../../source/internal_api_server/vllm_apis.rst:120 msgid "``GET /conf`` — Get Configuration" -msgstr "" +msgstr "``GET /conf`` — 获取配置" #: ../../source/internal_api_server/vllm_apis.rst:122 msgid "Get current LMCache engine configuration values." -msgstr "" +msgstr "获取当前 LMCache 引擎配置值。" #: ../../source/internal_api_server/vllm_apis.rst:125 #: ../../source/internal_api_server/vllm_apis.rst:162 msgid "**Path**: ``/conf``" -msgstr "" +msgstr "**路径**: ``/conf``" #: ../../source/internal_api_server/vllm_apis.rst:131 #: ../../source/internal_api_server/vllm_apis.rst:212 msgid "``names``" -msgstr "" +msgstr "``names``" #: ../../source/internal_api_server/vllm_apis.rst:131 msgid "(Optional) Comma-separated list of config names to filter" -msgstr "" +msgstr "（可选）以逗号分隔的配置名称列表，用于过滤。" #: ../../source/internal_api_server/vllm_apis.rst:134 msgid "" "**Response**: ``application/json`` — JSON object of configuration key-" "value pairs." -msgstr "" +msgstr "**响应**: ``application/json`` — 配置键值对的 JSON 对象。" #: ../../source/internal_api_server/vllm_apis.rst:146 msgid "``POST /conf`` — Update Configuration (Experimental)" -msgstr "" +msgstr "``POST /conf`` — 更新配置（实验性）" #: ../../source/internal_api_server/vllm_apis.rst:148 msgid "Update one or more configuration values at runtime." -msgstr "" +msgstr "在运行时更新一个或多个配置值。" #: ../../source/internal_api_server/vllm_apis.rst:152 msgid "" @@ -331,14 +331,14 @@ msgid "" "mutable at runtime by default unless explicitly marked as ``\"mutable\": " "False`` in ``_CONFIG_DEFINITIONS``. The default will be changed to " "**immutable** once the feature is stabilized." -msgstr "" +msgstr "此功能目前是 **实验性** 的。除非在 ``_CONFIG_DEFINITIONS`` 中明确标记为 ``\"mutable\": False``，否则所有配置键在运行时默认是可变的。一旦该功能稳定，默认值将更改为 **不可变**。" #: ../../source/internal_api_server/vllm_apis.rst:157 msgid "" "Updating a configuration only modifies the value in the " "``LMCacheEngineConfig`` object. If a component has already cached the " "value elsewhere, the change will **not** take effect for that component." -msgstr "" +msgstr "更新配置仅会修改 ``LMCacheEngineConfig`` 对象中的值。如果某个组件已经在其他地方缓存了该值，则该更改对该组件将**无效**。" #: ../../source/internal_api_server/vllm_apis.rst:161 #: ../../source/internal_api_server/vllm_apis.rst:269 @@ -352,319 +352,319 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:915 #: ../../source/internal_api_server/vllm_apis.rst:939 msgid "**Method**: ``POST``" -msgstr "" +msgstr "**方法**: ``POST``" #: ../../source/internal_api_server/vllm_apis.rst:163 #: ../../source/internal_api_server/vllm_apis.rst:471 msgid "**Content-Type**: ``application/json``" -msgstr "" +msgstr "**Content-Type**: ``application/json``" #: ../../source/internal_api_server/vllm_apis.rst:164 msgid "**Request Body**: JSON object with config name-value pairs." -msgstr "" +msgstr "**请求体**: JSON 对象，包含配置的名称-值对。" #: ../../source/internal_api_server/vllm_apis.rst:173 #: ../../source/internal_api_server/vllm_apis.rst:491 msgid "**Example Response** (HTTP 200):" -msgstr "" +msgstr "**示例响应** (HTTP 200):" #: ../../source/internal_api_server/vllm_apis.rst:184 msgid "**Example Response** (partial failure, HTTP 400):" -msgstr "" +msgstr "**示例响应**（部分失败，HTTP 400）：" #: ../../source/internal_api_server/vllm_apis.rst:193 msgid "**Error Cases**:" -msgstr "" +msgstr "**错误案例**:" #: ../../source/internal_api_server/vllm_apis.rst:195 msgid "Unknown config key → ``\"Unknown config\"``" -msgstr "" +msgstr "未知的配置键 → ``\"Unknown config\"``" #: ../../source/internal_api_server/vllm_apis.rst:196 msgid "Immutable config key → ``\"Config is not mutable at runtime\"``" -msgstr "" +msgstr "不可变配置键 → ``\"配置在运行时不可变\"``" #: ../../source/internal_api_server/vllm_apis.rst:197 msgid "Invalid JSON body → HTTP 400" -msgstr "" +msgstr "无效的 JSON 主体 → HTTP 400" #: ../../source/internal_api_server/vllm_apis.rst:201 msgid "``GET /meta`` — Engine Metadata" -msgstr "" +msgstr "``GET /meta`` — 引擎元数据" #: ../../source/internal_api_server/vllm_apis.rst:203 msgid "" "Get metadata of the LMCache engine (e.g., worker_id, model_name, " "kv_shape)." -msgstr "" +msgstr "获取 LMCache 引擎的元数据（例如，worker_id、model_name、kv_shape）。" #: ../../source/internal_api_server/vllm_apis.rst:206 msgid "**Path**: ``/meta``" -msgstr "" +msgstr "**路径**: ``/meta``" #: ../../source/internal_api_server/vllm_apis.rst:212 msgid "(Optional) Comma-separated list of attribute names to filter" -msgstr "" +msgstr "（可选）以逗号分隔的属性名称列表，用于过滤" #: ../../source/internal_api_server/vllm_apis.rst:215 msgid "**Response**: ``application/json`` — JSON object of metadata attributes." -msgstr "" +msgstr "**响应**: ``application/json`` — 元数据属性的 JSON 对象。" #: ../../source/internal_api_server/vllm_apis.rst:227 msgid "Cache Operations" -msgstr "" +msgstr "缓存操作" #: ../../source/internal_api_server/vllm_apis.rst:230 msgid "``DELETE /cache/clear`` — Clear Cache" -msgstr "" +msgstr "``DELETE /cache/clear`` — 清除缓存" #: ../../source/internal_api_server/vllm_apis.rst:232 msgid "Clear cached KV data from the LMCache engine." -msgstr "" +msgstr "从 LMCache 引擎中清除缓存的 KV 数据。" #: ../../source/internal_api_server/vllm_apis.rst:234 msgid "**Method**: ``DELETE``" -msgstr "" +msgstr "**方法**: ``DELETE``" #: ../../source/internal_api_server/vllm_apis.rst:235 msgid "**Path**: ``/cache/clear``" -msgstr "" +msgstr "**路径**: ``/cache/clear``" #: ../../source/internal_api_server/vllm_apis.rst:241 msgid "``locations``" -msgstr "" +msgstr "``locations``" #: ../../source/internal_api_server/vllm_apis.rst:241 msgid "list[str]" -msgstr "" +msgstr "列表[str]" #: ../../source/internal_api_server/vllm_apis.rst:241 msgid "" "(Optional) Storage backends to clear (e.g. ``LocalCPUBackend``, " "``LocalDiskBackend``). If not specified, clears all." -msgstr "" +msgstr "(可选) 要清除的存储后端（例如 ``LocalCPUBackend``、``LocalDiskBackend``）。如果未指定，则清除所有。" #: ../../source/internal_api_server/vllm_apis.rst:265 msgid "``POST /cache/store`` — Store KV Cache" -msgstr "" +msgstr "``POST /cache/store`` — 存储 KV Cache" #: ../../source/internal_api_server/vllm_apis.rst:267 msgid "Store KV cache data into the LMCache engine using mock tokens." -msgstr "" +msgstr "使用模拟令牌将 KV Cache 数据存储到 LMCache 引擎中。" #: ../../source/internal_api_server/vllm_apis.rst:270 msgid "**Path**: ``/cache/store``" -msgstr "" +msgstr "**路径**: ``/cache/store``" #: ../../source/internal_api_server/vllm_apis.rst:276 #: ../../source/internal_api_server/vllm_apis.rst:316 msgid "``tokens_mock``" -msgstr "" +msgstr "``tokens_mock``" #: ../../source/internal_api_server/vllm_apis.rst:276 msgid "" "Two comma-separated numbers: ``\"start,end\"`` (e.g. ``\"0,100\"`` " "generates tokens [0..99])" -msgstr "" +msgstr "两个用逗号分隔的数字：``\"start,end\"``（例如，``\"0,100\"`` 生成令牌 [0..99]）" #: ../../source/internal_api_server/vllm_apis.rst:294 msgid "**Error Response** (missing params, HTTP 400):" -msgstr "" +msgstr "**错误响应**（缺少参数，HTTP 400）：" #: ../../source/internal_api_server/vllm_apis.rst:305 msgid "``POST /cache/retrieve`` — Retrieve KV Cache" -msgstr "" +msgstr "``POST /cache/retrieve`` — 检索 KV Cache" #: ../../source/internal_api_server/vllm_apis.rst:307 msgid "Retrieve KV cache data from the LMCache engine using mock tokens." -msgstr "" +msgstr "使用模拟令牌从 LMCache 引擎检索 KV Cache 数据。" #: ../../source/internal_api_server/vllm_apis.rst:310 msgid "**Path**: ``/cache/retrieve``" -msgstr "" +msgstr "**路径**: ``/cache/retrieve``" #: ../../source/internal_api_server/vllm_apis.rst:316 msgid "Two comma-separated numbers: ``\"start,end\"`` (e.g. ``\"0,100\"``)" -msgstr "" +msgstr "两个用逗号分隔的数字：``\"start,end\"``（例如 ``\"0,100\"``）" #: ../../source/internal_api_server/vllm_apis.rst:337 msgid "``GET /cache/kvcache/check`` — KVCache Checksum" -msgstr "" +msgstr "``GET /cache/kvcache/check`` — KVCache 校验和" #: ../../source/internal_api_server/vllm_apis.rst:339 msgid "" "Compute MD5 checksums for kvcaches at specified slot_mapping positions. " "Used for verifying that stored and retrieved kvcaches are identical." -msgstr "" +msgstr "计算指定 slot_mapping 位置的 KVCache 的 MD5 校验和。用于验证存储和检索的 KVCache 是否相同。" #: ../../source/internal_api_server/vllm_apis.rst:343 msgid "**Path**: ``/cache/kvcache/check``" -msgstr "" +msgstr "**路径**: ``/cache/kvcache/check``" #: ../../source/internal_api_server/vllm_apis.rst:349 msgid "``slot_mapping``" -msgstr "" +msgstr "``slot_mapping``" #: ../../source/internal_api_server/vllm_apis.rst:349 msgid "" "Slot indices, comma-separated. Supports ranges: ``\"0,1,2,3\"`` or " "``\"1,2,3,[9,12],17,19\"``" -msgstr "" +msgstr "槽索引，以逗号分隔。支持范围：``\"0,1,2,3\"`` 或 ``\"1,2,3,[9,12],17,19\"``" #: ../../source/internal_api_server/vllm_apis.rst:350 msgid "``chunk_size``" -msgstr "" +msgstr "``chunk_size``" #: ../../source/internal_api_server/vllm_apis.rst:350 #: ../../source/internal_api_server/vllm_apis.rst:478 #: ../../source/internal_api_server/vllm_apis.rst:479 msgid "int" -msgstr "" +msgstr "整数" #: ../../source/internal_api_server/vllm_apis.rst:350 msgid "Chunk size for computing per-chunk checksums (required)" -msgstr "" +msgstr "每块计算每块校验和的块大小（必需）" #: ../../source/internal_api_server/vllm_apis.rst:351 msgid "``layerwise``" -msgstr "" +msgstr "``layerwise``" #: ../../source/internal_api_server/vllm_apis.rst:351 #: ../../source/internal_api_server/vllm_apis.rst:781 msgid "bool" -msgstr "" +msgstr "布尔值" #: ../../source/internal_api_server/vllm_apis.rst:351 msgid "If ``true``, output per-layer checksums per chunk (default: ``false``)" -msgstr "" +msgstr "如果 ``true``，则每个块输出每层的校验和（默认值：``false``）" #: ../../source/internal_api_server/vllm_apis.rst:364 msgid "**Example Response** (``layerwise=false``):" -msgstr "" +msgstr "**示例响应** (``layerwise=false``):" #: ../../source/internal_api_server/vllm_apis.rst:377 msgid "**Example Response** (``layerwise=true``):" -msgstr "" +msgstr "**示例响应** (``layerwise=true``):" #: ../../source/internal_api_server/vllm_apis.rst:395 msgid "``POST /cache/kvcache/record_slot`` — Toggle Slot Logging" -msgstr "" +msgstr "``POST /cache/kvcache/record_slot`` — 切换槽记录" #: ../../source/internal_api_server/vllm_apis.rst:397 msgid "" "Enable or disable KVCache slot_mapping logging during store/retrieve " "operations." -msgstr "" +msgstr "在存储/检索操作期间启用或禁用 KVCache slot_mapping 日志记录。" #: ../../source/internal_api_server/vllm_apis.rst:400 msgid "**Path**: ``/cache/kvcache/record_slot``" -msgstr "" +msgstr "**路径**: ``/cache/kvcache/record_slot``" #: ../../source/internal_api_server/vllm_apis.rst:406 msgid "``enabled``" -msgstr "" +msgstr "``enabled``" #: ../../source/internal_api_server/vllm_apis.rst:406 msgid "" "``\"true\"`` to enable, ``\"false\"`` to disable. Omit to query current " "status." -msgstr "" +msgstr "``\"true\"`` 表示启用，``\"false\"`` 表示禁用。省略此项以查询当前状态。" #: ../../source/internal_api_server/vllm_apis.rst:433 msgid "``GET /cache/kvcache/info`` — KVCache Information" -msgstr "" +msgstr "``GET /cache/kvcache/info`` — KVCache 信息" #: ../../source/internal_api_server/vllm_apis.rst:435 msgid "" "Get information about the current kvcaches structure including layer " "names, shapes, and device info." -msgstr "" +msgstr "获取当前 KVCache 结构的信息，包括层名称、形状和设备信息。" #: ../../source/internal_api_server/vllm_apis.rst:439 msgid "**Path**: ``/cache/kvcache/info``" -msgstr "" +msgstr "**路径**: ``/cache/kvcache/info``" #: ../../source/internal_api_server/vllm_apis.rst:465 msgid "``POST /cache/load-fs-chunks`` — Load FS Chunks" -msgstr "" +msgstr "``POST /cache/load-fs-chunks`` — 加载 FS 块" #: ../../source/internal_api_server/vllm_apis.rst:467 msgid "" "Load chunk files from FSConnector storage into LocalCPUBackend's hot " "cache." -msgstr "" +msgstr "从 FSConnector 存储加载块文件到 LocalCPUBackend 的热缓存中。" #: ../../source/internal_api_server/vllm_apis.rst:470 msgid "**Path**: ``/cache/load-fs-chunks``" -msgstr "" +msgstr "**路径**: ``/cache/load-fs-chunks``" #: ../../source/internal_api_server/vllm_apis.rst:472 msgid "**Request Body**:" -msgstr "" +msgstr "**请求体**：" #: ../../source/internal_api_server/vllm_apis.rst:475 msgid "Field" -msgstr "" +msgstr "字段" #: ../../source/internal_api_server/vllm_apis.rst:477 msgid "``config_path``" -msgstr "" +msgstr "``config_path``" #: ../../source/internal_api_server/vllm_apis.rst:477 msgid "Path to LMCache engine configuration YAML file (required)" -msgstr "" +msgstr "LMCache 引擎配置 YAML 文件的路径（必填）" #: ../../source/internal_api_server/vllm_apis.rst:478 msgid "``max_chunks``" -msgstr "" +msgstr "``max_chunks``" #: ../../source/internal_api_server/vllm_apis.rst:478 msgid "(Optional) Maximum number of chunks to load" -msgstr "" +msgstr "(可选) 最大加载块数" #: ../../source/internal_api_server/vllm_apis.rst:479 msgid "``max_failed_keys``" -msgstr "" +msgstr "``max_failed_keys``" #: ../../source/internal_api_server/vllm_apis.rst:479 msgid "Maximum failed keys to report (default: 10)" -msgstr "" +msgstr "报告的最大失败键数（默认：10）" #: ../../source/internal_api_server/vllm_apis.rst:483 msgid "**Tags**: ``cache-management``" -msgstr "" +msgstr "**标签**: ``cache-management``" #: ../../source/internal_api_server/vllm_apis.rst:503 msgid "**Error Response** (invalid config, HTTP 400):" -msgstr "" +msgstr "**错误响应**（无效配置，HTTP 400）：" #: ../../source/internal_api_server/vllm_apis.rst:515 msgid "Freeze Mode" -msgstr "" +msgstr "冻结模式" #: ../../source/internal_api_server/vllm_apis.rst:518 msgid "``PUT /freeze/enable`` — Enable Freeze Mode" -msgstr "" +msgstr "``PUT /freeze/enable`` — 启用冻结模式" #: ../../source/internal_api_server/vllm_apis.rst:520 msgid "Enable freeze mode for the LMCache engine. When enabled:" -msgstr "" +msgstr "为 LMCache 引擎启用冻结模式。启用后：" #: ../../source/internal_api_server/vllm_apis.rst:522 msgid "All store operations will be skipped (no new data stored)" -msgstr "" +msgstr "所有存储操作将被跳过（不存储新数据）" #: ../../source/internal_api_server/vllm_apis.rst:523 msgid "Only ``local_cpu`` backend will be used for retrieval" -msgstr "" +msgstr "仅使用 ``local_cpu`` 后端进行检索" #: ../../source/internal_api_server/vllm_apis.rst:524 msgid "No admit/evict messages will be generated" -msgstr "" +msgstr "不会生成 admit/evict 消息。" #: ../../source/internal_api_server/vllm_apis.rst:526 msgid "This protects the local_cpu hot cache from changes." -msgstr "" +msgstr "这可以保护本地 CPU 热缓存不受更改。" #: ../../source/internal_api_server/vllm_apis.rst:528 #: ../../source/internal_api_server/vllm_apis.rst:553 @@ -673,117 +673,117 @@ msgstr "" #: ../../source/internal_api_server/vllm_apis.rst:1029 #: ../../source/internal_api_server/vllm_apis.rst:1072 msgid "**Method**: ``PUT``" -msgstr "" +msgstr "**方法**: ``PUT``" #: ../../source/internal_api_server/vllm_apis.rst:529 msgid "**Path**: ``/freeze/enable``" -msgstr "" +msgstr "**路径**: ``/freeze/enable``" #: ../../source/internal_api_server/vllm_apis.rst:549 msgid "``PUT /freeze/disable`` — Disable Freeze Mode" -msgstr "" +msgstr "``PUT /freeze/disable`` — 禁用冻结模式" #: ../../source/internal_api_server/vllm_apis.rst:551 msgid "Disable freeze mode. Store operations will proceed normally." -msgstr "" +msgstr "禁用冻结模式。存储操作将正常进行。" #: ../../source/internal_api_server/vllm_apis.rst:554 msgid "**Path**: ``/freeze/disable``" -msgstr "" +msgstr "**路径**: ``/freeze/disable``" #: ../../source/internal_api_server/vllm_apis.rst:574 msgid "``GET /freeze/status`` — Freeze Status" -msgstr "" +msgstr "``GET /freeze/status`` — 冻结状态" #: ../../source/internal_api_server/vllm_apis.rst:576 msgid "Get the current freeze mode status." -msgstr "" +msgstr "获取当前冻结模式状态。" #: ../../source/internal_api_server/vllm_apis.rst:579 msgid "**Path**: ``/freeze/status``" -msgstr "" +msgstr "**路径**: ``/freeze/status``" #: ../../source/internal_api_server/vllm_apis.rst:599 msgid "Hot Cache" -msgstr "" +msgstr "热缓存" #: ../../source/internal_api_server/vllm_apis.rst:601 msgid "" "These endpoints control the hot cache feature of LocalCPUBackend. When " "hot cache is enabled, frequently accessed KV cache data will be kept in " "CPU memory for faster retrieval." -msgstr "" +msgstr "这些端点控制 LocalCPUBackend 的热缓存功能。当启用热缓存时，频繁访问的 KV Cache 数据将保留在 CPU 内存中，以便更快地检索。" #: ../../source/internal_api_server/vllm_apis.rst:606 msgid "``PUT /hot_cache/enable`` — Enable Hot Cache" -msgstr "" +msgstr "``PUT /hot_cache/enable`` — 启用热缓存" #: ../../source/internal_api_server/vllm_apis.rst:608 msgid "Enable hot cache for the LocalCPUBackend." -msgstr "" +msgstr "为 LocalCPUBackend 启用热缓存。" #: ../../source/internal_api_server/vllm_apis.rst:611 msgid "**Path**: ``/hot_cache/enable``" -msgstr "" +msgstr "**路径**: ``/hot_cache/enable``" #: ../../source/internal_api_server/vllm_apis.rst:631 msgid "``PUT /hot_cache/disable`` — Disable Hot Cache" -msgstr "" +msgstr "``PUT /hot_cache/disable`` — 禁用热缓存" #: ../../source/internal_api_server/vllm_apis.rst:633 msgid "" "Disable hot cache for the LocalCPUBackend. Existing hot cache entries " "will be cleared and no new data will be written." -msgstr "" +msgstr "禁用 LocalCPUBackend 的热缓存。现有的热缓存条目将被清除，且不会写入新数据。" #: ../../source/internal_api_server/vllm_apis.rst:637 msgid "**Path**: ``/hot_cache/disable``" -msgstr "" +msgstr "**路径**: ``/hot_cache/disable``" #: ../../source/internal_api_server/vllm_apis.rst:657 msgid "``GET /hot_cache/status`` — Hot Cache Status" -msgstr "" +msgstr "``GET /hot_cache/status`` — 热缓存状态" #: ../../source/internal_api_server/vllm_apis.rst:659 msgid "Get the current hot cache status of LocalCPUBackend." -msgstr "" +msgstr "获取 LocalCPUBackend 的当前热缓存状态。" #: ../../source/internal_api_server/vllm_apis.rst:662 msgid "**Path**: ``/hot_cache/status``" -msgstr "" +msgstr "**路径**: ``/hot_cache/status``" #: ../../source/internal_api_server/vllm_apis.rst:682 msgid "Chunk Statistics" -msgstr "" +msgstr "块统计信息" #: ../../source/internal_api_server/vllm_apis.rst:684 msgid "" "These endpoints manage chunk-level statistics collection via " "``ChunkStatisticsLookupClient``. They are only available when the lookup " "client supports statistics." -msgstr "" +msgstr "这些端点通过 ``ChunkStatisticsLookupClient`` 管理块级统计信息的收集。只有在查找客户端支持统计信息时，它们才可用。" #: ../../source/internal_api_server/vllm_apis.rst:690 msgid "Lookup Client/Server Management" -msgstr "" +msgstr "查找客户端/服务器管理" #: ../../source/internal_api_server/vllm_apis.rst:692 msgid "" "These endpoints allow runtime management of the lookup client and server." " They are useful for dynamically reconfiguring the lookup mechanism " "without restarting the service." -msgstr "" +msgstr "这些端点允许对查找客户端和服务器进行运行时管理。它们对于在不重启服务的情况下动态重新配置查找机制非常有用。" #: ../../source/internal_api_server/vllm_apis.rst:698 msgid "**Configuration Update Required First**" -msgstr "" +msgstr "**首先需要更新配置**" #: ../../source/internal_api_server/vllm_apis.rst:700 msgid "" "Before calling ``/lookup/create`` or ``/lookup/recreate``, you **MUST** " "update the configuration via the ``/conf`` API first. The new lookup " "client/server will be created using ``LookupClientFactory``." -msgstr "" +msgstr "在调用 ``/lookup/create`` 或 ``/lookup/recreate`` 之前，您 **必须** 首先通过 ``/conf`` API 更新配置。新的查找客户端/服务器将使用 ``LookupClientFactory`` 创建。" #: ../../source/internal_api_server/vllm_apis.rst:704 msgid "" @@ -791,104 +791,104 @@ msgid "" "``enable_scheduler_bypass_lookup``), you only need to update the " "**scheduler's** configuration and recreate its lookup client. The workers" " don't need changes in this case." -msgstr "" +msgstr "对于某些配置（例如，切换 ``enable_scheduler_bypass_lookup``），您只需更新 **调度器** 的配置并重新创建其查找客户端。在这种情况下，工作节点无需更改。" #: ../../source/internal_api_server/vllm_apis.rst:710 msgid "``GET /lookup/info`` — Lookup Client/Server Information" -msgstr "" +msgstr "``GET /lookup/info`` — 查找客户端/服务器信息" #: ../../source/internal_api_server/vllm_apis.rst:712 msgid "" "Get information about the current lookup client and server status. Shows " "wrapper chain if applicable (e.g., " "``HitLimitLookupClient(LMCacheLookupClient)``)." -msgstr "" +msgstr "获取当前查找客户端和服务器状态的信息。如果适用，显示包装链（例如，``HitLimitLookupClient(LMCacheLookupClient)``）。" #: ../../source/internal_api_server/vllm_apis.rst:716 msgid "**Path**: ``/lookup/info``" -msgstr "" +msgstr "**路径**: ``/lookup/info``" #: ../../source/internal_api_server/vllm_apis.rst:724 #: ../../source/internal_api_server/vllm_apis.rst:848 msgid "**Example Response** (scheduler):" -msgstr "" +msgstr "**示例响应**（调度器）：" #: ../../source/internal_api_server/vllm_apis.rst:734 #: ../../source/internal_api_server/vllm_apis.rst:858 msgid "**Example Response** (worker):" -msgstr "" +msgstr "**示例响应** (工作者):" #: ../../source/internal_api_server/vllm_apis.rst:746 msgid "``POST /lookup/close`` — Close Lookup Client/Server" -msgstr "" +msgstr "``POST /lookup/close`` — 关闭查找客户端/服务器" #: ../../source/internal_api_server/vllm_apis.rst:748 msgid "Close the current lookup client (scheduler) or server (worker)." -msgstr "" +msgstr "关闭当前的查找客户端（调度器）或服务器（工作节点）。" #: ../../source/internal_api_server/vllm_apis.rst:751 msgid "**Path**: ``/lookup/close``" -msgstr "" +msgstr "**路径**: ``/lookup/close``" #: ../../source/internal_api_server/vllm_apis.rst:770 msgid "``POST /lookup/create`` — Create Lookup Client/Server" -msgstr "" +msgstr "``POST /lookup/create`` — 创建查找客户端/服务器" #: ../../source/internal_api_server/vllm_apis.rst:772 msgid "" "Create a new lookup client (scheduler) or server (worker) using current " "config." -msgstr "" +msgstr "使用当前配置创建一个新的查找客户端（调度器）或服务器（工作进程）。" #: ../../source/internal_api_server/vllm_apis.rst:775 msgid "**Path**: ``/lookup/create``" -msgstr "" +msgstr "**路径**: ``/lookup/create``" #: ../../source/internal_api_server/vllm_apis.rst:781 msgid "``dryrun``" -msgstr "" +msgstr "``dryrun``" #: ../../source/internal_api_server/vllm_apis.rst:781 msgid "If ``true``, only show what would be created" -msgstr "" +msgstr "如果为 ``true``，则仅显示将要创建的内容。" #: ../../source/internal_api_server/vllm_apis.rst:794 msgid "**Example Response** (dryrun):" -msgstr "" +msgstr "**示例响应**（干运行）：" #: ../../source/internal_api_server/vllm_apis.rst:804 msgid "**Example Response** (actual create):" -msgstr "" +msgstr "**示例响应** (实际创建):" #: ../../source/internal_api_server/vllm_apis.rst:815 msgid "``POST /lookup/recreate`` — Recreate Lookup Client/Server" -msgstr "" +msgstr "``POST /lookup/recreate`` — 重新创建查找客户端/服务器" #: ../../source/internal_api_server/vllm_apis.rst:817 msgid "" "Recreate the lookup client or server (equivalent to close + create). The " "endpoint automatically determines which component based on role:" -msgstr "" +msgstr "重新创建查找客户端或服务器（相当于关闭 + 创建）。该端点会根据角色自动确定哪个组件：" #: ../../source/internal_api_server/vllm_apis.rst:820 msgid "**scheduler** role: recreates lookup client" -msgstr "" +msgstr "**scheduler** 角色：重新创建查找客户端" #: ../../source/internal_api_server/vllm_apis.rst:821 msgid "**worker** role: recreates lookup server" -msgstr "" +msgstr "**worker** 角色：重新创建查找服务器" #: ../../source/internal_api_server/vllm_apis.rst:824 msgid "**Path**: ``/lookup/recreate``" -msgstr "" +msgstr "**路径**: ``/lookup/recreate``" #: ../../source/internal_api_server/vllm_apis.rst:828 msgid "**Usage Flow**:" -msgstr "" +msgstr "**使用流程**:" #: ../../source/internal_api_server/vllm_apis.rst:870 msgid "**Client-only Changes**" -msgstr "" +msgstr "**仅客户端更改**" #: ../../source/internal_api_server/vllm_apis.rst:872 msgid "" @@ -896,63 +896,63 @@ msgid "" "``enable_scheduler_bypass_lookup``), you only need to update the " "scheduler's configuration and recreate its lookup client. Worker-side " "lookup servers don't need to be recreated in this case." -msgstr "" +msgstr "对于某些配置更改（例如，切换 ``enable_scheduler_bypass_lookup``），您只需更新调度器的配置并重新创建其查找客户端。在这种情况下，工作节点的查找服务器无需重新创建。" #: ../../source/internal_api_server/vllm_apis.rst:878 msgid "``POST /chunk_statistics/start`` — Start Statistics" -msgstr "" +msgstr "``POST /chunk_statistics/start`` — 开始统计" #: ../../source/internal_api_server/vllm_apis.rst:880 msgid "Start collecting chunk statistics." -msgstr "" +msgstr "开始收集块统计信息。" #: ../../source/internal_api_server/vllm_apis.rst:883 msgid "**Path**: ``/chunk_statistics/start``" -msgstr "" +msgstr "**路径**: ``/chunk_statistics/start``" #: ../../source/internal_api_server/vllm_apis.rst:900 msgid "**Error Response** (not supported, HTTP 400):" -msgstr "" +msgstr "**错误响应**（不支持，HTTP 400）：" #: ../../source/internal_api_server/vllm_apis.rst:911 msgid "``POST /chunk_statistics/stop`` — Stop Statistics" -msgstr "" +msgstr "``POST /chunk_statistics/stop`` — 停止统计" #: ../../source/internal_api_server/vllm_apis.rst:913 msgid "Stop collecting chunk statistics." -msgstr "" +msgstr "停止收集块统计信息。" #: ../../source/internal_api_server/vllm_apis.rst:916 msgid "**Path**: ``/chunk_statistics/stop``" -msgstr "" +msgstr "**路径**: ``/chunk_statistics/stop``" #: ../../source/internal_api_server/vllm_apis.rst:935 msgid "``POST /chunk_statistics/reset`` — Reset Statistics" -msgstr "" +msgstr "``POST /chunk_statistics/reset`` — 重置统计信息" #: ../../source/internal_api_server/vllm_apis.rst:937 msgid "Reset all collected chunk statistics." -msgstr "" +msgstr "重置所有收集的块统计信息。" #: ../../source/internal_api_server/vllm_apis.rst:940 msgid "**Path**: ``/chunk_statistics/reset``" -msgstr "" +msgstr "**路径**: ``/chunk_statistics/reset``" #: ../../source/internal_api_server/vllm_apis.rst:959 msgid "``GET /chunk_statistics/status`` — Statistics Status" -msgstr "" +msgstr "``GET /chunk_statistics/status`` — 统计状态" #: ../../source/internal_api_server/vllm_apis.rst:961 msgid "Get current chunk statistics and auto-exit configuration." -msgstr "" +msgstr "获取当前块统计信息和自动退出配置。" #: ../../source/internal_api_server/vllm_apis.rst:964 msgid "**Path**: ``/chunk_statistics/status``" -msgstr "" +msgstr "**路径**: ``/chunk_statistics/status``" #: ../../source/internal_api_server/vllm_apis.rst:990 msgid "Bypass Mode" -msgstr "" +msgstr "旁路模式" #: ../../source/internal_api_server/vllm_apis.rst:992 msgid "" @@ -960,61 +960,61 @@ msgid "" "runtime. Bypassed backends are excluded from ``contains``/``put``/``get``" " operations. This is useful for fault injection testing, isolating a " "problematic backend, or debugging without restarting the engine." -msgstr "" +msgstr "绕过模式允许在运行时动态跳过特定的存储后端。被绕过的后端将被排除在 ``contains``/``put``/``get`` 操作之外。这对于故障注入测试、隔离有问题的后端或在不重启引擎的情况下进行调试非常有用。" #: ../../source/internal_api_server/vllm_apis.rst:999 msgid "``GET /bypass/list`` — List Bypassed Backends" -msgstr "" +msgstr "``GET /bypass/list`` — 列出被绕过的后端" #: ../../source/internal_api_server/vllm_apis.rst:1001 msgid "List all currently bypassed backends and all available backend names." -msgstr "" +msgstr "列出所有当前被绕过的后端和所有可用的后端名称。" #: ../../source/internal_api_server/vllm_apis.rst:1004 msgid "**Path**: ``/bypass/list``" -msgstr "" +msgstr "**路径**: ``/bypass/list``" #: ../../source/internal_api_server/vllm_apis.rst:1024 msgid "``PUT /bypass/add`` — Add a Backend to Bypass List" -msgstr "" +msgstr "``PUT /bypass/add`` — 将后端添加到绕过列表" #: ../../source/internal_api_server/vllm_apis.rst:1026 msgid "" "Add a backend to the bypass list. The bypassed backend will be excluded " "from ``contains``/``put``/``get`` operations." -msgstr "" +msgstr "将后端添加到绕过列表中。被绕过的后端将被排除在 ``contains``/``put``/``get`` 操作之外。" #: ../../source/internal_api_server/vllm_apis.rst:1030 msgid "**Path**: ``/bypass/add``" -msgstr "" +msgstr "**路径**: ``/bypass/add``" #: ../../source/internal_api_server/vllm_apis.rst:1036 #: ../../source/internal_api_server/vllm_apis.rst:1079 msgid "``backend_name``" -msgstr "" +msgstr "``backend_name``" #: ../../source/internal_api_server/vllm_apis.rst:1036 msgid "Name of the backend to bypass (required)" -msgstr "" +msgstr "要绕过的后端名称（必填）" #: ../../source/internal_api_server/vllm_apis.rst:1057 #: ../../source/internal_api_server/vllm_apis.rst:1100 msgid "**Error Response** (unknown backend, HTTP 400):" -msgstr "" +msgstr "**错误响应**（未知后端，HTTP 400）：" #: ../../source/internal_api_server/vllm_apis.rst:1068 msgid "``PUT /bypass/remove`` — Remove a Backend from Bypass List" -msgstr "" +msgstr "``PUT /bypass/remove`` — 从旁路列表中移除后端" #: ../../source/internal_api_server/vllm_apis.rst:1070 msgid "Remove a backend from the bypass list, restoring it to normal operation." -msgstr "" +msgstr "从旁路列表中移除后端，恢复其正常操作。" #: ../../source/internal_api_server/vllm_apis.rst:1073 msgid "**Path**: ``/bypass/remove``" -msgstr "" +msgstr "**路径**: ``/bypass/remove``" #: ../../source/internal_api_server/vllm_apis.rst:1079 msgid "Name of the backend to restore (required)" -msgstr "" +msgstr "恢复的后端名称（必填）" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/async_loading.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/async_loading.po index 939ccc345ec..85610367254 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/async_loading.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/async_loading.po @@ -21,15 +21,15 @@ msgstr "" #: ../../source/kv_cache/async_loading.rst:2 msgid "Async Loading" -msgstr "" +msgstr "异步加载" #: ../../source/kv_cache/async_loading.rst:6 msgid "Table of Contents" -msgstr "" +msgstr "目录" #: ../../source/kv_cache/async_loading.rst:9 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/async_loading.rst:11 msgid "" @@ -37,111 +37,111 @@ msgid "" "`19330 `_, and " "limitations of the LMCache ``async_loading`` feature. It focuses on " "LMCache v1 integration with vLLM and the internal storage pipeline." -msgstr "" +msgstr "本文解释了 LMCache ``async_loading`` 功能的原理、优点、与 vLLM PR `19330 `_ 的区别以及局限性。它重点介绍了 LMCache v1 与 vLLM 的集成以及内部存储管道。" #: ../../source/kv_cache/async_loading.rst:14 msgid "Key change of components in this feature include:" -msgstr "" +msgstr "此功能中组件的主要变化包括：" #: ../../source/kv_cache/async_loading.rst:16 msgid "LMCache async lookup client/server (ZMQ-based)" -msgstr "" +msgstr "LMCache 异步查找客户端/服务器（基于 ZMQ）" #: ../../source/kv_cache/async_loading.rst:17 msgid "Storage manager orchestrating backends and concurrency" -msgstr "" +msgstr "存储管理器协调后端和并发性" #: ../../source/kv_cache/async_loading.rst:18 msgid "Cache engine async API entrypoints" -msgstr "" +msgstr "缓存引擎异步 API 入口点" #: ../../source/kv_cache/async_loading.rst:19 msgid "vLLM adapter integration points" -msgstr "" +msgstr "vLLM 适配器集成点" #: ../../source/kv_cache/async_loading.rst:22 msgid "Principle and Theory" -msgstr "" +msgstr "原理与理论" #: ../../source/kv_cache/async_loading.rst:24 msgid "" "At a high level, ``async_loading`` decouples scheduler-side lookup from " "worker-side prefetch/retrieval, allowing overlap between I/O and " "computation while preserving prefix-based correctness." -msgstr "" +msgstr "从高层次来看，``async_loading`` 将调度器端的查找与工作端的预取/检索解耦，允许 I/O 和计算之间的重叠，同时保持基于前缀的正确性。" #: ../../source/kv_cache/async_loading.rst:26 msgid "The scheduler sends lookup requests with token chunk hashes and offsets." -msgstr "" +msgstr "调度器发送带有令牌块哈希和偏移量的查找请求。" #: ../../source/kv_cache/async_loading.rst:27 msgid "" "Worker-side servers perform tiered ``batched_async_contains`` over " "available backends and eagerly launch non-blocking batched get operations" " for hit prefixes." -msgstr "" +msgstr "工作端服务器在可用后端上执行分层的 ``batched_async_contains``，并积极启动非阻塞的批量获取操作以处理命中前缀。" #: ../../source/kv_cache/async_loading.rst:28 msgid "" "Completion is tracked via an ``EventManager`` to safely deliver loaded " "memory objects back to the requesting path." -msgstr "" +msgstr "完成情况通过 ``EventManager`` 进行跟踪，以安全地将加载的内存对象返回给请求路径。" #: ../../source/kv_cache/async_loading.rst:29 msgid "" "A weighted semaphore with an ``AsyncSerializer`` prevents allocator " "deadlocks by shaping concurrency according to chunk budget." -msgstr "" +msgstr "带权信号量与 ``AsyncSerializer`` 结合使用，通过根据块预算调整并发性来防止分配器死锁。" #: ../../source/kv_cache/async_loading.rst:31 msgid "The following Mermaid sequence diagram illustrates the end-to-end flow:" -msgstr "" +msgstr "以下的 Mermaid 序列图展示了端到端的流程：" #: ../../source/kv_cache/async_loading.rst:63 msgid "Architecture (Worker Side)" -msgstr "" +msgstr "架构（工作端）" #: ../../source/kv_cache/async_loading.rst:96 msgid "Benefits" -msgstr "" +msgstr "好处" #: ../../source/kv_cache/async_loading.rst:98 msgid "Performance overlap" -msgstr "" +msgstr "性能重叠" #: ../../source/kv_cache/async_loading.rst:99 msgid "" "**I/O–Compute Overlap**: Decoupling lookup/prefetch from loading enables " "fetching KV chunks while vLLM continues scheduling/computation." -msgstr "" +msgstr "**I/O–计算重叠**：将查找/预取与加载解耦，使得在 vLLM 继续调度/计算的同时可以获取 KV 块。" #: ../../source/kv_cache/async_loading.rst:100 msgid "Robustness and error handling" -msgstr "" +msgstr "鲁棒性和错误处理" #: ../../source/kv_cache/async_loading.rst:101 msgid "" "**Event-driven Synchronization**: ``EventManager`` ensures safe hand-off " "of futures and avoids race conditions between threads and the async loop." -msgstr "" +msgstr "**事件驱动同步**：``EventManager`` 确保未来对象的安全交接，并避免线程与异步循环之间的竞争条件。" #: ../../source/kv_cache/async_loading.rst:102 msgid "" "**Backpressure & Deadlock Avoidance**: ``AsyncSerializer`` with a " "weighted semaphore caps concurrent chunk retrievals based on allocator " "budget, preventing starvation or allocator lockups." -msgstr "" +msgstr "**背压与死锁避免**：``AsyncSerializer`` 使用加权信号量限制并发块检索，基于分配器预算，防止饥饿或分配器锁定。" #: ../../source/kv_cache/async_loading.rst:103 msgid "" "**Graceful Miss Path**: Immediate response with ``None`` hit tokens when " "nothing is retrievable; worker returns quickly without stalling the " "scheduler." -msgstr "" +msgstr "**优雅未命中路径**：当无法检索到任何内容时，立即响应 ``None`` 命中令牌；工作线程快速返回，不会阻塞调度器。" #: ../../source/kv_cache/async_loading.rst:106 msgid "Comparison with vLLM Load Failure Recovery feature" -msgstr "" +msgstr "与 vLLM 加载失败恢复功能的比较" #: ../../source/kv_cache/async_loading.rst:108 msgid "" @@ -152,63 +152,63 @@ msgid "" "affected requests for recomputation from a valid prefix. By contrast, " "LMCache’s ``async_loading`` is an externalized caching layer with its own" " client/server, storage backends, and concurrency control." -msgstr "" +msgstr "`VLLM_PR_19330 `_ 引入了一个故障恢复机制，用于 vLLM 的 KV 连接器基础设施，能够优雅地处理 KV 缓存加载失败，通过自动检测失败的块加载并仅重新调度受影响的请求，从有效前缀中进行重计算。相比之下，LMCache 的 ``async_loading`` 是一个外部缓存层，具有自己的客户端/服务器、存储后端和并发控制。" #: ../../source/kv_cache/async_loading.rst:112 msgid "Limitations" -msgstr "" +msgstr "限制" #: ../../source/kv_cache/async_loading.rst:114 msgid "" "Only works with vllm merged `VLLM_PR_23620 `_" -msgstr "" +msgstr "仅适用于合并了 `VLLM_PR_23620 `_ 的 vllm" #: ../../source/kv_cache/async_loading.rst:115 msgid "" "Backend support constraint: This feature currently requires backends that" " implement ``batched_async_contains``; limited to a few backends, e.g.:" -msgstr "" +msgstr "后端支持限制：此功能当前需要实现 ``batched_async_contains`` 的后端；仅限于少数后端，例如：" #: ../../source/kv_cache/async_loading.rst:116 msgid "``LocalCpuBackend``" -msgstr "" +msgstr "``LocalCpuBackend``" #: ../../source/kv_cache/async_loading.rst:117 msgid "``LocalDiskBackend``" -msgstr "" +msgstr "``LocalDiskBackend``" #: ../../source/kv_cache/async_loading.rst:118 msgid "``S3Connector``" -msgstr "" +msgstr "``S3Connector``" #: ../../source/kv_cache/async_loading.rst:119 msgid "``FSConnector``" -msgstr "" +msgstr "``FSConnector``" #: ../../source/kv_cache/async_loading.rst:120 msgid "``RedisConnector/RedisClusterConnector``" -msgstr "" +msgstr "``RedisConnector/RedisClusterConnector``" #: ../../source/kv_cache/async_loading.rst:123 msgid "Future Work" -msgstr "" +msgstr "未来工作" #: ../../source/kv_cache/async_loading.rst:125 msgid "" "Introduce a default ``batched_async_contains`` implementation, so all " "backends can support ``async_loading``." -msgstr "" +msgstr "引入默认的 ``batched_async_contains`` 实现，以便所有后端都可以支持 ``async_loading``。" #: ../../source/kv_cache/async_loading.rst:126 msgid "" "Add metrics and observability to track the number of asynchronous lookup " "requests and the number of occupied ``MemoryObj`` instances." -msgstr "" +msgstr "添加指标和可观察性，以跟踪异步查找请求的数量和占用的 ``MemoryObj`` 实例的数量。" #: ../../source/kv_cache/async_loading.rst:127 msgid "" "Improve the lookup framework by passing vLLM prefix cache hit tokens so " "that async lookup can skip loading parts already hit in vLLM." -msgstr "" +msgstr "通过传递 vLLM 前缀缓存命中令牌来改进查找框架，以便异步查找可以跳过已在 vLLM 中命中的部分加载。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/caching_policies.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/caching_policies.po index b8247b30615..b2f6ae41f37 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/caching_policies.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/caching_policies.po @@ -21,23 +21,23 @@ msgstr "" #: ../../source/kv_cache/caching_policies.rst:2 msgid "Using Different Caching Policies" -msgstr "" +msgstr "使用不同的缓存策略" #: ../../source/kv_cache/caching_policies.rst:4 msgid "LMCache supports multiple caching policies." -msgstr "" +msgstr "LMCache 支持多种缓存策略。" #: ../../source/kv_cache/caching_policies.rst:6 msgid "" "For example, to use LRU, you can set the environment variable " "``LMCACHE_CACHE_POLICY=LRU`` or set it in the configuration file with " "``cache_policy=\"LRU\"``." -msgstr "" +msgstr "例如，要使用 LRU，您可以设置环境变量 ``LMCACHE_CACHE_POLICY=LRU`` 或在配置文件中设置 ``cache_policy=\\\"LRU\\\"``。" #: ../../source/kv_cache/caching_policies.rst:8 msgid "" "Currently, LMCache supports \"LRU\" (Least Recently Used), \"MRU\" (Most " "Recently Used), \"LFU\" (Least Frequently Used) and \"FIFO\" (First-In-" "First-Out) caching policies." -msgstr "" +msgstr "目前，LMCache 支持 \\\"LRU\\\"（最近最少使用）、\\\"MRU\\\"（最近最多使用）、\\\"LFU\\\"（最不常用）和 \\\"FIFO\\\"（先进先出）缓存策略。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/multiprocess_mode.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/multiprocess_mode.po index c2d8b5b118a..99a6aec6589 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/multiprocess_mode.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/multiprocess_mode.po @@ -21,18 +21,18 @@ msgstr "" #: ../../source/kv_cache/multiprocess_mode.rst:4 msgid "Multiprocess Mode (Redirect)" -msgstr "" +msgstr "多进程模式（重定向）" #: ../../source/kv_cache/multiprocess_mode.rst:7 msgid "" "This page has moved. The multiprocess mode documentation is now at " ":doc:`/mp/index`." -msgstr "" +msgstr "此页面已移动。多进程模式文档现在位于 :doc:`/mp/index`。" #: ../../source/kv_cache/multiprocess_mode.rst:10 msgid "" "Please see the new :doc:`/mp/index` section for complete documentation " "including quick start, configuration reference, L2 storage, deployment, " "observability, and architecture guides." -msgstr "" +msgstr "请参阅新的 :doc:`/mp/index` 部分以获取完整的文档，包括快速入门、配置参考、L2 存储、部署、可观察性和架构指南。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po index c569d13ca0f..c6436f40b1b 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/kv_cache/p2p_sharing.rst:4 msgid "P2P KV Cache Sharing" -msgstr "" +msgstr "P2P KV Cache 共享" #: ../../source/kv_cache/p2p_sharing.rst:6 msgid "" @@ -30,64 +30,64 @@ msgid "" "server. This approach provides high-performance cache sharing with " "reduced latency and improved scalability, especially beneficial in " "distributed inference scenarios." -msgstr "" +msgstr "P2P（点对点）KV 缓存共享允许多个服务引擎实例之间直接传输缓存，而无需集中式缓存服务器。这种方法提供了高性能的缓存共享，减少了延迟并提高了可扩展性，特别是在分布式推理场景中非常有利。" #: ../../source/kv_cache/p2p_sharing.rst:8 msgid "" "LMCache supports P2P sharing through a controller-based architecture " "using NIXL (NVIDIA Inference Xfer Library) for optimized data transfer " "between instances." -msgstr "" +msgstr "LMCache 支持通过基于控制器的架构进行 P2P 共享，使用 NIXL（NVIDIA 推理传输库）优化实例之间的数据传输。" #: ../../source/kv_cache/p2p_sharing.rst:11 msgid "Prerequisites" -msgstr "" +msgstr "前提条件" #: ../../source/kv_cache/p2p_sharing.rst:13 msgid "**Multi-GPU Setup**: Your server should have at least 2 GPUs" -msgstr "" +msgstr "**多 GPU 设置**：您的服务器应该至少有 2 个 GPU" #: ../../source/kv_cache/p2p_sharing.rst:14 msgid "**NIC**: RDMA is recommended for more performance." -msgstr "" +msgstr "**网络接口卡**：建议使用 RDMA 以获得更好的性能。" #: ../../source/kv_cache/p2p_sharing.rst:15 msgid "**NIXL**: Install from `NIXL `_" -msgstr "" +msgstr "**NIXL**: 从 `NIXL `_ 安装" #: ../../source/kv_cache/p2p_sharing.rst:16 msgid "" "**vLLM**: v1 version is required, refer to :ref:`installation_guide` for " "details." -msgstr "" +msgstr "**vLLM**: 需要 v1 版本，详细信息请参阅 :ref:`installation_guide`。" #: ../../source/kv_cache/p2p_sharing.rst:17 msgid "**LMCache**: Install from :ref:`installation_guide`" -msgstr "" +msgstr "**LMCache**: 从 :ref:`installation_guide` 安装" #: ../../source/kv_cache/p2p_sharing.rst:20 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/kv_cache/p2p_sharing.rst:22 msgid "Create two configuration files for the P2P sharing setup." -msgstr "" +msgstr "为 P2P 共享设置创建两个配置文件。" #: ../../source/kv_cache/p2p_sharing.rst:25 msgid "**Instance 1 Configuration (example1.yaml)**:" -msgstr "" +msgstr "**实例 1 配置 (example1.yaml)**:" #: ../../source/kv_cache/p2p_sharing.rst:51 msgid "**Instance 2 Configuration (example2.yaml)**:" -msgstr "" +msgstr "**实例 2 配置 (example2.yaml)**:" #: ../../source/kv_cache/p2p_sharing.rst:78 msgid "Setup and Usage" -msgstr "" +msgstr "设置和使用" #: ../../source/kv_cache/p2p_sharing.rst:80 msgid "**Step 1: Start the LMCache Controller**" -msgstr "" +msgstr "**步骤 1：启动 LMCache 控制器**" #: ../../source/kv_cache/p2p_sharing.rst:86 msgid "" @@ -95,91 +95,91 @@ msgid "" "**controller_pull_url** and **controller_reply_url** in the configuration" " files. Port 9000 is the controller main port, which is arbitrary and can" " be changed." -msgstr "" +msgstr "确保在配置文件中将 8300 和 8400 端口设置在 **controller_pull_url** 和 **controller_reply_url** 中。9000 端口是控制器主端口，可以任意更改。" #: ../../source/kv_cache/p2p_sharing.rst:89 msgid "After starting the controller, access the WebUI at:" -msgstr "" +msgstr "启动控制器后，访问 WebUI：" #: ../../source/kv_cache/p2p_sharing.rst:91 msgid "http://localhost:9000/" -msgstr "" +msgstr "http://localhost:9000/" #: ../../source/kv_cache/p2p_sharing.rst:93 msgid "**Step 2: Start vLLM Engines with LMCache Workers**" -msgstr "" +msgstr "**步骤 2：使用 LMCache 工作线程启动 vLLM 引擎**" #: ../../source/kv_cache/p2p_sharing.rst:95 msgid "If the NIC supports RDMA:" -msgstr "" +msgstr "如果网络接口卡支持 RDMA：" #: ../../source/kv_cache/p2p_sharing.rst:101 msgid "If the NIC does not support RDMA:" -msgstr "" +msgstr "如果网络接口卡不支持 RDMA：" #: ../../source/kv_cache/p2p_sharing.rst:107 msgid "Start vLLM engine 1 at port 8010:" -msgstr "" +msgstr "在端口 8010 启动 vLLM 引擎 1：" #: ../../source/kv_cache/p2p_sharing.rst:117 msgid "Start vLLM engine 2 at port 8011:" -msgstr "" +msgstr "在端口 8011 启动 vLLM 引擎 2：" #: ../../source/kv_cache/p2p_sharing.rst:127 msgid "**Step 3: Test P2P Cache Sharing**" -msgstr "" +msgstr "**步骤 3：测试 P2P 缓存共享**" #: ../../source/kv_cache/p2p_sharing.rst:129 msgid "Send a request to vLLM engine 1 to populate the cache:" -msgstr "" +msgstr "向 vLLM 引擎 1 发送请求以填充缓存：" #: ../../source/kv_cache/p2p_sharing.rst:141 msgid "" "Send the same request to vLLM engine 2 to demonstrate cache retrieval " "from **engine 1**:" -msgstr "" +msgstr "将相同的请求发送到 vLLM 引擎 2，以演示从 **引擎 1** 中检索缓存：" #: ../../source/kv_cache/p2p_sharing.rst:154 msgid "Expected Output" -msgstr "" +msgstr "预期输出" #: ../../source/kv_cache/p2p_sharing.rst:156 msgid "" "When the second request successfully retrieves cache from the first " "instance, you should see logs similar to:" -msgstr "" +msgstr "当第二个请求成功从第一个实例检索缓存时，您应该看到类似于以下的日志：" #: ../../source/kv_cache/p2p_sharing.rst:163 msgid "" "These logs indicate successful P2P connection establishment and high-" "throughput cache retrieval." -msgstr "" +msgstr "这些日志表明成功建立了 P2P 连接并高吞吐量地检索了缓存。" #: ../../source/kv_cache/p2p_sharing.rst:167 msgid "**Step 4: Benchmarking P2P Cache Sharing**" -msgstr "" +msgstr "**步骤 4：基准测试 P2P 缓存共享**" #: ../../source/kv_cache/p2p_sharing.rst:169 msgid "Send a request workload to instance 1 to populate the cache:" -msgstr "" +msgstr "向实例 1 发送请求工作负载以填充缓存：" #: ../../source/kv_cache/p2p_sharing.rst:183 msgid "" "Send the same request workload to instance 2 to demonstrate cache " "retrieval from **instance 1**:" -msgstr "" +msgstr "将相同的请求工作负载发送到实例 2，以演示从 **实例 1** 中检索缓存：" #: ../../source/kv_cache/p2p_sharing.rst:199 msgid "Benchmark Results" -msgstr "" +msgstr "基准测试结果" #: ../../source/kv_cache/p2p_sharing.rst:201 msgid "First instance metrics:" -msgstr "" +msgstr "第一次实例指标：" #: ../../source/kv_cache/p2p_sharing.rst:216 msgid "Second instance metrics:" -msgstr "" +msgstr "第二个实例指标：" #: ../../source/kv_cache/p2p_sharing.rst:231 #, python-format @@ -189,5 +189,5 @@ msgid "" "P2P sharing. With LMCache P2P sharing enabled, the time to first token " "(TTFT) is reduced by 54.7%, from 2.286 s to 1.036 s, with a 63.6% " "reduction in total inference time (37.957 s → 13.814 s)." -msgstr "" +msgstr "在这个例子中，long_doc_qa 中的预热轮次指标被使用，因为在一个实例内没有重用现有的 KV Cache 以仅从 P2P 共享中受益。启用 LMCache P2P 共享后，首次令牌时间 (TTFT) 减少了 54.7%，从 2.286 秒降至 1.036 秒，总推理时间减少了 63.6%（37.957 秒 → 13.814 秒）。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/3fs.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/3fs.po index 07c303866bf..186afb2dbc7 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/3fs.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/3fs.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache/storage_backends/3fs.rst:2 msgid "3FS" -msgstr "" +msgstr "3FS" #: ../../source/kv_cache/storage_backends/3fs.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/3fs.rst:9 msgid "" @@ -36,124 +36,124 @@ msgid "" "cluster by FUSE interfaces. It can't leverage 3FS high performance. This " "particular backend uses 3FS native USRBIO(User Space Ring Based IO) " "interfaces to access 3FS storage cluster which get high performance." -msgstr "" +msgstr "3FS（火飞文件系统）是一个 AI 原生的分布式文件系统，提供高性能和低延迟。它是 LMCache 卸载 KV Cache 的一个支持选项。尽管 FSConnector 后端可以与 3FS 存储集群一起工作，但它通过 FUSE 接口访问 3FS 存储集群，无法利用 3FS 的高性能。这个特定的后端使用 3FS 原生的 USRBIO（用户空间环形 IO）接口访问 3FS 存储集群，从而获得高性能。" #: ../../source/kv_cache/storage_backends/3fs.rst:16 msgid "Configure LMCache 3FS Offloading" -msgstr "" +msgstr "配置 LMCache 3FS 卸载" #: ../../source/kv_cache/storage_backends/3fs.rst:18 msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" -msgstr "" +msgstr "通过 ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml`` 传入" #: ../../source/kv_cache/storage_backends/3fs.rst:20 msgid "Example ``config.yaml``:" -msgstr "" +msgstr "示例 ``config.yaml``：" #: ../../source/kv_cache/storage_backends/3fs.rst:66 msgid "There are 2 methods to config hf3fs remote backend:" -msgstr "" +msgstr "配置 hf3fs 远程后端有两种方法：" #: ../../source/kv_cache/storage_backends/3fs.rst:67 msgid "Plugin name mode, uses the parameter remote_storage_plugins(Recommend)" -msgstr "" +msgstr "插件名称模式，使用参数 remote_storage_plugins（推荐）" #: ../../source/kv_cache/storage_backends/3fs.rst:68 msgid "" "URL mode, uses the parameter remote_url(Deprecated, will be removed in a " "future)" -msgstr "" +msgstr "URL 模式，使用参数 remote_url（已弃用，将在未来删除）" #: ../../source/kv_cache/storage_backends/3fs.rst:70 msgid "" "For URL mode, the base_path is contained in the url. For plugin name " "mode, there are 2 methods to set base_path:" -msgstr "" +msgstr "对于 URL 模式，base_path 包含在 url 中。对于插件名称模式，有两种方法可以设置 base_path：" #: ../../source/kv_cache/storage_backends/3fs.rst:71 #, python-brace-format msgid "" "remote_storage_plugin.{plugin name}.base_path, it sets the base_path for " "a plugin instance." -msgstr "" +msgstr "remote_storage_plugin.{plugin name}.base_path，它为插件实例设置 base_path。" #: ../../source/kv_cache/storage_backends/3fs.rst:72 msgid "e.g.:remote_storage_plugin.hf3fs.primary.base_path" -msgstr "" +msgstr "e.g.:remote_storage_plugin.hf3fs.primary.base_path" #: ../../source/kv_cache/storage_backends/3fs.rst:73 msgid "hf3fs_base_path, it set the base_path for all plugin instances" -msgstr "" +msgstr "hf3fs_base_path，它为所有插件实例设置了 base_path" #: ../../source/kv_cache/storage_backends/3fs.rst:76 msgid "Installation" -msgstr "" +msgstr "安装" #: ../../source/kv_cache/storage_backends/3fs.rst:80 msgid "**Prerequisites:**" -msgstr "" +msgstr "**先决条件：**" #: ../../source/kv_cache/storage_backends/3fs.rst:82 msgid "" "A Machine with at least one GPU. You can adjust the max model length of " "your vllm instance depending on your GPU memory." -msgstr "" +msgstr "一台至少配备一块 GPU 的机器。您可以根据您的显存调整 vllm 实例的最大模型长度。" #: ../../source/kv_cache/storage_backends/3fs.rst:84 msgid "vllm and lmcache installed" -msgstr "" +msgstr "已安装 vllm 和 LMCache" #: ../../source/kv_cache/storage_backends/3fs.rst:86 msgid "**Step 1. Install 3FS hf3fs_py_usrbio package**" -msgstr "" +msgstr "**步骤 1. 安装 3FS hf3fs_py_usrbio 包**" #: ../../source/kv_cache/storage_backends/3fs.rst:88 msgid "" "The inference server need install 3FS hf3fs_py_usrbio package, recommend " "to build the package from source:" -msgstr "" +msgstr "推理服务器需要安装 3FS hf3fs_py_usrbio 包，建议从源代码构建该包：" #: ../../source/kv_cache/storage_backends/3fs.rst:99 msgid "" "`3FS Build `_" -msgstr "" +msgstr "`3FS Build `_" #: ../../source/kv_cache/storage_backends/3fs.rst:102 msgid "**Step 2. Setup 3FS storage cluster**" -msgstr "" +msgstr "**步骤 2. 设置 3FS 存储集群**" #: ../../source/kv_cache/storage_backends/3fs.rst:104 msgid "" "`3FS Setup `_" -msgstr "" +msgstr "`3FS 设置 `_" #: ../../source/kv_cache/storage_backends/3fs.rst:107 msgid "**Step 3. Deploy 3FS FUSE client in inference server**" -msgstr "" +msgstr "**步骤 3. 在推理服务器上部署 3FS FUSE 客户端**" #: ../../source/kv_cache/storage_backends/3fs.rst:109 msgid "" "The inference server must deploy 3FS FUSE client(a fuse daemon process " "provided by 3FS), otherwise, it can't access 3FS storage cluster" -msgstr "" +msgstr "推理服务器必须部署 3FS FUSE 客户端（由 3FS 提供的 FUSE 守护进程），否则无法访问 3FS 存储集群。" #: ../../source/kv_cache/storage_backends/3fs.rst:112 msgid "" "`Setup 3FS FUSE Client `_" -msgstr "" +msgstr "`设置 3FS FUSE 客户端 `_" #: ../../source/kv_cache/storage_backends/3fs.rst:115 msgid "**Step 4. Start a vLLM server with 3FS offloading enabled**" -msgstr "" +msgstr "**步骤 4. 启动一个启用 3FS 卸载的 vLLM 服务器**" #: ../../source/kv_cache/storage_backends/3fs.rst:117 msgid "Create a lmcache configuration file called: ``3fs-offload.yaml``" -msgstr "" +msgstr "创建一个名为 ``3fs-offload.yaml`` 的 lmcache 配置文件" #: ../../source/kv_cache/storage_backends/3fs.rst:155 msgid "Start vllm:" -msgstr "" +msgstr "启动 vLLM：" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/cpu_ram.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/cpu_ram.po index 4f7531246e8..05f9d308ee4 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/cpu_ram.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/cpu_ram.po @@ -21,41 +21,41 @@ msgstr "" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:2 msgid "CPU RAM" -msgstr "" +msgstr "CPU 内存" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:9 msgid "" "CPU RAM and Local Storage are the two ways of offloading KV cache onto " "non-GPU memory of the same machine that is running inference." -msgstr "" +msgstr "CPU RAM 和本地存储是将 KV Cache 卸载到同一台运行推理的机器上的非 GPU 内存的两种方式。" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:13 msgid "Two ways to configure LMCache CPU Offloading:" -msgstr "" +msgstr "配置 LMCache CPU 卸载的两种方式：" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:15 msgid "**1. Environment Variables:**" -msgstr "" +msgstr "**1. 环境变量：**" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:26 msgid "**2. Configuration File**:" -msgstr "" +msgstr "**2. 配置文件**:" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:28 msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" -msgstr "" +msgstr "通过 ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml`` 传入" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:30 msgid "Example ``config.yaml``:" -msgstr "" +msgstr "示例 ``config.yaml``:" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:42 msgid "CPU RAM Explanation:" -msgstr "" +msgstr "CPU RAM 说明：" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:44 msgid "" @@ -65,7 +65,7 @@ msgid "" " an intermediate buffer when transferring KV caches with the GPU. This " "means it is possible to set ``LMCACHE_LOCAL_CPU=False`` even though " "``LMCACHE_MAX_LOCAL_CPU_SIZE`` is set to a non-zero number." -msgstr "" +msgstr "``LMCACHE_MAX_LOCAL_CPU_SIZE`` 是 LMCache 将保留的页面锁定（用于快速 GPU 传输）CPU 内存的数量，必须设置为大于 0 的数字，因为本地和远程后端在使用 GPU 传输 KV 缓存时也会使用 CPU 内存作为中间缓冲区。这意味着即使 ``LMCACHE_MAX_LOCAL_CPU_SIZE`` 设置为非零数字，也可以将 ``LMCACHE_LOCAL_CPU=False``。" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:51 msgid "" @@ -76,7 +76,7 @@ msgid "" "RAM is required for any disk or remote transfers, the CPU KV caches will " "be LRU evicted to make space so there is no danger of running out of " "pinned CPU RAM." -msgstr "" +msgstr "然而，建议*始终*将``LMCACHE_LOCAL_CPU=True``设置为真（默认值为``True``，因此如果不指定，CPU 卸载将自动启用），因为这允许 LMCache 保留的所有当前未使用的固定 CPU 内存用于保存 KV 缓存。当固定 CPU 内存需要用于任何磁盘或远程传输时，CPU KV 缓存将被 LRU 逐出以腾出空间，因此不会出现固定 CPU 内存耗尽的危险。" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:56 msgid "" @@ -86,7 +86,7 @@ msgid "" "<./infinistore>`), we can think of the CPU RAM as a \"hot cache\" that " "will contain the \"hottest\" (most recently accessed)subset of KV caches " "from Disk and Remote storage." -msgstr "" +msgstr "当 ``LMCACHE_LOCAL_CPU=True`` 与磁盘后端或远程后端 (:doc:`Redis <./redis>`、:doc:`Mooncake <./mooncake>`、:doc:`Valkey <./valkey>` 或 :doc:`Infinistore <./infinistore>`) 一起使用时，我们可以将 CPU 内存视为一个“热缓存”，它将包含来自磁盘和远程存储的“最热”（最近访问过的）KV 缓存子集。" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:61 msgid "" @@ -96,80 +96,80 @@ msgid "" "there). This can preemptively avoid the latency of the disk and remote KV" " transfer if we predict these tokens will be requested soon (e.g. " "structured or agentic workflows)." -msgstr "" +msgstr "因此，缓存引擎还具有一个 **预取** 机制，可以将指定令牌的 KV 缓存从磁盘或远程存储预加载到固定的 CPU RAM 中（*前提是* 这些令牌的 KV 缓存已经存储在那里）。如果我们预测这些令牌将很快被请求（例如，结构化或自主工作流），这可以预先避免磁盘和远程 KV 传输的延迟。" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:69 msgid "Online Inference Example" -msgstr "" +msgstr "在线推理示例" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:71 msgid "Let's feel the TTFT (time to first token) differential!" -msgstr "" +msgstr "让我们感受一下 TTFT（首次令牌时间）差异！" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:75 msgid "**Prerequisites:**" -msgstr "" +msgstr "**前提条件：**" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:77 msgid "" "A Machine with at least one GPU. Adjust the max model length of your vllm" " instance depending on your GPU memory and the long context you want to " "use." -msgstr "" +msgstr "一台至少配备一个 GPU 的机器。根据您的显存和想要使用的长上下文调整 vllm 实例的最大模型长度。" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:79 msgid "" "vllm and lmcache installed (:doc:`Installation Guide " "<../../getting_started/installation>`)" -msgstr "" +msgstr "已安装 vllm 和 LMCache (:doc:`安装指南 <../../getting_started/installation>`)" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:81 msgid "Hugging Face access to ``meta-llama/Meta-Llama-3.1-8B-Instruct``" -msgstr "" +msgstr "Hugging Face 访问 ``meta-llama/Meta-Llama-3.1-8B-Instruct``" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:87 msgid "A few packages:" -msgstr "" +msgstr "一些软件包：" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:93 msgid "**Step 0. Set up a directory for this example:**" -msgstr "" +msgstr "**步骤 0. 为此示例设置一个目录：**" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:100 msgid "**Step 1. Prepare a long context!**" -msgstr "" +msgstr "**步骤 1. 准备一个长上下文！**" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:102 msgid "" "We want a context long enough that vllm's prefix caching will not be able" " to hold the KV caches in GPU memory and LMCache is necessary to keep KV " "caches in non-GPU memory:" -msgstr "" +msgstr "我们希望上下文足够长，以至于 vLLM 的前缀缓存无法在显存中保持 KV 缓存，因此需要 LMCache 将 KV 缓存保存在非显存中：" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:110 msgid "**Step 2. Start a vLLM server with CPU offloading enabled:**" -msgstr "" +msgstr "**步骤 2. 启动一个启用 CPU 卸载的 vLLM 服务器:**" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:112 msgid "Create a an lmcache configuration file called: ``cpu-offload.yaml``" -msgstr "" +msgstr "创建一个名为 ``cpu-offload.yaml`` 的 lmcache 配置文件" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:120 msgid "" "If you don't want to use a config file, uncomment the first three " "environment variables and then comment out the ``LMCACHE_CONFIG_FILE`` " "below:" -msgstr "" +msgstr "如果您不想使用配置文件，请取消注释前面三个环境变量，然后将下面的 ``LMCACHE_CONFIG_FILE`` 注释掉：" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:135 msgid "" "``--kv-transfer-config``: This is the parameter that actually tells vLLM " "to use LMCache for KV cache offloading." -msgstr "" +msgstr "``--kv-transfer-config``: 这是实际告诉 vLLM 使用 LMCache 进行 KV Cache 卸载的参数。" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:136 msgid "``kv_connector``: Specifies the LMCache connector for vLLM V1" -msgstr "" +msgstr "``kv_connector``: 指定 vLLM V1 的 LMCache 连接器" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:137 msgid "" @@ -177,52 +177,52 @@ msgid "" "(important because we will run two queries and the first will " "produce/store a KV cache while the second will consume/load that KV " "cache)" -msgstr "" +msgstr "``kv_role``: 设置为 \\\"kv_both\\\" 以同时存储和加载 KV Cache（重要，因为我们将运行两个查询，第一个将生成/存储 KV Cache，而第二个将消费/加载该 KV Cache）" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:139 msgid "**Step 3. Query TTFT improvements with LMCache:**" -msgstr "" +msgstr "**步骤 3. 使用 LMCache 查询 TTFT 改进：**" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:141 msgid "" "Once the Open AI compatible server is running on default vllm port 8000, " "let's query it twice with the same long context!" -msgstr "" +msgstr "一旦兼容 Open AI 的服务器在默认的 vllm 端口 8000 上运行，让我们用相同的长上下文查询两次！" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:143 msgid "Create a script called ``query-twice.py`` and paste the following code:" -msgstr "" +msgstr "创建一个名为 ``query-twice.py`` 的脚本，并粘贴以下代码：" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:209 msgid "Then run:" -msgstr "" +msgstr "然后运行：" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:215 msgid "" "Since we're in streaming mode, you'll be able to feel the TTFT " "differential in real time!" -msgstr "" +msgstr "由于我们处于流式模式，您将能够实时感受到 TTFT 差异！" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:218 msgid "**Example Output:**" -msgstr "" +msgstr "**示例输出：**" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:243 msgid "" "If you look at the logs of your vLLM server, you should see (the logs are" " truncated for cleanliness):" -msgstr "" +msgstr "如果你查看 vLLM 服务器的日志，你应该会看到（日志已为保持整洁而截断）：" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:271 msgid "Tips:" -msgstr "" +msgstr "提示：" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:273 msgid "" "If you want to run the ``query-twice.py`` script multiple times, you'll " "need to either restart the vLLM LMCache server or change the prefix of " "the context you pass in since you've already warmed LMCache." -msgstr "" +msgstr "如果您想多次运行 ``query-twice.py`` 脚本，您需要重新启动 vLLM LMCache 服务器或更改您传入的上下文前缀，因为您已经预热了 LMCache。" #: ../../source/kv_cache/storage_backends/cpu_ram.rst:275 msgid "" @@ -231,5 +231,5 @@ msgid "" "length and modify ``query-twice.py`` to use more of the long context. " "LMCache TTFT improvement becomes more pronounced as the context length " "increases!" -msgstr "" +msgstr "这里的最大模型长度是通过运行仅有 23GB 显存的 L4 决定的。如果您有更多内存，可以增加最大模型长度并修改 ``query-twice.py`` 以使用更多的长上下文。随着上下文长度的增加，LMCache 的 TTFT 改进变得更加明显！" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/custom_backend.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/custom_backend.po index aa3363b9f9d..9e5359c5118 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/custom_backend.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/custom_backend.po @@ -21,19 +21,19 @@ msgstr "" #: ../../source/kv_cache/storage_backends/custom_backend.rst:2 msgid "Custom Storage Backends" -msgstr "" +msgstr "自定义存储后端" #: ../../source/kv_cache/storage_backends/custom_backend.rst:4 msgid "" "LMCache supports integrating custom storage backends through dynamic " "loading or plug and play capability. This allows extending cache storage " "capabilities without modifying core code." -msgstr "" +msgstr "LMCache 支持通过动态加载或即插即用能力集成自定义存储后端。这允许在不修改核心代码的情况下扩展缓存存储能力。" #: ../../source/kv_cache/storage_backends/custom_backend.rst:6 msgid "" "See :doc:`Storage Plugins " "<../../developer_guide/extending_lmcache/storage_plugins>` for more " "details." -msgstr "" +msgstr "有关更多详细信息，请参阅 :doc:`存储插件 <../../developer_guide/extending_lmcache/storage_plugins>`。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/dax.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/dax.po index e875210caf1..54cd0cf1162 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/dax.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/dax.po @@ -21,15 +21,15 @@ msgstr "" #: ../../source/kv_cache/storage_backends/dax.rst:1 msgid "# SPDX-License-Identifier: Apache-2.0" -msgstr "" +msgstr "# SPDX-License-Identifier: Apache-2.0" #: ../../source/kv_cache/storage_backends/dax.rst:4 msgid "Device-DAX (/dev/dax)" -msgstr "" +msgstr "设备-DAX (/dev/dax)" #: ../../source/kv_cache/storage_backends/dax.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/dax.rst:9 msgid "" @@ -37,27 +37,27 @@ msgid "" "``mmap(MAP_SHARED)`` and uses the mapped region as a fixed-size arena for" " KV cache chunks. Typical ``/dev/dax`` devices include persistent memory," " CXL-attached memory, and other byte-addressable memory devices." -msgstr "" +msgstr "DAX 存储插件通过 ``mmap(MAP_SHARED)`` 映射 ``/dev/dax`` 设备，并将映射区域用作 KV Cache 块的固定大小区域。典型的 ``/dev/dax`` 设备包括持久内存、CXL 附加内存和其他字节可寻址内存设备。" #: ../../source/kv_cache/storage_backends/dax.rst:14 msgid "" "Data stored on the DAX device may survive process restarts, but is not " "guaranteed to be durable." -msgstr "" +msgstr "存储在 DAX 设备上的数据可能在进程重启后仍然存在，但不保证其持久性。" #: ../../source/kv_cache/storage_backends/dax.rst:17 msgid "" "KV cache data is stored in the DAX region as part of the backend's " "storage flow. Reads copy data back into CPU-backed memory objects." -msgstr "" +msgstr "KV Cache 数据作为后端存储流程的一部分存储在 DAX 区域。读取操作会将数据复制回 CPU 内存对象。" #: ../../source/kv_cache/storage_backends/dax.rst:22 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/kv_cache/storage_backends/dax.rst:42 msgid "Multiprocess Mode" -msgstr "" +msgstr "多进程模式" #: ../../source/kv_cache/storage_backends/dax.rst:44 msgid "" @@ -65,48 +65,48 @@ msgid "" "adapter named ``dax``. The MP adapter uses the normal L2 adapter " "``submit -> event fd -> query`` contract; no vLLM connector protocol " "changes are required." -msgstr "" +msgstr "在 LMCache 多进程模式下，Device-DAX 被配置为名为 ``dax`` 的内置 L2 适配器。MP 适配器使用正常的 L2 适配器 ``submit -> event fd -> query`` 合同；不需要 vLLM 连接器协议的更改。" #: ../../source/kv_cache/storage_backends/dax.rst:64 msgid "The ``--l2-adapter`` JSON accepts these fields:" -msgstr "" +msgstr "``--l2-adapter`` JSON 接受以下字段：" #: ../../source/kv_cache/storage_backends/dax.rst:66 msgid "``device_path``: required path to a readable and writable DAX device." -msgstr "" +msgstr "``device_path``: 必须可读写的 DAX 设备路径。" #: ../../source/kv_cache/storage_backends/dax.rst:67 msgid "" "``max_dax_size_gb``: required mapped size in GiB. The value must fit " "within the device capacity when capacity can be determined with " "``fstat``." -msgstr "" +msgstr "``max_dax_size_gb``: 必需的映射大小，以 GiB 为单位。当可以通过 ``fstat`` 确定容量时，值必须适合设备容量。" #: ../../source/kv_cache/storage_backends/dax.rst:69 msgid "" "``slot_bytes``: required fixed slot size in bytes. It must be large " "enough for one full LMCache chunk." -msgstr "" +msgstr "``slot_bytes``: 所需的固定槽大小（以字节为单位）。它必须足够大，以容纳一个完整的 LMCache 块。" #: ../../source/kv_cache/storage_backends/dax.rst:71 msgid "``num_store_workers``: optional store worker count, default ``1``." -msgstr "" +msgstr "``num_store_workers``: 可选的存储工作线程数量，默认值为 ``1``。" #: ../../source/kv_cache/storage_backends/dax.rst:72 msgid "``num_lookup_workers``: optional lookup worker count, default ``1``." -msgstr "" +msgstr "``num_lookup_workers``: 可选查找工作线程数量，默认为 ``1``。" #: ../../source/kv_cache/storage_backends/dax.rst:73 msgid "" "``num_load_workers``: optional load worker count, default ``min(4, " "os.cpu_count())``." -msgstr "" +msgstr "``num_load_workers``: 可选的加载工作线程数量，默认 ``min(4, os.cpu_count())``。" #: ../../source/kv_cache/storage_backends/dax.rst:75 msgid "" "``persist_enabled``: accepted by common MP L2 parsing but ignored by " "``dax`` in this release." -msgstr "" +msgstr "``persist_enabled``: 被通用 MP L2 解析接受，但在此版本中被 ``dax`` 忽略。" #: ../../source/kv_cache/storage_backends/dax.rst:78 msgid "" @@ -114,7 +114,7 @@ msgid "" "in this release. Closing and reopening the server on the same DAX path " "starts with an empty index, so previously written bytes are not " "discoverable after restart." -msgstr "" +msgstr "MP DAX 在内存中存储不透明的 ``ObjectKey`` 值，并且在此版本中仅为易失性。关闭并重新打开同一路径上的服务器时，索引将为空，因此之前写入的字节在重启后无法被发现。" #: ../../source/kv_cache/storage_backends/dax.rst:83 msgid "" @@ -123,140 +123,140 @@ msgid "" "restart recovery. Capacity accounting and eviction are slot-based: a " "stored object occupies one slot even if its payload is smaller than " "``slot_bytes``." -msgstr "" +msgstr "MP DAX 每个 LMCache 服务器使用一个映射的设备路径。它不添加每个 TP 的 DAX 分区、多设备条带、设备内元数据或重启恢复。容量管理和逐出是基于插槽的：即使存储对象的有效负载小于 ``slot_bytes``，它仍占用一个插槽。" #: ../../source/kv_cache/storage_backends/dax.rst:90 msgid "Using The Batched Restore Path" -msgstr "" +msgstr "使用批量恢复路径" #: ../../source/kv_cache/storage_backends/dax.rst:92 msgid "" "The current DAX optimization is a staged batched restore path for " "retrieval. It is enabled automatically whenever the DAX backend is " "configured. No extra feature flag is required." -msgstr "" +msgstr "当前的 DAX 优化是一个分阶段的批量恢复路径，用于检索。只要配置了 DAX 后端，它会自动启用。无需额外的功能标志。" #: ../../source/kv_cache/storage_backends/dax.rst:96 msgid "The retrieve flow is:" -msgstr "" +msgstr "检索流程是：" #: ../../source/kv_cache/storage_backends/dax.rst:98 msgid "Reserve a batched set of readable DAX chunks." -msgstr "" +msgstr "保留一组可读的 DAX 块。" #: ../../source/kv_cache/storage_backends/dax.rst:99 msgid "Allocate CPU restore buffers from ``LocalCPUBackend``." -msgstr "" +msgstr "从 ``LocalCPUBackend`` 分配 CPU 恢复缓冲区。" #: ../../source/kv_cache/storage_backends/dax.rst:100 msgid "" "Copy DAX data into a backend-owned pinned staging slab in coalesced " "regions." -msgstr "" +msgstr "将 DAX 数据复制到后端拥有的固定暂存块中的合并区域。" #: ../../source/kv_cache/storage_backends/dax.rst:101 msgid "Copy from the staging slab into the final CPU ``MemoryObj`` outputs." -msgstr "" +msgstr "从暂存块复制到最终的 CPU ``MemoryObj`` 输出。" #: ../../source/kv_cache/storage_backends/dax.rst:102 msgid "Upload those CPU outputs through the normal GPU connector path." -msgstr "" +msgstr "通过正常的 GPU 连接器路径上传这些 CPU 输出。" #: ../../source/kv_cache/storage_backends/dax.rst:104 msgid "" "The store flow is unchanged: KV data is still staged through CPU memory " "before being written into the DAX arena." -msgstr "" +msgstr "存储流程保持不变：KV 数据仍然通过 CPU 内存进行暂存，然后写入 DAX 区域。" #: ../../source/kv_cache/storage_backends/dax.rst:107 msgid "The new DAX tuning knobs control the batched restore path:" -msgstr "" +msgstr "新的 DAX 调优控制批量恢复路径的旋钮：" #: ../../source/kv_cache/storage_backends/dax.rst:109 msgid "" "``dax.restore_workers``: number of persistent worker threads used to " "execute restore regions in parallel." -msgstr "" +msgstr "``dax.restore_workers``: 用于并行执行恢复区域的持久工作线程数量。" #: ../../source/kv_cache/storage_backends/dax.rst:111 msgid "" "``dax.restore_max_regions``: maximum number of restore regions in one " "wave. Larger values increase parallelism but also increase slab space " "requirements." -msgstr "" +msgstr "``dax.restore_max_regions``: 一次波次中最大恢复区域的数量。较大的值会增加并行性，但也会增加 slab 空间的需求。" #: ../../source/kv_cache/storage_backends/dax.rst:113 msgid "" "``dax.retrieve_staging_slab_bytes``: total size in bytes of the reusable " "pinned retrieve slab. This must be large enough to hold one full chunk " "per configured restore region." -msgstr "" +msgstr "``dax.retrieve_staging_slab_bytes``: 可重用的固定检索块的总大小（以字节为单位）。这必须足够大，以容纳每个配置的恢复区域的一个完整块。" #: ../../source/kv_cache/storage_backends/dax.rst:117 msgid "For a first pass, start with:" -msgstr "" +msgstr "对于第一次尝试，从以下开始：" #: ../../source/kv_cache/storage_backends/dax.rst:119 msgid "" "``dax.restore_workers`` equal to the number of CPU workers you want " "devoted to DAX restores" -msgstr "" +msgstr "``dax.restore_workers`` 等于您希望用于 DAX 恢复的 CPU 工作线程数量" #: ../../source/kv_cache/storage_backends/dax.rst:121 msgid "``dax.restore_max_regions`` equal to ``dax.restore_workers``" -msgstr "" +msgstr "``dax.restore_max_regions`` 等于 ``dax.restore_workers``" #: ../../source/kv_cache/storage_backends/dax.rst:122 msgid "" "``dax.retrieve_staging_slab_bytes`` at least ``dax.restore_max_regions * " "full_chunk_size``, then scale upward if larger batched restores are " "common" -msgstr "" +msgstr "``dax.retrieve_staging_slab_bytes`` 至少为 ``dax.restore_max_regions * full_chunk_size``，如果批量恢复较大，则向上调整。" #: ../../source/kv_cache/storage_backends/dax.rst:126 msgid "" "If retrieve throughput is low, increase the slab size first, then " "increase worker and region counts together. If CPU pressure is high, " "reduce ``dax.restore_workers`` and ``dax.restore_max_regions``." -msgstr "" +msgstr "如果提取吞吐量较低，首先增加块大小，然后一起增加工作线程和区域数量。如果 CPU 压力较高，减少 ``dax.restore_workers`` 和 ``dax.restore_max_regions``。" #: ../../source/kv_cache/storage_backends/dax.rst:132 msgid "Runtime Requirements" -msgstr "" +msgstr "运行时要求" #: ../../source/kv_cache/storage_backends/dax.rst:134 msgid "" "``extra_config['dax.device_path']`` is required and must point to a " "readable and writable DAX device." -msgstr "" +msgstr "``extra_config['dax.device_path']`` 是必需的，并且必须指向一个可读写的 DAX 设备。" #: ../../source/kv_cache/storage_backends/dax.rst:136 msgid "" "The process must have read-write access to the DAX device (e.g., via " "appropriate permissions or group membership)." -msgstr "" +msgstr "该进程必须对 DAX 设备具有读写访问权限（例如，通过适当的权限或组成员资格）。" #: ../../source/kv_cache/storage_backends/dax.rst:138 msgid "" "``LocalCPUBackend`` must be enabled because DAX reads return CPU-backed " "memory objects." -msgstr "" +msgstr "``LocalCPUBackend`` 必须启用，因为 DAX 读取返回的是 CPU 支持的内存对象。" #: ../../source/kv_cache/storage_backends/dax.rst:143 msgid "Validation and Current Limits" -msgstr "" +msgstr "验证和当前限制" #: ../../source/kv_cache/storage_backends/dax.rst:145 msgid "" "Tensor parallelism is currently limited to TP=1 (``metadata.world_size ==" " 1``)." -msgstr "" +msgstr "当前张量并行性仅限于 TP=1 (``metadata.world_size == 1``)。" #: ../../source/kv_cache/storage_backends/dax.rst:147 msgid "" "Only single-tensor chunk layouts are supported. Multi-tensor put requests" " are rejected." -msgstr "" +msgstr "仅支持单张量块布局。多张量放置请求将被拒绝。" #: ../../source/kv_cache/storage_backends/dax.rst:149 msgid "" @@ -264,11 +264,11 @@ msgid "" " restore executors. The slab and region count can be tuned with " "``dax.restore_workers``, ``dax.restore_max_regions``, and " "``dax.retrieve_staging_slab_bytes``." -msgstr "" +msgstr "批量恢复使用后端拥有的检索暂存块和持久恢复执行器。可以通过 ``dax.restore_workers``、``dax.restore_max_regions`` 和 ``dax.retrieve_staging_slab_bytes`` 调整暂存块和区域数量。" #: ../../source/kv_cache/storage_backends/dax.rst:153 msgid "" "Blocking batched restore preserves positional output semantics, while " "asynchronous batched restore returns only the consecutive hit prefix." -msgstr "" +msgstr "阻塞批量恢复保留了位置输出语义，而异步批量恢复仅返回连续的命中前缀。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/eic.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/eic.po index 0944d58f303..98b59fc8ec6 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/eic.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/eic.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/kv_cache/storage_backends/eic.rst:2 msgid "EIC" -msgstr "" +msgstr "EIC" #: ../../source/kv_cache/storage_backends/eic.rst:4 msgid "" @@ -29,19 +29,19 @@ msgid "" "Cache. It supports RDMA, GDR and has the capabilities of distributed " "disaster tolerance and expansion. You can understand the principles and " "architecture of EIC through these articles:" -msgstr "" +msgstr "EIC（弹性瞬时缓存）是为 LLM KV Cache 设计的分布式数据库。它支持 RDMA、GDR，并具备分布式灾难容忍和扩展能力。您可以通过这些文章了解 EIC 的原理和架构：" #: ../../source/kv_cache/storage_backends/eic.rst:7 msgid "https://mp.weixin.qq.com/s/tasDqXf0Gxr3o_WCJ2IJUQ" -msgstr "" +msgstr "https://mp.weixin.qq.com/s/tasDqXf0Gxr3o_WCJ2IJUQ" #: ../../source/kv_cache/storage_backends/eic.rst:8 msgid "https://mp.weixin.qq.com/s/b_4YhTa96Zeklh23lv8qBw" -msgstr "" +msgstr "https://mp.weixin.qq.com/s/b_4YhTa96Zeklh23lv8qBw" #: ../../source/kv_cache/storage_backends/eic.rst:11 msgid "Deploy EIC" -msgstr "" +msgstr "部署 EIC" #: ../../source/kv_cache/storage_backends/eic.rst:13 msgid "" @@ -50,23 +50,23 @@ msgid "" "provide particular image in volcano engine, which integrates various " "optimizations based on the official image. You may use " "tests/v1/storage_backend/test_eic.py to detect the connectivity of EIC." -msgstr "" +msgstr "您可以访问官方链接 https://console.volcengine.com/eic 并通过 Web UI 在您的计算集群上部署 EIC KVCache。此外，我们在火山引擎中提供了特定的镜像，该镜像基于官方镜像集成了各种优化。您可以使用 tests/v1/storage_backend/test_eic.py 来检测 EIC 的连接性。" #: ../../source/kv_cache/storage_backends/eic.rst:17 msgid "Deploy Model With EIC" -msgstr "" +msgstr "使用 EIC 部署模型" #: ../../source/kv_cache/storage_backends/eic.rst:19 msgid "You can enable EIC KVCache offload with the official interface, such as" -msgstr "" +msgstr "您可以通过官方接口启用 EIC KVCache 卸载，例如" #: ../../source/kv_cache/storage_backends/eic.rst:30 msgid "Example ``config.yaml``:" -msgstr "" +msgstr "示例 ``config.yaml``:" #: ../../source/kv_cache/storage_backends/eic.rst:40 msgid "" "For more details, you can see " "https://www.volcengine.com/docs/85848/1749188." -msgstr "" +msgstr "有关更多详细信息，请参见 https://www.volcengine.com/docs/85848/1749188。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/gds.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/gds.po index f0761822f0e..78b413ed9e1 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/gds.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/gds.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache/storage_backends/gds.rst:2 msgid "GDS Backend" -msgstr "" +msgstr "GDS 后端" #: ../../source/kv_cache/storage_backends/gds.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/gds.rst:9 msgid "" @@ -35,31 +35,31 @@ msgid "" "Storage) optimizations are used for zero-copy I/O from GPU memory to " "storage systems. Supports both NVIDIA cuFile and AMD hipFile for GPU-" "direct storage." -msgstr "" +msgstr "该后端可以与任何文件系统配合使用，无论是本地的、远程的，还是带有基于 GDS 的优化的远程文件系统。远程文件系统允许多个 LMCache 实例无缝共享数据。GDS（GPU-Direct Storage）优化用于实现从显存到存储系统的零拷贝 I/O。支持 NVIDIA cuFile 和 AMD hipFile 进行 GPU 直接存储。" #: ../../source/kv_cache/storage_backends/gds.rst:17 msgid "Ways to configure LMCache GDS Backend" -msgstr "" +msgstr "配置 LMCache GDS 后端的方法" #: ../../source/kv_cache/storage_backends/gds.rst:19 msgid "**1. Environment Variables:**" -msgstr "" +msgstr "**1. 环境变量：**" #: ../../source/kv_cache/storage_backends/gds.rst:33 msgid "**2. Configuration File**:" -msgstr "" +msgstr "**2. 配置文件**:" #: ../../source/kv_cache/storage_backends/gds.rst:35 msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" -msgstr "" +msgstr "通过 ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml`` 传入" #: ../../source/kv_cache/storage_backends/gds.rst:37 msgid "Example ``config.yaml``:" -msgstr "" +msgstr "示例 ``config.yaml``:" #: ../../source/kv_cache/storage_backends/gds.rst:52 msgid "Multi-Path (Multi-Device) Support" -msgstr "" +msgstr "多路径（多设备）支持" #: ../../source/kv_cache/storage_backends/gds.rst:54 msgid "" @@ -69,56 +69,56 @@ msgid "" "path. Currently only ``\"by_gpu\"`` is supported (the default), which " "selects a path based on the device index (``device_id % num_paths``), so " "traffic is spread evenly across the drives without any manual pinning." -msgstr "" +msgstr "当系统有多个 NVMe 驱动器时，可以通过在 ``gds_path`` 中指定以逗号分隔的路径列表来分配 GDS I/O。``gds_path_sharding`` 选项控制每个 GPU 工作线程如何选择其路径。目前仅支持 ``\"by_gpu\"``（默认），该选项根据设备索引（``device_id % num_paths``）选择路径，因此流量在驱动器之间均匀分配，无需手动固定。" #: ../../source/kv_cache/storage_backends/gds.rst:61 msgid "" "**Why this helps:** a single PCIe Gen 4 x4 NVMe tops out at ~7 GB/s. With" " four drives the aggregate bandwidth can reach ~28 GB/s, matching what " "multi-GPU systems need for KV cache eviction and prefetch." -msgstr "" +msgstr "**为什么这有帮助：** 单个 PCIe Gen 4 x4 NVMe 的最大速度约为 7 GB/s。使用四个驱动器时，聚合带宽可以达到约 28 GB/s，满足多 GPU 系统对 KV Cache 逐出和预取的需求。" #: ../../source/kv_cache/storage_backends/gds.rst:65 msgid "**Environment variables:**" -msgstr "" +msgstr "**环境变量：**" #: ../../source/kv_cache/storage_backends/gds.rst:72 msgid "**YAML config:**" -msgstr "" +msgstr "**YAML 配置：**" #: ../../source/kv_cache/storage_backends/gds.rst:79 msgid "With the above configuration on a 4-GPU node:" -msgstr "" +msgstr "在 4-GPU 节点上使用上述配置：" #: ../../source/kv_cache/storage_backends/gds.rst:81 msgid "``cuda:0`` writes to ``/mnt/nvme0/cache``" -msgstr "" +msgstr "``cuda:0`` 写入 ``/mnt/nvme0/cache``" #: ../../source/kv_cache/storage_backends/gds.rst:82 msgid "``cuda:1`` writes to ``/mnt/nvme1/cache``" -msgstr "" +msgstr "``cuda:1`` 写入 ``/mnt/nvme1/cache``" #: ../../source/kv_cache/storage_backends/gds.rst:83 msgid "``cuda:2`` writes to ``/mnt/nvme2/cache``" -msgstr "" +msgstr "``cuda:2`` 写入 ``/mnt/nvme2/cache``" #: ../../source/kv_cache/storage_backends/gds.rst:84 msgid "``cuda:3`` writes to ``/mnt/nvme3/cache``" -msgstr "" +msgstr "``cuda:3`` 写入 ``/mnt/nvme3/cache``" #: ../../source/kv_cache/storage_backends/gds.rst:86 msgid "" "If there are more GPUs than paths, the assignment wraps around (e.g. " "``cuda:4`` maps back to ``/mnt/nvme0/cache``). A single path (no commas) " "works exactly as before." -msgstr "" +msgstr "如果 GPU 的数量超过路径数量，则分配会循环（例如，``cuda:4`` 映射回 ``/mnt/nvme0/cache``）。单个路径（没有逗号）与之前的工作方式完全相同。" #: ../../source/kv_cache/storage_backends/gds.rst:90 msgid "" "All directories are created automatically at startup. Every path in the " "list must reside on a filesystem that the rest of the GDS configuration " "expects (e.g., all paths on GDS-capable mounts when using cuFile)." -msgstr "" +msgstr "所有目录在启动时会自动创建。列表中的每个路径必须位于 GDS 配置所期望的文件系统上（例如，在使用 cuFile 时，所有路径都必须位于支持 GDS 的挂载点上）。" #: ../../source/kv_cache/storage_backends/gds.rst:94 msgid "" @@ -128,11 +128,11 @@ msgid "" "will still discover entries that were written to ``/mnt/nvme1/cache`` by " "``cuda:1`` in a prior run. Writes, however, always go to the single " "affinity-selected path." -msgstr "" +msgstr "**读取行为：** 在启动时，后端会扫描 **所有** 配置的路径以查找先前存储的 KV Cache 条目，而不考虑 GPU 亲和性。这意味着一个写入亲和性为 ``/mnt/nvme0/cache`` 的 ``cuda:0`` 工作线程仍然会发现之前由 ``cuda:1`` 写入到 ``/mnt/nvme1/cache`` 的条目。然而，写入始终会发送到单个亲和性选择的路径。" #: ../../source/kv_cache/storage_backends/gds.rst:110 msgid "GDS Buffer Size Explanation" -msgstr "" +msgstr "GDS 缓冲区大小说明" #: ../../source/kv_cache/storage_backends/gds.rst:112 msgid "" @@ -143,118 +143,118 @@ msgid "" "has 80GiBs of VRAM would be to start with 8GiB and set ``--gpu-memory-" "utilization 0.85`` and depending on your workflow fine-tune it from " "there." -msgstr "" +msgstr "后端目前预先注册缓冲区空间以加速 GDS 操作。此缓冲区空间在显存中注册，因此在设置时应考虑 ``--gpu-memory-utilization`` 等来自 ``vllm`` 的选项。例如，对于通常具有 80GiB 显存的 H100，一个好的经验法则是从 8GiB 开始，并设置 ``--gpu-memory-utilization 0.85``，然后根据您的工作流程进行微调。" #: ../../source/kv_cache/storage_backends/gds.rst:120 msgid "Using AMD hipFile" -msgstr "" +msgstr "使用 AMD hipFile" #: ../../source/kv_cache/storage_backends/gds.rst:124 msgid "" "hipFile is alpha software and has been tested on limited hardware. For " "full installation details, see the `hipFile install guide " "`__." -msgstr "" +msgstr "hipFile 是 alpha 版本的软件，仅在有限的硬件上进行了测试。有关完整的安装细节，请参阅 `hipFile 安装指南 `__。" #: ../../source/kv_cache/storage_backends/gds.rst:128 #: ../../source/kv_cache/storage_backends/gds.rst:179 msgid "**Prerequisites:**" -msgstr "" +msgstr "**先决条件：**" #: ../../source/kv_cache/storage_backends/gds.rst:130 msgid "" "**ROCm >= 7.2** with ``amdgpu-dkms >= 30.20.1`` (see the `ROCm quick " "start installation guide `__)" -msgstr "" +msgstr "**ROCm >= 7.2** 和 ``amdgpu-dkms >= 30.20.1`` （请参阅 `ROCm 快速安装指南 `__）" #: ../../source/kv_cache/storage_backends/gds.rst:132 msgid "**Supported storage:** local NVMe drives only" -msgstr "" +msgstr "**支持的存储：** 仅限本地 NVMe 驱动器" #: ../../source/kv_cache/storage_backends/gds.rst:133 msgid "**Supported filesystems:** ext4 (mounted with ``data=ordered``) and xfs" -msgstr "" +msgstr "**支持的文件系统：** ext4（以 ``data=ordered`` 挂载）和 xfs" #: ../../source/kv_cache/storage_backends/gds.rst:134 msgid "**Kernel:** ``CONFIG_PCI_P2PDMA`` must be enabled" -msgstr "" +msgstr "**内核：** ``CONFIG_PCI_P2PDMA`` 必须启用" #: ../../source/kv_cache/storage_backends/gds.rst:136 msgid "**Quick install (Ubuntu 24.04):**" -msgstr "" +msgstr "**快速安装（Ubuntu 24.04）：**" #: ../../source/kv_cache/storage_backends/gds.rst:147 msgid "" "You can verify that the HIP libraries and kernel support AIS (AMD " "Infinity Storage) by running:" -msgstr "" +msgstr "您可以通过运行以下命令来验证 HIP 库和内核是否支持 AIS（AMD Infinity Storage）：" #: ../../source/kv_cache/storage_backends/gds.rst:153 msgid "" "Successful output will show ``True`` for ``Kernel P2PDMA support``, ``HIP" " runtime``, and ``amdgpu``." -msgstr "" +msgstr "成功的输出将显示 ``Kernel P2PDMA support``、``HIP runtime`` 和 ``amdgpu`` 的值为 ``True``。" #: ../../source/kv_cache/storage_backends/gds.rst:155 msgid "**LMCache configuration:**" -msgstr "" +msgstr "**LMCache 配置：**" #: ../../source/kv_cache/storage_backends/gds.rst:157 msgid "To use AMD hipFile instead of NVIDIA cuFile, set the GDS backend:" -msgstr "" +msgstr "要使用 AMD hipFile 而不是 NVIDIA cuFile，请设置 GDS 后端：" #: ../../source/kv_cache/storage_backends/gds.rst:159 msgid "**Environment Variables:**" -msgstr "" +msgstr "**环境变量：**" #: ../../source/kv_cache/storage_backends/gds.rst:165 msgid "**Configuration File:**" -msgstr "" +msgstr "**配置文件：**" #: ../../source/kv_cache/storage_backends/gds.rst:171 msgid "" "Note: The ``gds_buffer_size`` configuration is used for both cuFile and " "hipFile buffers." -msgstr "" +msgstr "注意：``gds_buffer_size`` 配置用于 cuFile 和 hipFile 缓冲区。" #: ../../source/kv_cache/storage_backends/gds.rst:175 msgid "Setup Example" -msgstr "" +msgstr "设置示例" #: ../../source/kv_cache/storage_backends/gds.rst:181 msgid "" "A Machine with at least one GPU. You can adjust the max model length of " "your vllm instance depending on your GPU memory." -msgstr "" +msgstr "一台至少配备一块 GPU 的机器。您可以根据您的显存调整 vllm 实例的最大模型长度。" #: ../../source/kv_cache/storage_backends/gds.rst:183 msgid "A mounted file system. A file system supportings GDS will work best." -msgstr "" +msgstr "一个挂载的文件系统。支持 GDS 的文件系统效果最佳。" #: ../../source/kv_cache/storage_backends/gds.rst:185 msgid "" "vllm and lmcache installed (:doc:`Installation Guide " "<../../getting_started/installation>`)" -msgstr "" +msgstr "vllm 和 lmcache 已安装 (:doc:`安装指南 <../../getting_started/installation>`)" #: ../../source/kv_cache/storage_backends/gds.rst:187 msgid "Hugging Face access to ``meta-llama/Llama-3.1-8B-Instruct``" -msgstr "" +msgstr "Hugging Face 访问 ``meta-llama/Llama-3.1-8B-Instruct``" #: ../../source/kv_cache/storage_backends/gds.rst:193 msgid "**Step 1. Create cache directory under your file system mount:**" -msgstr "" +msgstr "**步骤 1. 在您的文件系统挂载下创建缓存目录：**" #: ../../source/kv_cache/storage_backends/gds.rst:195 msgid "" "To find all the types of file systems supporting GDS in your system, use " "`gdscheck` from NVIDIA:" -msgstr "" +msgstr "要查找系统中支持 GDS 的所有文件系统类型，请使用 NVIDIA 的 `gdscheck`：" #: ../../source/kv_cache/storage_backends/gds.rst:201 msgid "Check with your storage vendor on how to mount the remote file system." -msgstr "" +msgstr "请向您的存储供应商咨询如何挂载远程文件系统。" #: ../../source/kv_cache/storage_backends/gds.rst:203 msgid "" @@ -263,32 +263,32 @@ msgid "" "source driver that works with any standard [NFS " "RDMA](https://datatracker.ietf.org/doc/html/rfc5532) server. More vendor-" "specific instructions will be added here in the future)." -msgstr "" +msgstr "（例如，如果您想使用支持 GDS 的 NFS 驱动程序，可以尝试修改过的 [NFS stack](https://vastnfs.vastdata.com/)，这是一个与任何标准 [NFS RDMA](https://datatracker.ietf.org/doc/html/rfc5532) 服务器兼容的开源驱动程序。将来会在这里添加更多特定于供应商的说明。）" #: ../../source/kv_cache/storage_backends/gds.rst:209 msgid "" "Create a directory under the file systew mount (the name here is " "arbitrary):" -msgstr "" +msgstr "在文件系统挂载下创建一个目录（这里的名称是任意的）：" #: ../../source/kv_cache/storage_backends/gds.rst:215 msgid "**Step 2. Start a vLLM server with file backend enabled:**" -msgstr "" +msgstr "**步骤 2. 启动一个启用文件后端的 vLLM 服务器：**" #: ../../source/kv_cache/storage_backends/gds.rst:217 msgid "Create a an lmcache configuration file called: ``gds-backend.yaml``" -msgstr "" +msgstr "创建一个名为 ``gds-backend.yaml`` 的 lmcache 配置文件" #: ../../source/kv_cache/storage_backends/gds.rst:226 msgid "" "If you don't want to use a config file, uncomment the first three " "environment variables and then comment out the ``LMCACHE_CONFIG_FILE`` " "below:" -msgstr "" +msgstr "如果您不想使用配置文件，请取消注释前面三个环境变量，然后注释掉下面的 ``LMCACHE_CONFIG_FILE``：" #: ../../source/kv_cache/storage_backends/gds.rst:244 msgid "POSIX fallback" -msgstr "" +msgstr "POSIX 回退" #: ../../source/kv_cache/storage_backends/gds.rst:246 msgid "" @@ -297,28 +297,28 @@ msgid "" "`RuntimeError: cuFileHandleRegister failed (cuFile err=5030, cuda_err=0)`" " may be throwned. Thus, backend can be configured to fallback to its own " "POSIX implementation when the usage of the GDS APIs is not successful." -msgstr "" +msgstr "在某些情况下，libcufile 实现了自己的内部 POSIX 回退，而 `GdsBackend` 并不知情。在其他情况下，可能会抛出类似 `RuntimeError: cuFileHandleRegister failed (cuFile err=5030, cuda_err=0)` 的错误。因此，当 GDS API 的使用不成功时，可以配置后端回退到其自己的 POSIX 实现。" #: ../../source/kv_cache/storage_backends/gds.rst:250 msgid "" "To force `GdsBackend` not use GDS APIs for any reason, you can override " "its behavior via configuration:" -msgstr "" +msgstr "要强制 `GdsBackend` 在任何情况下都不使用 GDS API，您可以通过配置覆盖其行为：" #: ../../source/kv_cache/storage_backends/gds.rst:256 msgid "Or via environment variable:" -msgstr "" +msgstr "或者通过环境变量：" #: ../../source/kv_cache/storage_backends/gds.rst:262 msgid "" "The ``gds_backend`` field (default: ``cufile``) selects which GDS library" " to use. Supported backends are ``cufile`` (NVIDIA cuFile) and " "``hipfile`` (AMD hipFile):" -msgstr "" +msgstr "``gds_backend`` 字段（默认值：``cufile``）用于选择使用哪个 GDS 库。支持的后端有 ``cufile``（NVIDIA cuFile）和 ``hipfile``（AMD hipFile）：" #: ../../source/kv_cache/storage_backends/gds.rst:270 msgid "" "Note that under this mode it would still use CUDA APIs to map and do " "operations the pre-registered GPU memory." -msgstr "" +msgstr "请注意，在此模式下，它仍将使用 CUDA API 来映射和操作预注册的显存。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/hfbucket.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/hfbucket.po index 15dce22767c..83eaca541b7 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/hfbucket.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/hfbucket.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/kv_cache/storage_backends/hfbucket.rst:2 msgid "Hugging Face Buckets Backend" -msgstr "" +msgstr "Hugging Face Buckets 后端" #: ../../source/kv_cache/storage_backends/hfbucket.rst:4 msgid "" @@ -29,86 +29,86 @@ msgid "" "Bucket using LMCache's built-in remote storage plugin framework. This is " "a persistent remote backend that fits warm and cold KV cache persistence " "better than the hottest local tiers." -msgstr "" +msgstr "Hugging Face Buckets 后端使用 LMCache 内置的远程存储插件框架，将 LMCache 块存储在 Hugging Face Bucket 中。这是一个持久的远程后端，适合于温暖和冷却的 KV Cache 持久化，而不是最热的本地层。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:10 msgid "When to use it" -msgstr "" +msgstr "何时使用它" #: ../../source/kv_cache/storage_backends/hfbucket.rst:12 msgid "Use the HFBucket backend when you want:" -msgstr "" +msgstr "当您想要时，请使用 HFBucket 后端：" #: ../../source/kv_cache/storage_backends/hfbucket.rst:14 msgid "A Hub-native persistent store for KV cache data." -msgstr "" +msgstr "一个 Hub 原生的持久存储，用于 KV Cache 数据。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:15 msgid "" "A remote backend that can be configured through " "``remote_storage_plugins``." -msgstr "" +msgstr "一个可以通过 ``remote_storage_plugins`` 配置的远程后端。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:16 msgid "Multiple named bucket instances in one LMCache deployment." -msgstr "" +msgstr "在一个 LMCache 部署中有多个命名的桶实例。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:18 msgid "" "Avoid using it as the primary hot path for the lowest-latency cache " "lookups. Local CPU, local disk, and other lower-latency backends are a " "better fit for the hottest cache tier." -msgstr "" +msgstr "避免将其用作最低延迟缓存查找的主要热路径。本地 CPU、本地磁盘和其他低延迟后端更适合最热门的缓存层。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:24 msgid "Requirements and limitations" -msgstr "" +msgstr "要求和限制" #: ../../source/kv_cache/storage_backends/hfbucket.rst:26 msgid "" "LMCache uses ``huggingface_hub`` bucket APIs for uploads, downloads, " "listing, and deletes." -msgstr "" +msgstr "LMCache 使用 ``huggingface_hub`` 存储桶 API 进行上传、下载、列出和删除操作。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:28 msgid "The first built-in release is intentionally conservative:" -msgstr "" +msgstr "首次内置版本故意采取保守态度：" #: ../../source/kv_cache/storage_backends/hfbucket.rst:30 msgid "Only full chunks are supported." -msgstr "" +msgstr "仅支持完整块。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:31 msgid "Partial chunk uploads are rejected." -msgstr "" +msgstr "部分块上传被拒绝。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:32 msgid "" "Downloads are rejected when the stored object size does not match the " "expected full LMCache chunk size." -msgstr "" +msgstr "当存储的对象大小与预期的完整 LMCache 块大小不匹配时，下载将被拒绝。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:34 msgid "Chunk metadata is not stored in the bucket objects." -msgstr "" +msgstr "块元数据未存储在桶对象中。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:38 msgid "Minimal configuration" -msgstr "" +msgstr "最小配置" #: ../../source/kv_cache/storage_backends/hfbucket.rst:57 msgid "Multiple instances" -msgstr "" +msgstr "多个实例" #: ../../source/kv_cache/storage_backends/hfbucket.rst:59 msgid "" "Use instance-qualified plugin names to configure more than one bucket-" "backed remote store in the same LMCache config." -msgstr "" +msgstr "使用实例限定的插件名称来配置同一 LMCache 配置中的多个基于桶的远程存储。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:73 msgid "Configuration reference" -msgstr "" +msgstr "配置参考" #: ../../source/kv_cache/storage_backends/hfbucket.rst:75 msgid "" @@ -116,53 +116,53 @@ msgid "" "``extra_config.remote_storage_plugin..*`` where " "``plugin_name`` is either ``hfbucket`` or an instance-qualified name such" " as ``hfbucket.prod``." -msgstr "" +msgstr "所有配置键都位于 ``extra_config.remote_storage_plugin..*`` 下，其中 ``plugin_name`` 可以是 ``hfbucket`` 或者实例限定名称，例如 ``hfbucket.prod``。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:79 msgid "" "``bucket_handle`` (required): Hugging Face Bucket handle in " "``hf://buckets//[/]`` format." -msgstr "" +msgstr "``bucket_handle``（必需）：Hugging Face Bucket 句柄，格式为 ``hf://buckets//[/]``。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:81 msgid "" "``token_env`` (optional, default ``HF_TOKEN``): Environment variable used" " to resolve the Hugging Face access token." -msgstr "" +msgstr "``token_env``（可选，默认 ``HF_TOKEN``）：用于解析 Hugging Face 访问令牌的环境变量。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:83 msgid "" "``token`` (optional): Direct token override. ``token_env`` takes " "precedence when both are set." -msgstr "" +msgstr "``token``（可选）：直接的令牌覆盖。当两者都设置时，``token_env`` 优先。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:85 msgid "" "``create_bucket_if_missing`` (optional, default ``false``): Lazily create" " the bucket on the first write path." -msgstr "" +msgstr "``create_bucket_if_missing`` (可选，默认 ``false``)：在第一次写入路径时延迟创建存储桶。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:87 msgid "" "``download_tmp_dir`` (optional): Root directory for connector-local " "download scratch space. On Linux, pointing this at a tmpfs mount such as " "``/dev/shm/lmcache-hfbucket`` avoids the disk write on the download path." -msgstr "" +msgstr "``download_tmp_dir`` (可选): 连接器本地下载临时空间的根目录。在 Linux 上，将其指向 tmpfs 挂载，例如 ``/dev/shm/lmcache-hfbucket``，可以避免下载路径上的磁盘写入。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:90 msgid "" "``metadata_cache_ttl_secs`` (optional, default ``30``): TTL for cached " "exact existence and size metadata." -msgstr "" +msgstr "``metadata_cache_ttl_secs``（可选，默认值为 ``30``）：缓存确切存在性和大小元数据的生存时间（TTL）。" #: ../../source/kv_cache/storage_backends/hfbucket.rst:95 msgid "Notes" -msgstr "" +msgstr "注意事项" #: ../../source/kv_cache/storage_backends/hfbucket.rst:97 msgid "" "The backend stores objects under the configured bucket prefix using a " "reversible encoding of LMCache keys, so ``list()`` returns LMCache key " "strings instead of raw bucket object paths." -msgstr "" +msgstr "后端在配置的桶前缀下存储对象，使用 LMCache 键的可逆编码，因此 ``list()`` 返回 LMCache 键字符串，而不是原始桶对象路径。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/index.po index 1bcb2e16501..b796d6d3b9a 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/index.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/index.po @@ -21,15 +21,15 @@ msgstr "" #: ../../source/kv_cache/storage_backends/index.rst:2 msgid "Using Different Storage Backends" -msgstr "" +msgstr "使用不同的存储后端" #: ../../source/kv_cache/storage_backends/index.rst:4 msgid "" "LMCache supports various storage backends to offload and share KV cache " "data." -msgstr "" +msgstr "LMCache 支持多种存储后端以卸载和共享 KV Cache 数据。" #: ../../source/kv_cache/storage_backends/index.rst:7 msgid "Supported Backends" -msgstr "" +msgstr "支持的后端" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/infinistore.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/infinistore.po index b5a057948a9..ee4f70a21c4 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/infinistore.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/infinistore.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache/storage_backends/infinistore.rst:2 msgid "InfiniStore" -msgstr "" +msgstr "InfiniStore" #: ../../source/kv_cache/storage_backends/infinistore.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/infinistore.rst:9 msgid "" @@ -34,11 +34,11 @@ msgid "" "clusters, whether the cluster is in prefill-decoding disaggregation mode " "or not. InfiniStore provides high-performance and low-latency KV cache " "transfer and KV cache reuse among inference nodes in the cluster." -msgstr "" +msgstr "`InfiniStore `_ 是一个开源的高性能 KV 存储。它旨在支持 LLM 推理集群，无论集群是否处于分离式 Prefill-解码模式。InfiniStore 提供高性能和低延迟的 KV Cache 传输以及集群中推理节点之间的 KV Cache 重用。" #: ../../source/kv_cache/storage_backends/infinistore.rst:11 msgid "There are two major scenarios how InfiniStore supports:" -msgstr "" +msgstr "InfiniStore 支持的主要场景有两个：" #: ../../source/kv_cache/storage_backends/infinistore.rst:13 msgid "" @@ -46,7 +46,7 @@ msgid "" "workloads are separated into two node pools: prefill nodes and decoding " "nodes. InfiniStore enables KV cache transfer among these two types of " "nodes, and also KV cache reuse." -msgstr "" +msgstr "分离式 Prefill-解码集群：在这种模式下，推理工作负载被分为两个节点池：Prefill 节点和解码节点。InfiniStore 允许这两种类型节点之间的 KV Cache 转移，并且支持 KV Cache 重用。" #: ../../source/kv_cache/storage_backends/infinistore.rst:14 msgid "" @@ -54,150 +54,150 @@ msgid "" "are mixed on every node. InfiniStore serves as an extra large KV cache " "pool in addition to GPU cache and local CPU cache, and also enables " "cross-node KV cache reuse." -msgstr "" +msgstr "非分离式集群：在这种模式下，Prefill 和解码工作负载在每个节点上混合。InfiniStore 作为一个额外的大型 KV Cache 池，除了 GPU 缓存和本地 CPU 缓存外，还支持跨节点的 KV Cache 重用。" #: ../../source/kv_cache/storage_backends/infinistore.rst:16 msgid "InfiniStore Usage Diagram" -msgstr "" +msgstr "InfiniStore 使用示意图" #: ../../source/kv_cache/storage_backends/infinistore.rst:19 msgid "" "For more details, please refer to the `InfiniStore Documentation " "`_." -msgstr "" +msgstr "有关更多详细信息，请参阅 `InfiniStore 文档 `_。" #: ../../source/kv_cache/storage_backends/infinistore.rst:21 msgid "" "InfiniStore supports both RDMA and TCP for transport. LMCache’s " "InfiniStore connector only uses the RDMA transport." -msgstr "" +msgstr "InfiniStore 支持 RDMA 和 TCP 作为传输方式。LMCache 的 InfiniStore 连接器仅使用 RDMA 传输。" #: ../../source/kv_cache/storage_backends/infinistore.rst:25 msgid "Quick Start" -msgstr "" +msgstr "快速开始" #: ../../source/kv_cache/storage_backends/infinistore.rst:27 msgid "Install InfiniStore via pip:" -msgstr "" +msgstr "通过 pip 安装 InfiniStore：" #: ../../source/kv_cache/storage_backends/infinistore.rst:33 msgid "This package includes the InfiniStore server and the Python bindings." -msgstr "" +msgstr "该软件包包括 InfiniStore 服务器和 Python 绑定。" #: ../../source/kv_cache/storage_backends/infinistore.rst:35 msgid "" "To build InfiniStore from source, follow the instructions in the `GitHub " "repository `_." -msgstr "" +msgstr "要从源代码构建 InfiniStore，请按照 `GitHub 仓库 `_ 中的说明进行操作。" #: ../../source/kv_cache/storage_backends/infinistore.rst:38 msgid "Setup and Deployment" -msgstr "" +msgstr "设置和部署" #: ../../source/kv_cache/storage_backends/infinistore.rst:40 msgid "**Prerequisites:**" -msgstr "" +msgstr "**前提条件：**" #: ../../source/kv_cache/storage_backends/infinistore.rst:42 msgid "Machine with at least one GPU for vLLM inference" -msgstr "" +msgstr "具有至少一个 GPU 以进行 vLLM 推理的机器" #: ../../source/kv_cache/storage_backends/infinistore.rst:43 msgid "RDMA-capable network hardware and drivers" -msgstr "" +msgstr "支持 RDMA 的网络硬件和驱动程序" #: ../../source/kv_cache/storage_backends/infinistore.rst:44 msgid "Python 3.8+ with pip" -msgstr "" +msgstr "支持 pip 的 Python 3.8 及以上版本" #: ../../source/kv_cache/storage_backends/infinistore.rst:45 msgid "vLLM and LMCache installed" -msgstr "" +msgstr "已安装 vLLM 和 LMCache" #: ../../source/kv_cache/storage_backends/infinistore.rst:47 msgid "**Step 1: Start InfiniStore Server**" -msgstr "" +msgstr "**步骤 1：启动 InfiniStore 服务器**" #: ../../source/kv_cache/storage_backends/infinistore.rst:49 msgid "For InfiniBand based RDMA:" -msgstr "" +msgstr "对于基于 InfiniBand 的 RDMA：" #: ../../source/kv_cache/storage_backends/infinistore.rst:55 msgid "For RoCE based RDMA:" -msgstr "" +msgstr "对于基于 RoCE 的 RDMA：" #: ../../source/kv_cache/storage_backends/infinistore.rst:61 msgid "" "You can also specify the ``--hint-gid-index`` option to set the GID index" " for the InfiniStore server. This is useful when you are in a k8s managed" " environment." -msgstr "" +msgstr "您还可以指定 ``--hint-gid-index`` 选项来设置 InfiniStore 服务器的 GID 索引。当您处于 k8s 管理的环境中时，这非常有用。" #: ../../source/kv_cache/storage_backends/infinistore.rst:63 msgid "**Step 2: Create Configuration File**" -msgstr "" +msgstr "**步骤 2：创建配置文件**" #: ../../source/kv_cache/storage_backends/infinistore.rst:65 msgid "Create your ``infinistore-config.yaml``:" -msgstr "" +msgstr "创建你的 ``infinistore-config.yaml``：" #: ../../source/kv_cache/storage_backends/infinistore.rst:75 msgid "**Step 3: Start vLLM with InfiniStore**" -msgstr "" +msgstr "**步骤 3：使用 InfiniStore 启动 vLLM**" #: ../../source/kv_cache/storage_backends/infinistore.rst:88 msgid "**Step 4: Verify the Setup**" -msgstr "" +msgstr "**步骤 4：验证设置**" #: ../../source/kv_cache/storage_backends/infinistore.rst:90 msgid "Test the integration with a sample request:" -msgstr "" +msgstr "使用示例请求测试集成：" #: ../../source/kv_cache/storage_backends/infinistore.rst:103 msgid "**Debugging Tips:**" -msgstr "" +msgstr "**调试提示：**" #: ../../source/kv_cache/storage_backends/infinistore.rst:105 msgid "**Enable verbose logging:**" -msgstr "" +msgstr "**启用详细日志记录：**" #: ../../source/kv_cache/storage_backends/infinistore.rst:111 msgid "**Check server status:**" -msgstr "" +msgstr "**检查服务器状态：**" #: ../../source/kv_cache/storage_backends/infinistore.rst:120 msgid "Query TTFT Improvement" -msgstr "" +msgstr "查询 TTFT 改进" #: ../../source/kv_cache/storage_backends/infinistore.rst:122 msgid "" "Once the OpenAI compatible server is running, let's query it twice and " "see the TTFT improvement." -msgstr "" +msgstr "一旦 OpenAI 兼容的服务器运行起来，我们就可以查询两次，看看 TTFT 的改进。" #: ../../source/kv_cache/storage_backends/infinistore.rst:124 msgid "Run vLLM's serving benchmark twice with the following parameters:" -msgstr "" +msgstr "使用以下参数运行 vLLM 的服务基准测试两次：" #: ../../source/kv_cache/storage_backends/infinistore.rst:139 msgid "**Example Output:**" -msgstr "" +msgstr "**示例输出：**" #: ../../source/kv_cache/storage_backends/infinistore.rst:141 msgid "For the first run, you might see:" -msgstr "" +msgstr "首次运行时，您可能会看到：" #: ../../source/kv_cache/storage_backends/infinistore.rst:166 msgid "For the second run, you should see a significant reduction in TTFT:" -msgstr "" +msgstr "对于第二次运行，您应该看到 TTFT 有显著减少：" #: ../../source/kv_cache/storage_backends/infinistore.rst:191 msgid "TTFT Improvement: 33.323 seconds (12.6x faster)." -msgstr "" +msgstr "TTFT 改进：33.323 秒（快 12.6 倍）。" #: ../../source/kv_cache/storage_backends/infinistore.rst:193 msgid "**Tips:**" -msgstr "" +msgstr "**提示：**" #: ../../source/kv_cache/storage_backends/infinistore.rst:195 msgid "" @@ -205,7 +205,7 @@ msgid "" "to either restart the vLLM LMCache server and the InfiniStore server, or " "change the ``--seed`` parameter to a different value each time, since " "you've already warmed up LMCache." -msgstr "" +msgstr "如果您想多次运行 vLLM 的服务基准测试，您需要重启 vLLM LMCache 服务器和 InfiniStore 服务器，或者每次将 ``--seed`` 参数更改为不同的值，因为您已经预热了 LMCache。" #: ../../source/kv_cache/storage_backends/infinistore.rst:196 msgid "" @@ -214,19 +214,19 @@ msgid "" "memory utilization and increase the max model length to use more of the " "long context. LMCache TTFT improvement becomes more pronounced as the " "context length increases!" -msgstr "" +msgstr "这里的基准测试结果是通过运行一个具有 48GB 显存的 L40，并使用 ``--gpu-memory-utilization 0.8`` 生成的。您可以调整显存利用率并增加最大模型长度，以使用更多的长上下文。随着上下文长度的增加，LMCache 的 TTFT 改进变得更加明显！" #: ../../source/kv_cache/storage_backends/infinistore.rst:200 msgid "Additional Resources" -msgstr "" +msgstr "附加资源" #: ../../source/kv_cache/storage_backends/infinistore.rst:202 msgid "" "`InfiniStore Documentation " "`_" -msgstr "" +msgstr "`InfiniStore 文档 `_" #: ../../source/kv_cache/storage_backends/infinistore.rst:203 msgid "`GitHub Repository `_" -msgstr "" +msgstr "`GitHub 仓库 `_" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/local_storage.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/local_storage.po index e12d505ee2a..f4faccd404b 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/local_storage.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/local_storage.po @@ -21,44 +21,44 @@ msgstr "" #: ../../source/kv_cache/storage_backends/local_storage.rst:2 msgid "Local storage" -msgstr "" +msgstr "本地存储" #: ../../source/kv_cache/storage_backends/local_storage.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/local_storage.rst:9 msgid "" "CPU RAM and Local Storage are the two ways of offloading KV cache onto " "non-GPU memory of the same machine that is running inference." -msgstr "" +msgstr "CPU RAM 和本地存储是将 KV Cache 卸载到同一台运行推理的机器的非 GPU 内存的两种方式。" #: ../../source/kv_cache/storage_backends/local_storage.rst:14 msgid "Two ways to configure LMCache Disk Offloading:" -msgstr "" +msgstr "配置 LMCache 磁盘卸载的两种方式：" #: ../../source/kv_cache/storage_backends/local_storage.rst:17 msgid "**1. Environment Variables:**" -msgstr "" +msgstr "**1. 环境变量：**" #: ../../source/kv_cache/storage_backends/local_storage.rst:35 msgid "**2. Configuration File**:" -msgstr "" +msgstr "**2. 配置文件**:" #: ../../source/kv_cache/storage_backends/local_storage.rst:37 msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" -msgstr "" +msgstr "通过 ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml`` 传入" #: ../../source/kv_cache/storage_backends/local_storage.rst:54 msgid "Multi-Path (Multi-Device) Disk Offloading" -msgstr "" +msgstr "多路径（多设备）磁盘卸载" #: ../../source/kv_cache/storage_backends/local_storage.rst:56 msgid "" "If you have **multiple NVMe devices** (or any independent mount points), " "you can assign each GPU its own disk path so that each device writes to a" " dedicated drive." -msgstr "" +msgstr "如果您有 **多个 NVMe 设备**（或任何独立的挂载点），您可以为每个 GPU 分配自己的磁盘路径，以便每个设备写入专用驱动器。" #: ../../source/kv_cache/storage_backends/local_storage.rst:59 msgid "" @@ -70,38 +70,38 @@ msgid "" "num_paths``), so all KV cache files from a given GPU land on the same " "NVMe. This is especially useful when GPUs and NVMe devices share a PCIe " "switch or NUMA node." -msgstr "" +msgstr "在 ``local_disk`` 中指定一个 **以逗号分隔的路径列表**。每个路径可以选择性地使用 ``file://`` 前缀。 ``local_disk_path_sharding`` 选项控制每个 GPU 工作线程如何选择其路径。目前仅支持 ``\"by_gpu\"``（默认），它根据设备索引（``device_id % num_paths``）选择路径，因此来自给定 GPU 的所有 KV 缓存文件都会落在同一个 NVMe 上。当 GPU 和 NVMe 设备共享 PCIe 交换机或 NUMA 节点时，这尤其有用。" #: ../../source/kv_cache/storage_backends/local_storage.rst:67 msgid "For example, with two GPUs and two paths:" -msgstr "" +msgstr "例如，使用两个 GPU 和两个路径：" #: ../../source/kv_cache/storage_backends/local_storage.rst:69 msgid "``cuda:0`` → ``/mnt/nvme0/kvcache/``" -msgstr "" +msgstr "``cuda:0`` → ``/mnt/nvme0/kvcache/``" #: ../../source/kv_cache/storage_backends/local_storage.rst:70 msgid "``cuda:1`` → ``/mnt/nvme1/kvcache/``" -msgstr "" +msgstr "``cuda:1`` → ``/mnt/nvme1/kvcache/``" #: ../../source/kv_cache/storage_backends/local_storage.rst:72 msgid "``max_local_disk_size`` is the **total budget** shared across all paths." -msgstr "" +msgstr "``max_local_disk_size`` 是所有路径共享的 **总预算**。" #: ../../source/kv_cache/storage_backends/local_storage.rst:74 msgid "**Environment variable example:**" -msgstr "" +msgstr "**环境变量示例：**" #: ../../source/kv_cache/storage_backends/local_storage.rst:82 msgid "**YAML example:**" -msgstr "" +msgstr "**YAML 示例：**" #: ../../source/kv_cache/storage_backends/local_storage.rst:92 msgid "" "Each GPU worker uses only its assigned path, so O_DIRECT alignment is " "determined by that path's filesystem block size. Different devices may " "have different block sizes without issue." -msgstr "" +msgstr "每个 GPU 工作节点仅使用其分配的路径，因此 O_DIRECT 对齐由该路径的文件系统块大小决定。不同设备可能具有不同的块大小而不会出现问题。" #: ../../source/kv_cache/storage_backends/local_storage.rst:98 msgid "" @@ -110,11 +110,11 @@ msgid "" "bandwidth from both devices simultaneously). The multi-path feature is " "most useful when you cannot or do not want to reconfigure the block " "devices — for example, when they already have other data." -msgstr "" +msgstr "如果您能够使用内核级 RAID 0（例如 ``mdadm --level=0``），您将获得真正的块级条带化（即使是单个大文件也可以同时使用两个设备的带宽）。当您无法或不想重新配置块设备时，多路径功能最为有用——例如，当它们已经包含其他数据时。" #: ../../source/kv_cache/storage_backends/local_storage.rst:105 msgid "Local Storage Explanation:" -msgstr "" +msgstr "本地存储说明：" #: ../../source/kv_cache/storage_backends/local_storage.rst:107 msgid "" @@ -122,7 +122,7 @@ msgid "" "(``local_disk`` is set to ``None``) and the max local disk size is set to" " 0GB instead of 5GB like the default max local cpu size since the disk " "space is not strictly necessary for LMCache to function." -msgstr "" +msgstr "与 CPU RAM 卸载不同，磁盘卸载默认是 *禁用* 的（``local_disk`` 设置为 ``None``），最大本地磁盘大小设置为 0GB，而不是像默认最大本地 CPU 大小那样设置为 5GB，因为磁盘空间对于 LMCache 的功能并不是严格必要的。" #: ../../source/kv_cache/storage_backends/local_storage.rst:111 msgid "" @@ -130,7 +130,7 @@ msgid "" "the pinned CPU RAM, the disk backend will create one file per KV cache " "chunk as they are stored, evicting if capacity is exceeded (LRU " "currently)." -msgstr "" +msgstr "此外，与固定的 CPU RAM 不同，磁盘后端不会贪婪地提前分配最大空间，而是会在存储时为每个 KV 缓存块创建一个文件，如果超出容量则逐出（当前使用 LRU）。" #: ../../source/kv_cache/storage_backends/local_storage.rst:114 msgid "" @@ -143,144 +143,144 @@ msgid "" "storage (i.e. ``LMCACHE_LOCAL_CPU=True`` should be set, see :doc:`CPU RAM" " <./cpu_ram>`) for specified tokens (these KV caches are also still kept " "in the disk)." -msgstr "" +msgstr "磁盘和远程后端（参见 :doc:`Redis <./redis>`、:doc:`Mooncake <./mooncake>`、:doc:`Valkey <./valkey>`、:doc:`InfiniStore <./infinistore>`）具有异步 put() 操作，因此 IO 延迟不会在阻塞 get() 操作的同时减慢推理。 本地磁盘后端还具有 prefetch() 操作，该操作将主动将 KV 缓存从磁盘移动到 CPU 内存卸载存储（即应设置 ``LMCACHE_LOCAL_CPU=True``，参见 :doc:`CPU RAM <./cpu_ram>`），以便为指定的令牌预取（这些 KV 缓存仍然保留在磁盘中）。" #: ../../source/kv_cache/storage_backends/local_storage.rst:121 msgid "Architecture Overview" -msgstr "" +msgstr "架构概述" #: ../../source/kv_cache/storage_backends/local_storage.rst:123 msgid "" "The following diagram shows the overall architecture of the Local Disk " "Backend:" -msgstr "" +msgstr "下图展示了本地磁盘后端的整体架构：" #: ../../source/kv_cache/storage_backends/local_storage.rst:171 msgid "**Key Components:**" -msgstr "" +msgstr "**关键组件：**" #: ../../source/kv_cache/storage_backends/local_storage.rst:173 msgid "" "**Metadata Dictionary**: Maps each ``CacheEngineKey`` to its disk " "metadata (file path, size, shape, dtype, pin status)" -msgstr "" +msgstr "**元数据字典**：将每个 ``CacheEngineKey`` 映射到其磁盘元数据（文件路径、大小、形状、数据类型、固定状态）" #: ../../source/kv_cache/storage_backends/local_storage.rst:174 msgid "" "**Cache Policy**: Configurable eviction policy (LRU, LFU, FIFO, or MRU) " "that tracks access patterns and decides which entries to evict when space" " is needed" -msgstr "" +msgstr "**缓存策略**：可配置的逐出策略（LRU、LFU、FIFO 或 MRU），跟踪访问模式并决定在需要空间时逐出哪些条目" #: ../../source/kv_cache/storage_backends/local_storage.rst:175 msgid "" "**LocalDiskWorker**: Async task executor with priority queue - prefetch " "tasks run first (priority 0), then deletes (priority 1), then saves " "(priority 2)" -msgstr "" +msgstr "**LocalDiskWorker**: 带有优先级队列的异步任务执行器 - 预取任务优先运行（优先级 0），然后是删除（优先级 1），最后是保存（优先级 2）" #: ../../source/kv_cache/storage_backends/local_storage.rst:176 msgid "" "**Local Disk**: Filesystem where KV cache chunks are stored as individual" " ``.pt`` files" -msgstr "" +msgstr "**本地磁盘**：KV Cache 块作为单独的 ``.pt`` 文件存储的文件系统" #: ../../source/kv_cache/storage_backends/local_storage.rst:180 msgid "Save Flow (PUT)" -msgstr "" +msgstr "保存流程 (PUT)" #: ../../source/kv_cache/storage_backends/local_storage.rst:199 msgid "Load Flow (GET)" -msgstr "" +msgstr "加载流程 (GET)" #: ../../source/kv_cache/storage_backends/local_storage.rst:221 msgid "Online Inference Example" -msgstr "" +msgstr "在线推理示例" #: ../../source/kv_cache/storage_backends/local_storage.rst:223 msgid "" "This example is almost identical to the :doc:`CPU RAM <./cpu_ram>` " "example." -msgstr "" +msgstr "这个示例与 :doc:`CPU RAM <./cpu_ram>` 示例几乎相同。" #: ../../source/kv_cache/storage_backends/local_storage.rst:225 msgid "Let's feel the TTFT (time to first token) differential!" -msgstr "" +msgstr "让我们感受一下 TTFT（首次令牌时间）差异！" #: ../../source/kv_cache/storage_backends/local_storage.rst:229 msgid "**Prerequisites:**" -msgstr "" +msgstr "**前提条件：**" #: ../../source/kv_cache/storage_backends/local_storage.rst:231 msgid "" "A Machine with at least one GPU. Adjust the max model length of your vllm" " instance depending on your GPU memory and the long context you want to " "use." -msgstr "" +msgstr "一台至少配备一块 GPU 的机器。根据您的显存和希望使用的长上下文调整 vllm 实例的最大模型长度。" #: ../../source/kv_cache/storage_backends/local_storage.rst:233 msgid "" "vllm and lmcache installed (:doc:`Installation Guide " "<../../getting_started/installation>`)" -msgstr "" +msgstr "安装了 vllm 和 LMCache (:doc:`安装指南 <../../getting_started/installation>`)" #: ../../source/kv_cache/storage_backends/local_storage.rst:235 msgid "Hugging Face access to ``meta-llama/Meta-Llama-3.1-8B-Instruct``" -msgstr "" +msgstr "Hugging Face 访问 ``meta-llama/Meta-Llama-3.1-8B-Instruct``" #: ../../source/kv_cache/storage_backends/local_storage.rst:241 msgid "A few packages:" -msgstr "" +msgstr "一些软件包：" #: ../../source/kv_cache/storage_backends/local_storage.rst:249 msgid "**Step 0. Set up a directory for this example:**" -msgstr "" +msgstr "**步骤 0. 为此示例设置一个目录：**" #: ../../source/kv_cache/storage_backends/local_storage.rst:256 msgid "**Step 1. Prepare a long context!**" -msgstr "" +msgstr "**步骤 1. 准备一个长上下文！**" #: ../../source/kv_cache/storage_backends/local_storage.rst:258 msgid "" "We want a context long enough that vllm's prefix caching will not be able" " to hold the KV caches in GPU memory and LMCache is necessary to keep KV " "caches in non-GPU memory:" -msgstr "" +msgstr "我们希望上下文足够长，以至于 vLLM 的前缀缓存无法将 KV 缓存保留在显存中，因此需要 LMCache 将 KV 缓存保留在非显存中：" #: ../../source/kv_cache/storage_backends/local_storage.rst:266 msgid "**Step 2. Start a vLLM server with Disk offloading enabled:**" -msgstr "" +msgstr "**步骤 2. 启动一个启用磁盘卸载的 vLLM 服务器：**" #: ../../source/kv_cache/storage_backends/local_storage.rst:268 msgid "" "*Generally, it is not recommended but we will disable CPU offloading to " "feel just the disk offloading latency.*" -msgstr "" +msgstr "*通常不推荐这样做，但我们将禁用 CPU 卸载，以便仅感受磁盘卸载的延迟。*" #: ../../source/kv_cache/storage_backends/local_storage.rst:270 msgid "Create a an lmcache configuration file called: ``disk-offload.yaml``" -msgstr "" +msgstr "创建一个名为 ``disk-offload.yaml`` 的 lmcache 配置文件" #: ../../source/kv_cache/storage_backends/local_storage.rst:272 msgid "Example ``config.yaml``:" -msgstr "" +msgstr "示例 ``config.yaml``：" #: ../../source/kv_cache/storage_backends/local_storage.rst:282 msgid "" "If you don't want to use a config file, uncomment the first five " "environment variables and then comment out the ``LMCACHE_CONFIG_FILE`` " "below:" -msgstr "" +msgstr "如果您不想使用配置文件，请取消注释前五个环境变量，然后注释掉下面的 ``LMCACHE_CONFIG_FILE``：" #: ../../source/kv_cache/storage_backends/local_storage.rst:299 msgid "" "``--kv-transfer-config``: This is the parameter that actually tells vLLM " "to use LMCache for KV cache offloading." -msgstr "" +msgstr "``--kv-transfer-config``: 这是实际告诉 vLLM 使用 LMCache 进行 KV Cache 卸载的参数。" #: ../../source/kv_cache/storage_backends/local_storage.rst:300 msgid "``kv_connector``: Specifies the LMCache connector for vLLM V1" -msgstr "" +msgstr "``kv_connector``: 指定 vLLM V1 的 LMCache 连接器" #: ../../source/kv_cache/storage_backends/local_storage.rst:301 msgid "" @@ -288,31 +288,31 @@ msgid "" "(important because we will run two queries and the first will " "produce/store a KV cache while the second will consume/load that KV " "cache)" -msgstr "" +msgstr "``kv_role``: 设置为 \\\"kv_both\\\" 以同时存储和加载 KV Cache（重要，因为我们将运行两个查询，第一个将生成/存储一个 KV Cache，而第二个将消费/加载该 KV Cache）" #: ../../source/kv_cache/storage_backends/local_storage.rst:304 msgid "**Step 3. Query TTFT improvements with LMCache:**" -msgstr "" +msgstr "**步骤 3. 使用 LMCache 查询 TTFT 改进：**" #: ../../source/kv_cache/storage_backends/local_storage.rst:306 msgid "" "Once the Open AI compatible server is running on default vllm port 8000, " "let's query it twice with the same long context!" -msgstr "" +msgstr "一旦 Open AI 兼容的服务器在默认的 vllm 端口 8000 上运行，让我们用相同的长上下文查询两次！" #: ../../source/kv_cache/storage_backends/local_storage.rst:308 msgid "Create a script called ``query-twice.py`` and paste the following code:" -msgstr "" +msgstr "创建一个名为 ``query-twice.py`` 的脚本，并粘贴以下代码：" #: ../../source/kv_cache/storage_backends/local_storage.rst:374 msgid "Then run:" -msgstr "" +msgstr "然后运行：" #: ../../source/kv_cache/storage_backends/local_storage.rst:380 msgid "" "Since we're in streaming mode, you'll be able to feel the TTFT " "differential in real time!" -msgstr "" +msgstr "由于我们处于流式模式，您将能够实时感受到 TTFT 差异！" #: ../../source/kv_cache/storage_backends/local_storage.rst:383 msgid "" @@ -321,36 +321,36 @@ msgid "" "RAM is checked before the disk by LMCache. In practice, the disk will be " "capable of storing a larger quantity of KV caches so the CPU RAM " "offloading will only be able to store a subset of the disk's KV caches." -msgstr "" +msgstr "请注意，如果我们启用 ``LMCACHE_LOCAL_CPU=True``，我们将仅使用来自 :doc:`CPU RAM <./cpu_ram>` 的相同示例，因为 LMCache 在检查磁盘之前会先检查 CPU RAM。实际上，磁盘能够存储更多的 KV 缓存，因此 CPU RAM 的卸载只能存储磁盘 KV 缓存的一个子集。" #: ../../source/kv_cache/storage_backends/local_storage.rst:389 msgid "**Example Output:**" -msgstr "" +msgstr "**示例输出：**" #: ../../source/kv_cache/storage_backends/local_storage.rst:415 msgid "TTFT Improvement: 6.166 seconds (42.6x faster)" -msgstr "" +msgstr "TTFT 改进：6.166 秒（快 42.6 倍）" #: ../../source/kv_cache/storage_backends/local_storage.rst:417 msgid "" "If you look at the logs of your vLLM server, you should see (the logs are" " truncated for cleanliness):" -msgstr "" +msgstr "如果你查看 vLLM 服务器的日志，你应该会看到（日志已为整洁而截断）：" #: ../../source/kv_cache/storage_backends/local_storage.rst:442 msgid "Check out your KV Cache in your SSD:" -msgstr "" +msgstr "查看您 SSD 中的 KV Cache：" #: ../../source/kv_cache/storage_backends/local_storage.rst:451 msgid "Tips:" -msgstr "" +msgstr "提示：" #: ../../source/kv_cache/storage_backends/local_storage.rst:453 msgid "" "If you want to run the ``query-twice.py`` script multiple times, you'll " "need to either restart the vLLM LMCache server or change the prefix of " "the context you pass in since you've already warmed LMCache." -msgstr "" +msgstr "如果您想多次运行 ``query-twice.py`` 脚本，您需要重启 vLLM LMCache 服务器或更改您传入的上下文前缀，因为您已经预热了 LMCache。" #: ../../source/kv_cache/storage_backends/local_storage.rst:455 msgid "" @@ -359,5 +359,5 @@ msgid "" "length and modify ``query-twice.py`` to use more of the long context. " "LMCache TTFT improvement becomes more pronounced as the context length " "increases!" -msgstr "" +msgstr "这里的最大模型长度是通过仅使用 23GB 显存的 L4 运行决定的。如果您有更多内存，可以增加最大模型长度并修改 ``query-twice.py`` 以使用更多的长上下文。随着上下文长度的增加，LMCache 的 TTFT 改进变得更加明显！" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/maru.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/maru.po index f7428e32fc2..97ac6c020ee 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/maru.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/maru.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache/storage_backends/maru.rst:2 msgid "Maru" -msgstr "" +msgstr "Maru" #: ../../source/kv_cache/storage_backends/maru.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/maru.rst:9 msgid "" @@ -33,171 +33,171 @@ msgid "" "cache storage engine built on CXL shared memory, designed for LLM " "inference scenarios where multiple instances need to share a KV cache " "with minimal latency." -msgstr "" +msgstr "`Maru `_ 是一个高性能的 KV Cache 存储引擎，基于 CXL 共享内存构建，旨在用于 LLM 推理场景，在这些场景中，多个实例需要以最小延迟共享 KV Cache。" #: ../../source/kv_cache/storage_backends/maru.rst:12 msgid "KV Cache Sharing: Without vs With Maru" -msgstr "" +msgstr "KV Cache 共享：没有 Maru 与有 Maru" #: ../../source/kv_cache/storage_backends/maru.rst:15 msgid "" "For architecture details, see the `Maru documentation `_." -msgstr "" +msgstr "有关架构的详细信息，请参阅 `Maru 文档 `_。" #: ../../source/kv_cache/storage_backends/maru.rst:18 msgid "Quick Start" -msgstr "" +msgstr "快速开始" #: ../../source/kv_cache/storage_backends/maru.rst:20 msgid "Install Maru:" -msgstr "" +msgstr "安装 Maru：" #: ../../source/kv_cache/storage_backends/maru.rst:28 msgid "" "This installs ``maru-server``, ``maru-resourced``, and the ``maru`` " "Python package." -msgstr "" +msgstr "这将安装 ``maru-server``、``maru-resourced`` 和 ``maru`` Python 包。" #: ../../source/kv_cache/storage_backends/maru.rst:31 msgid "Deploy Model With Maru" -msgstr "" +msgstr "使用 Maru 部署模型" #: ../../source/kv_cache/storage_backends/maru.rst:33 msgid "" "**Prerequisites:** CXL device (``/dev/dax*``), Python 3.12+, vLLM and " "LMCache installed." -msgstr "" +msgstr "**前提条件：** CXL 设备（``/dev/dax*``），安装 Python 3.12+、vLLM 和 LMCache。" #: ../../source/kv_cache/storage_backends/maru.rst:35 msgid "**1. Start the Maru Server**" -msgstr "" +msgstr "**1. 启动 Maru 服务器**" #: ../../source/kv_cache/storage_backends/maru.rst:41 msgid "**2. Create configuration file** (``maru-config.yaml``):" -msgstr "" +msgstr "**2. 创建配置文件** (``maru-config.yaml``):" #: ../../source/kv_cache/storage_backends/maru.rst:54 msgid "**3. Start vLLM with Maru**" -msgstr "" +msgstr "**3. 使用 Maru 启动 vLLM**" #: ../../source/kv_cache/storage_backends/maru.rst:66 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/kv_cache/storage_backends/maru.rst:68 msgid "**LMCache Parameters:**" -msgstr "" +msgstr "**LMCache 参数:**" #: ../../source/kv_cache/storage_backends/maru.rst:74 #: ../../source/kv_cache/storage_backends/maru.rst:90 msgid "Parameter" -msgstr "" +msgstr "参数" #: ../../source/kv_cache/storage_backends/maru.rst:75 #: ../../source/kv_cache/storage_backends/maru.rst:91 msgid "Default" -msgstr "" +msgstr "默认" #: ../../source/kv_cache/storage_backends/maru.rst:76 #: ../../source/kv_cache/storage_backends/maru.rst:92 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/kv_cache/storage_backends/maru.rst:77 msgid "``maru_path``" -msgstr "" +msgstr "``maru_path``" #: ../../source/kv_cache/storage_backends/maru.rst:78 msgid "Required" -msgstr "" +msgstr "必需" #: ../../source/kv_cache/storage_backends/maru.rst:79 msgid "Maru server URL (format: ``maru://host:port``)" -msgstr "" +msgstr "Maru 服务器 URL（格式：``maru://host:port``）" #: ../../source/kv_cache/storage_backends/maru.rst:80 msgid "``maru_pool_size``" -msgstr "" +msgstr "``maru_pool_size``" #: ../../source/kv_cache/storage_backends/maru.rst:81 msgid "``4.0``" -msgstr "" +msgstr "``4.0``" #: ../../source/kv_cache/storage_backends/maru.rst:82 msgid "CXL memory pool size per instance in GB (e.g., ``4``, ``0.5``)" -msgstr "" +msgstr "每个实例的 CXL 内存池大小（单位：GB，例如 ``4``、``0.5``）" #: ../../source/kv_cache/storage_backends/maru.rst:84 msgid "**Advanced Parameters (via extra_config):**" -msgstr "" +msgstr "**高级参数（通过 extra_config）：**" #: ../../source/kv_cache/storage_backends/maru.rst:93 msgid "``maru_instance_id``" -msgstr "" +msgstr "``maru_instance_id``" #: ../../source/kv_cache/storage_backends/maru.rst:94 msgid "auto UUID" -msgstr "" +msgstr "自动 UUID" #: ../../source/kv_cache/storage_backends/maru.rst:95 msgid "Unique client instance identifier" -msgstr "" +msgstr "唯一客户端实例标识符" #: ../../source/kv_cache/storage_backends/maru.rst:96 msgid "``maru_timeout_ms``" -msgstr "" +msgstr "``maru_timeout_ms``" #: ../../source/kv_cache/storage_backends/maru.rst:97 msgid "5000" -msgstr "" +msgstr "5000" #: ../../source/kv_cache/storage_backends/maru.rst:98 msgid "ZMQ RPC socket timeout in milliseconds" -msgstr "" +msgstr "ZMQ RPC 套接字超时（毫秒）" #: ../../source/kv_cache/storage_backends/maru.rst:99 msgid "``maru_use_async_rpc``" -msgstr "" +msgstr "``maru_use_async_rpc``" #: ../../source/kv_cache/storage_backends/maru.rst:100 #: ../../source/kv_cache/storage_backends/maru.rst:106 msgid "true" -msgstr "" +msgstr "真" #: ../../source/kv_cache/storage_backends/maru.rst:101 msgid "Async DEALER-ROUTER RPC (``false`` for synchronous REQ-REP)" -msgstr "" +msgstr "异步 DEALER-ROUTER RPC (``false`` 表示同步 REQ-REP)" #: ../../source/kv_cache/storage_backends/maru.rst:102 msgid "``maru_max_inflight``" -msgstr "" +msgstr "``maru_max_inflight``" #: ../../source/kv_cache/storage_backends/maru.rst:103 msgid "64" -msgstr "" +msgstr "64" #: ../../source/kv_cache/storage_backends/maru.rst:104 msgid "Max concurrent async RPC requests" -msgstr "" +msgstr "最大并发异步 RPC 请求数" #: ../../source/kv_cache/storage_backends/maru.rst:105 msgid "``maru_eager_map``" -msgstr "" +msgstr "``maru_eager_map``" #: ../../source/kv_cache/storage_backends/maru.rst:107 msgid "Pre-map all shared regions on connect" -msgstr "" +msgstr "在连接时预先映射所有共享区域" #: ../../source/kv_cache/storage_backends/maru.rst:110 msgid "Additional Resources" -msgstr "" +msgstr "附加资源" #: ../../source/kv_cache/storage_backends/maru.rst:112 msgid "`Maru GitHub Repository `_" -msgstr "" +msgstr "`Maru GitHub Repository `_" #: ../../source/kv_cache/storage_backends/maru.rst:113 msgid "`Maru Documentation `_" -msgstr "" +msgstr "`Maru 文档 `_" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mock.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mock.po index d17a90bb437..8a27384363e 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mock.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mock.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/kv_cache/storage_backends/mock.rst:2 msgid "Mock" -msgstr "" +msgstr "模拟" #: ../../source/kv_cache/storage_backends/mock.rst:4 msgid "" @@ -29,48 +29,48 @@ msgid "" "the peeking latency, read throughput, and write throughput inside of the " "remote url. It will create copies of your KV cache in unmanaged local " "RAM." -msgstr "" +msgstr "LMCache 提供了一个模拟远程连接器，允许您手动设置远程 URL 中的预查看延迟、读取吞吐量和写入吞吐量。它将在未管理的本地 RAM 中创建您的 KV Cache 的副本。" #: ../../source/kv_cache/storage_backends/mock.rst:7 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/kv_cache/storage_backends/mock.rst:9 msgid "" "Create a configuration file (e.g., ``mock.yaml``) with the following " "content:" -msgstr "" +msgstr "创建一个配置文件（例如，``mock.yaml``），内容如下：" #: ../../source/kv_cache/storage_backends/mock.rst:18 msgid "" "The ``remote_url`` format is " "``mock://SIZE/?peeking_latency=LATENCY&read_throughput=READ_GBPS&write_throughput=WRITE_GBPS``" " where:" -msgstr "" +msgstr "``remote_url`` 的格式为 ``mock://SIZE/?peeking_latency=LATENCY&read_throughput=READ_GBPS&write_throughput=WRITE_GBPS``，其中：" #: ../../source/kv_cache/storage_backends/mock.rst:20 msgid "``SIZE``: Maximum storage size" -msgstr "" +msgstr "``SIZE``: 最大存储大小" #: ../../source/kv_cache/storage_backends/mock.rst:21 msgid "``peeking_latency``: Latency for peeking operations (in milliseconds)" -msgstr "" +msgstr "``peeking_latency``: 预取操作的延迟（以毫秒为单位）" #: ../../source/kv_cache/storage_backends/mock.rst:22 msgid "``read_throughput``: Read throughput (in GB/s)" -msgstr "" +msgstr "``read_throughput``: 读取吞吐量（以 GB/s 为单位）" #: ../../source/kv_cache/storage_backends/mock.rst:23 msgid "``write_throughput``: Write throughput (in GB/s)" -msgstr "" +msgstr "``write_throughput``: 写入吞吐量 (以 GB/s 为单位)" #: ../../source/kv_cache/storage_backends/mock.rst:26 msgid "Usage" -msgstr "" +msgstr "使用方法" #: ../../source/kv_cache/storage_backends/mock.rst:28 msgid "Deploy a serving engine with the mock remote backend:" -msgstr "" +msgstr "使用模拟远程后端部署服务引擎：" #: ../../source/kv_cache/storage_backends/mock.rst:37 msgid "" @@ -78,27 +78,27 @@ msgid "" "meaningless) logs on the second query to confirm that the throughput is " "slightly lower than 2 GB/s (the CPU <-> GPU allocation/transfer also has " "overhead)." -msgstr "" +msgstr "检查第二个查询的检索（存储是异步的，因此吞吐量没有意义）日志，以确认吞吐量略低于 2 GB/s（CPU <-> GPU 分配/传输也有开销）。" #: ../../source/kv_cache/storage_backends/mock.rst:40 msgid "Example Query" -msgstr "" +msgstr "示例查询" #: ../../source/kv_cache/storage_backends/mock.rst:42 msgid "Send a test request:" -msgstr "" +msgstr "发送测试请求：" #: ../../source/kv_cache/storage_backends/mock.rst:55 msgid "Expected Logs" -msgstr "" +msgstr "预期日志" #: ../../source/kv_cache/storage_backends/mock.rst:57 msgid "You should see logs similar to the following:" -msgstr "" +msgstr "您应该看到类似于以下的日志：" #: ../../source/kv_cache/storage_backends/mock.rst:64 msgid "" "The logs confirm that the throughput is slightly lower than 2 GB/s due to" " CPU <-> GPU allocation/transfer overhead." -msgstr "" +msgstr "日志确认吞吐量略低于 2 GB/s，原因是 CPU <-> GPU 分配/传输开销。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mooncake.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mooncake.po index bbda73c95c3..1aae11e7ba9 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mooncake.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mooncake.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache/storage_backends/mooncake.rst:2 msgid "Mooncake" -msgstr "" +msgstr "Mooncake" #: ../../source/kv_cache/storage_backends/mooncake.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/mooncake.rst:9 msgid "" @@ -34,204 +34,204 @@ msgid "" "inference scenarios. The system creates a distributed memory pool by " "aggregating memory space contributed by various client nodes, enabling " "efficient resource utilization across clusters." -msgstr "" +msgstr "`Mooncake `_ 是一个开源的分布式 KV Cache 存储系统，专为 LLM 推理场景而设计。该系统通过聚合多个客户端节点贡献的内存空间，创建一个分布式内存池，从而实现跨集群的高效资源利用。" #: ../../source/kv_cache/storage_backends/mooncake.rst:12 msgid "" "By pooling underutilized DRAM and SSD resources from multiple nodes, the " "system forms a unified distributed storage service that maximizes " "resource efficiency." -msgstr "" +msgstr "通过整合多个节点下的未充分利用的 DRAM 和 SSD 资源，该系统形成了一个统一的分布式存储服务，最大化资源效率。" #: ../../source/kv_cache/storage_backends/mooncake.rst:14 msgid "Mooncake Architecture Diagram" -msgstr "" +msgstr "Mooncake 架构图" #: ../../source/kv_cache/storage_backends/mooncake.rst:18 msgid "Key Features" -msgstr "" +msgstr "关键特性" #: ../../source/kv_cache/storage_backends/mooncake.rst:20 msgid "" "**Distributed memory pooling**: Aggregates memory contributions from " "multiple client nodes into a unified storage pool" -msgstr "" +msgstr "**分布式内存池**：将多个客户端节点的内存贡献聚合到一个统一的存储池中" #: ../../source/kv_cache/storage_backends/mooncake.rst:21 msgid "" "**High bandwidth utilization**: Supports striping and parallel I/O " "transfer of large objects, fully utilizing multi-NIC aggregated bandwidth" -msgstr "" +msgstr "**高带宽利用率**：支持大对象的条带化和并行 I/O 传输，充分利用多 NIC 聚合带宽" #: ../../source/kv_cache/storage_backends/mooncake.rst:22 msgid "" "**RDMA optimization**: Built on Transfer Engine with support for TCP, " "RDMA (InfiniBand/RoCEv2/eRDMA/NVIDIA GPUDirect)" -msgstr "" +msgstr "**RDMA 优化**：基于传输引擎构建，支持 TCP、RDMA（InfiniBand/RoCEv2/eRDMA/NVIDIA GPUDirect）" #: ../../source/kv_cache/storage_backends/mooncake.rst:23 msgid "" "**Dynamic resource scaling**: Supports dynamically adding and removing " "nodes for elastic resource management" -msgstr "" +msgstr "**动态资源扩展**：支持动态添加和移除节点以实现弹性资源管理" #: ../../source/kv_cache/storage_backends/mooncake.rst:25 msgid "" "For detailed architecture information, see the `Mooncake Architecture " "Guide `_." -msgstr "" +msgstr "有关详细的架构信息，请参阅 `Mooncake Architecture Guide `_。" #: ../../source/kv_cache/storage_backends/mooncake.rst:28 msgid "Quick Start" -msgstr "" +msgstr "快速开始" #: ../../source/kv_cache/storage_backends/mooncake.rst:30 msgid "Install Mooncake via pip:" -msgstr "" +msgstr "通过 pip 安装 Mooncake：" #: ../../source/kv_cache/storage_backends/mooncake.rst:36 msgid "This package includes all necessary components:" -msgstr "" +msgstr "此软件包包含所有必要的组件：" #: ../../source/kv_cache/storage_backends/mooncake.rst:38 msgid "" "``mooncake_master``: Master service that manages cluster metadata and " "coordinates distributed storage operations" -msgstr "" +msgstr "``mooncake_master``：管理集群元数据并协调分布式存储操作的主服务" #: ../../source/kv_cache/storage_backends/mooncake.rst:39 msgid "" "``mooncake_http_metadata_server``: HTTP-based metadata server used by the" " underlying transfer engine for connection establishment" -msgstr "" +msgstr "``mooncake_http_metadata_server``: 基于 HTTP 的元数据服务器，用于底层传输引擎的连接建立" #: ../../source/kv_cache/storage_backends/mooncake.rst:40 msgid "Mooncake Python bindings" -msgstr "" +msgstr "Mooncake Python 绑定" #: ../../source/kv_cache/storage_backends/mooncake.rst:42 msgid "" "For production deployments or custom builds, see the `Build Instructions " "`_." -msgstr "" +msgstr "有关生产部署或自定义构建，请参阅 `构建说明 `_。" #: ../../source/kv_cache/storage_backends/mooncake.rst:45 msgid "Setup and Deployment" -msgstr "" +msgstr "设置和部署" #: ../../source/kv_cache/storage_backends/mooncake.rst:47 msgid "**Prerequisites:**" -msgstr "" +msgstr "**前提条件：**" #: ../../source/kv_cache/storage_backends/mooncake.rst:49 msgid "Machine with at least one GPU for vLLM inference" -msgstr "" +msgstr "具有至少一个 GPU 的机器用于 vLLM 推理" #: ../../source/kv_cache/storage_backends/mooncake.rst:50 msgid "RDMA-capable network hardware and drivers (recommended) or TCP network" -msgstr "" +msgstr "支持 RDMA 的网络硬件和驱动程序（推荐）或 TCP 网络" #: ../../source/kv_cache/storage_backends/mooncake.rst:51 msgid "Python 3.8+ with pip" -msgstr "" +msgstr "Python 3.8+ 和 pip" #: ../../source/kv_cache/storage_backends/mooncake.rst:52 msgid "vLLM and LMCache installed" -msgstr "" +msgstr "已安装 vLLM 和 LMCache" #: ../../source/kv_cache/storage_backends/mooncake.rst:54 msgid "**Step 1: Start Infrastructure Services**" -msgstr "" +msgstr "**步骤 1：启动基础设施服务**" #: ../../source/kv_cache/storage_backends/mooncake.rst:56 msgid "Start the Mooncake master service (with built‑in HTTP metadata server):" -msgstr "" +msgstr "启动 Mooncake 主服务（带内置 HTTP 元数据服务器）：" #: ../../source/kv_cache/storage_backends/mooncake.rst:64 msgid "Expected output:" -msgstr "" +msgstr "预期输出：" #: ../../source/kv_cache/storage_backends/mooncake.rst:72 msgid "**Step 2: Create Configuration File**" -msgstr "" +msgstr "**步骤 2：创建配置文件**" #: ../../source/kv_cache/storage_backends/mooncake.rst:74 msgid "Create your ``mooncake-config.yaml``:" -msgstr "" +msgstr "创建您的 ``mooncake-config.yaml``：" #: ../../source/kv_cache/storage_backends/mooncake.rst:98 msgid "**Step 3: Start vLLM with Mooncake**" -msgstr "" +msgstr "**步骤 3：使用 Mooncake 启动 vLLM**" #: ../../source/kv_cache/storage_backends/mooncake.rst:111 msgid "**Step 4: Verify the Setup**" -msgstr "" +msgstr "**步骤 4：验证设置**" #: ../../source/kv_cache/storage_backends/mooncake.rst:113 msgid "Test the integration with a sample request:" -msgstr "" +msgstr "使用示例请求测试集成：" #: ../../source/kv_cache/storage_backends/mooncake.rst:126 msgid "**Debugging Tips:**" -msgstr "" +msgstr "**调试提示：**" #: ../../source/kv_cache/storage_backends/mooncake.rst:128 msgid "**Enable verbose logging:**" -msgstr "" +msgstr "**启用详细日志记录：**" #: ../../source/kv_cache/storage_backends/mooncake.rst:134 msgid "**Check service status:**" -msgstr "" +msgstr "**检查服务状态：**" #: ../../source/kv_cache/storage_backends/mooncake.rst:142 msgid "**Monitor metrics:**" -msgstr "" +msgstr "**监控指标：**" #: ../../source/kv_cache/storage_backends/mooncake.rst:144 msgid "" "Access metrics at ``http://localhost:9003`` when master service is " "running." -msgstr "" +msgstr "当主服务运行时，可以在 ``http://localhost:9003`` 访问指标。" #: ../../source/kv_cache/storage_backends/mooncake.rst:147 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/kv_cache/storage_backends/mooncake.rst:149 msgid "**LMCache Parameters:**" -msgstr "" +msgstr "**LMCache 参数：**" #: ../../source/kv_cache/storage_backends/mooncake.rst:155 #: ../../source/kv_cache/storage_backends/mooncake.rst:186 msgid "Parameter" -msgstr "" +msgstr "参数" #: ../../source/kv_cache/storage_backends/mooncake.rst:156 #: ../../source/kv_cache/storage_backends/mooncake.rst:187 msgid "Default" -msgstr "" +msgstr "默认" #: ../../source/kv_cache/storage_backends/mooncake.rst:157 #: ../../source/kv_cache/storage_backends/mooncake.rst:188 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/kv_cache/storage_backends/mooncake.rst:158 msgid "``chunk_size``" -msgstr "" +msgstr "``chunk_size``" #: ../../source/kv_cache/storage_backends/mooncake.rst:159 msgid "256" -msgstr "" +msgstr "256" #: ../../source/kv_cache/storage_backends/mooncake.rst:160 msgid "Number of tokens per KV chunk" -msgstr "" +msgstr "每个 KV 块的令牌数量" #: ../../source/kv_cache/storage_backends/mooncake.rst:161 msgid "``remote_url``" -msgstr "" +msgstr "``remote_url``" #: ../../source/kv_cache/storage_backends/mooncake.rst:162 #: ../../source/kv_cache/storage_backends/mooncake.rst:171 @@ -239,124 +239,124 @@ msgstr "" #: ../../source/kv_cache/storage_backends/mooncake.rst:193 #: ../../source/kv_cache/storage_backends/mooncake.rst:196 msgid "Required" -msgstr "" +msgstr "必需" #: ../../source/kv_cache/storage_backends/mooncake.rst:163 msgid "Mooncake store connection URL (format: ``mooncakestore://host:port/``)." -msgstr "" +msgstr "Mooncake 存储连接 URL（格式：``mooncakestore://host:port/``）。" #: ../../source/kv_cache/storage_backends/mooncake.rst:164 msgid "``remote_serde``" -msgstr "" +msgstr "``remote_serde``" #: ../../source/kv_cache/storage_backends/mooncake.rst:165 msgid "\"naive\"" -msgstr "" +msgstr "“简单”" #: ../../source/kv_cache/storage_backends/mooncake.rst:166 msgid "Serialization method for remote storage" -msgstr "" +msgstr "远程存储的序列化方法" #: ../../source/kv_cache/storage_backends/mooncake.rst:167 msgid "``local_cpu``" -msgstr "" +msgstr "``local_cpu``" #: ../../source/kv_cache/storage_backends/mooncake.rst:168 #: ../../source/kv_cache/storage_backends/mooncake.rst:220 #: ../../source/kv_cache/storage_backends/mooncake.rst:223 #: ../../source/kv_cache/storage_backends/mooncake.rst:226 msgid "False" -msgstr "" +msgstr "假" #: ../../source/kv_cache/storage_backends/mooncake.rst:169 msgid "" "Enable/disable local CPU caching (set to False for pure Mooncake " "evaluation)" -msgstr "" +msgstr "启用/禁用本地 CPU 缓存（设置为 False 以进行纯 Mooncake 评估）" #: ../../source/kv_cache/storage_backends/mooncake.rst:170 msgid "``max_local_cpu_size``" -msgstr "" +msgstr "``max_local_cpu_size``" #: ../../source/kv_cache/storage_backends/mooncake.rst:172 msgid "Maximum local CPU cache size in GB (required even when local_cpu is False)" -msgstr "" +msgstr "本地 CPU 缓存的最大大小（以 GB 为单位）（即使 local_cpu 为 False 也需要）" #: ../../source/kv_cache/storage_backends/mooncake.rst:173 msgid "``numa_mode``" -msgstr "" +msgstr "``numa_mode``" #: ../../source/kv_cache/storage_backends/mooncake.rst:174 msgid "\"auto\"" -msgstr "" +msgstr "自动" #: ../../source/kv_cache/storage_backends/mooncake.rst:175 msgid "" "NUMA binding mode. \"auto\" is recommended on multi‑NIC/multi‑NUMA " "systems to reduce tail latency." -msgstr "" +msgstr "NUMA 绑定模式。在多 NIC/多 NUMA 系统上，推荐使用 \\\"auto\\\" 以减少尾部延迟。" #: ../../source/kv_cache/storage_backends/mooncake.rst:176 msgid "``pre_caching_hash_algorithm``" -msgstr "" +msgstr "``pre_caching_hash_algorithm``" #: ../../source/kv_cache/storage_backends/mooncake.rst:177 msgid "\"sha256_cbor_64bit\"" -msgstr "" +msgstr "sha256_cbor_64bit" #: ../../source/kv_cache/storage_backends/mooncake.rst:178 msgid "" "Hash used for pre-caching keying. For cross‑process consistency, fix " "``PYTHONHASHSEED`` (e.g., export ``PYTHONHASHSEED=0``)." -msgstr "" +msgstr "用于预缓存键的哈希。为了跨进程的一致性，固定 ``PYTHONHASHSEED``（例如，导出 ``PYTHONHASHSEED=0``）。" #: ../../source/kv_cache/storage_backends/mooncake.rst:180 msgid "**Mooncake Parameters (via extra_config):**" -msgstr "" +msgstr "**Mooncake 参数（通过 extra_config）：**" #: ../../source/kv_cache/storage_backends/mooncake.rst:189 msgid "``local_hostname``" -msgstr "" +msgstr "``local_hostname``" #: ../../source/kv_cache/storage_backends/mooncake.rst:191 msgid "Hostname/IP of the local node for Mooncake client identification" -msgstr "" +msgstr "Mooncake 客户端识别的本地节点的主机名/IP" #: ../../source/kv_cache/storage_backends/mooncake.rst:192 msgid "``metadata_server``" -msgstr "" +msgstr "``metadata_server``" #: ../../source/kv_cache/storage_backends/mooncake.rst:194 msgid "" "HTTP metadata server address. When starting master with " "``--enable_http_metadata_server=1``, it exposes this endpoint." -msgstr "" +msgstr "HTTP 元数据服务器地址。当使用 ``--enable_http_metadata_server=1`` 启动主服务器时，它会公开此端点。" #: ../../source/kv_cache/storage_backends/mooncake.rst:195 msgid "``master_server_address``" -msgstr "" +msgstr "``master_server_address``" #: ../../source/kv_cache/storage_backends/mooncake.rst:197 msgid "Mooncake master service address (host:port format)" -msgstr "" +msgstr "Mooncake 主服务地址（主机:端口格式）" #: ../../source/kv_cache/storage_backends/mooncake.rst:198 msgid "``protocol``" -msgstr "" +msgstr "``protocol``" #: ../../source/kv_cache/storage_backends/mooncake.rst:199 msgid "\"rdma\"" -msgstr "" +msgstr "rdma" #: ../../source/kv_cache/storage_backends/mooncake.rst:200 msgid "" "Communication protocol (\"rdma\" for high performance; \"tcp\" for " "compatibility)" -msgstr "" +msgstr "通信协议（“rdma”用于高性能；“tcp”用于兼容性）" #: ../../source/kv_cache/storage_backends/mooncake.rst:201 msgid "``device_name``" -msgstr "" +msgstr "``device_name``" #: ../../source/kv_cache/storage_backends/mooncake.rst:202 #: ../../source/kv_cache/storage_backends/mooncake.rst:217 @@ -367,29 +367,29 @@ msgstr "" msgid "" "RDMA device specification (e.g., \"erdma_0,erdma_1\" or " "\"mlx5_0,mlx5_1\"). Leave empty for autodetection in most setups." -msgstr "" +msgstr "RDMA 设备规格（例如，\\\"erdma_0,erdma_1\\\" 或 \\\"mlx5_0,mlx5_1\\\"）。在大多数设置中留空以进行自动检测。" #: ../../source/kv_cache/storage_backends/mooncake.rst:204 msgid "``global_segment_size``" -msgstr "" +msgstr "``global_segment_size``" #: ../../source/kv_cache/storage_backends/mooncake.rst:205 msgid "21474836480" -msgstr "" +msgstr "21474836480" #: ../../source/kv_cache/storage_backends/mooncake.rst:206 msgid "" "**Memory size contributed by each vLLM worker** in bytes (e.g., 20 GiB " "recommended)" -msgstr "" +msgstr "**每个 vLLM 工作线程贡献的内存大小**（以字节为单位，例如，推荐 20 GiB）" #: ../../source/kv_cache/storage_backends/mooncake.rst:207 msgid "``local_buffer_size``" -msgstr "" +msgstr "``local_buffer_size``" #: ../../source/kv_cache/storage_backends/mooncake.rst:208 msgid "0" -msgstr "" +msgstr "0" #: ../../source/kv_cache/storage_backends/mooncake.rst:209 msgid "" @@ -402,55 +402,55 @@ msgid "" "registering LMCache's large CPU buffer can fail on constrained devices. " "In those cases, consider enabling ``save_chunk_meta: True`` and sizing " "``local_buffer_size`` instead." -msgstr "" +msgstr "Mooncake 使用的本地缓冲区大小（以字节为单位）。行为取决于 ``save_chunk_meta``：- 当 ``save_chunk_meta: False``（推荐）时，LMCache 使用其本地 CPU 后端进行零拷贝 RDMA，因此 Mooncake 的 ``local_buffer_size`` 可以为 ``0``。- 当 ``save_chunk_meta: True`` 时，Mooncake 使用其自己的本地缓冲区；将其设置为合适的值（例如，几个 GiB）。- 注意：某些 RDMA NIC 有内存注册限制；在受限设备上注册 LMCache 的大 CPU 缓冲区可能会失败。在这些情况下，请考虑启用 ``save_chunk_meta: True`` 并调整 ``local_buffer_size`` 的大小。" #: ../../source/kv_cache/storage_backends/mooncake.rst:213 msgid "``transfer_timeout``" -msgstr "" +msgstr "``transfer_timeout``" #: ../../source/kv_cache/storage_backends/mooncake.rst:214 msgid "1" -msgstr "" +msgstr "1" #: ../../source/kv_cache/storage_backends/mooncake.rst:215 msgid "Timeout for transfer operations in seconds" -msgstr "" +msgstr "传输操作的超时时间（以秒为单位）" #: ../../source/kv_cache/storage_backends/mooncake.rst:216 msgid "``storage_root_dir``" -msgstr "" +msgstr "``storage_root_dir``" #: ../../source/kv_cache/storage_backends/mooncake.rst:218 msgid "The root directory for persistence (e.g., \"/mnt/mooncake\")" -msgstr "" +msgstr "持久化的根目录（例如，\\\"/mnt/mooncake\\\"）" #: ../../source/kv_cache/storage_backends/mooncake.rst:219 msgid "``save_chunk_meta``" -msgstr "" +msgstr "``save_chunk_meta``" #: ../../source/kv_cache/storage_backends/mooncake.rst:221 msgid "" "Whether to save chunk metadata alongside data. Set to ``False`` to enable" " the optimized zero‑copy path in LMCache." -msgstr "" +msgstr "是否在数据旁边保存块元数据。设置为 ``False`` 以启用 LMCache 中的优化零拷贝路径。" #: ../../source/kv_cache/storage_backends/mooncake.rst:222 msgid "``use_exists_sync``" -msgstr "" +msgstr "``use_exists_sync``" #: ../../source/kv_cache/storage_backends/mooncake.rst:224 msgid "" "Use synchronous existence checks to avoid async scheduling overhead in " "hot paths." -msgstr "" +msgstr "使用同步存在性检查以避免热路径中的异步调度开销。" #: ../../source/kv_cache/storage_backends/mooncake.rst:225 msgid "``mooncake_prefer_local_alloc``" -msgstr "" +msgstr "``mooncake_prefer_local_alloc``" #: ../../source/kv_cache/storage_backends/mooncake.rst:227 msgid "Prefer allocating on the local segment when possible." -msgstr "" +msgstr "在可能的情况下，优先在本地段上分配。" #: ../../source/kv_cache/storage_backends/mooncake.rst:230 msgid "" @@ -458,68 +458,68 @@ msgid "" "of memory each vLLM worker contributes to the distributed memory pool. " "The total cluster memory available for KV cache storage will be: " "``number_of_vllm_workers × global_segment_size``." -msgstr "" +msgstr "**理解 global_segment_size**：该参数定义了每个 vLLM 工作线程为分布式内存池贡献的内存量。KV Cache 存储可用的总集群内存将为：``number_of_vllm_workers × global_segment_size``。" #: ../../source/kv_cache/storage_backends/mooncake.rst:233 msgid "" "Adjust this value based on your available system memory and expected " "cache requirements." -msgstr "" +msgstr "根据您的可用系统内存和预期的缓存需求调整此值。" #: ../../source/kv_cache/storage_backends/mooncake.rst:236 msgid "" "If you consistently get misses (no Mooncake hits), ensure all processes " "use the same hashing seed: ``export PYTHONHASHSEED=0``. This keeps " "pre‑caching keys consistent across processes." -msgstr "" +msgstr "如果您持续遇到未命中（没有 Mooncake 命中），请确保所有进程使用相同的哈希种子：``export PYTHONHASHSEED=0``。这可以保持跨进程的预缓存键一致。" #: ../../source/kv_cache/storage_backends/mooncake.rst:239 msgid "" "RDMA device(s) usually do not need to be specified; leaving " "``device_name`` empty works for most deployments." -msgstr "" +msgstr "RDMA 设备通常不需要指定；将 ``device_name`` 留空适用于大多数部署。" #: ../../source/kv_cache/storage_backends/mooncake.rst:242 msgid "Additional Resources" -msgstr "" +msgstr "附加资源" #: ../../source/kv_cache/storage_backends/mooncake.rst:244 msgid "" "`Mooncake Store Architecture `_" -msgstr "" +msgstr "`Mooncake Store Architecture `_" #: ../../source/kv_cache/storage_backends/mooncake.rst:245 msgid "" "`Mooncake Store Deployment Guide `_" -msgstr "" +msgstr "`Mooncake Store 部署指南 `_" #: ../../source/kv_cache/storage_backends/mooncake.rst:246 msgid "" "`Mooncake Store Python API Reference `_" -msgstr "" +msgstr "`Mooncake Store Python API 参考 `_" #: ../../source/kv_cache/storage_backends/mooncake.rst:247 msgid "" "`Transfer Engine Documentation `_" -msgstr "" +msgstr "`传输引擎文档 `_" #: ../../source/kv_cache/storage_backends/mooncake.rst:248 msgid "" "`Build Instructions `_" -msgstr "" +msgstr "`构建说明 `_" #: ../../source/kv_cache/storage_backends/mooncake.rst:249 msgid "`GitHub Repository `_" -msgstr "" +msgstr "`GitHub 仓库 `_" #: ../../source/kv_cache/storage_backends/mooncake.rst:250 msgid "" "`LMCache Integration Guide `_" -msgstr "" +msgstr "`LMCache 集成指南 `_" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/nixl.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/nixl.po index 9c335bfdb9c..73caf1be4be 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/nixl.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/nixl.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache/storage_backends/nixl.rst:3 msgid "Nixl" -msgstr "" +msgstr "Nixl" #: ../../source/kv_cache/storage_backends/nixl.rst:8 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/nixl.rst:10 msgid "" @@ -35,69 +35,69 @@ msgid "" "and GPU) and storage through a modular plug-in architecture, enabling " "efficient data transfer and coordination between different components of " "the inference pipeline." -msgstr "" +msgstr "NIXL（NVIDIA 推理传输库）是一个高性能库，旨在加速 AI 推理框架中的点对点通信。它通过模块化插件架构提供对各种类型内存（CPU 和 GPU）和存储的抽象，使得推理管道中不同组件之间的数据传输和协调更加高效。" #: ../../source/kv_cache/storage_backends/nixl.rst:12 msgid "" "LMCache supports using NIXL as a storage backend, allowing using NIXL to " "save either GPU or CPU memory into storage." -msgstr "" +msgstr "LMCache 支持将 NIXL 用作存储后端，允许将显存或 CPU 内存保存到存储中。" #: ../../source/kv_cache/storage_backends/nixl.rst:15 msgid "Prerequisites" -msgstr "" +msgstr "前提条件" #: ../../source/kv_cache/storage_backends/nixl.rst:17 msgid "**LMCache**: Install with ``pip install lmcache``" -msgstr "" +msgstr "**LMCache**: 使用 ``pip install lmcache`` 安装" #: ../../source/kv_cache/storage_backends/nixl.rst:18 msgid "" "**NIXL**: Install from `NIXL GitHub repository `_" -msgstr "" +msgstr "**NIXL**: 从 `NIXL GitHub repository `_ 安装" #: ../../source/kv_cache/storage_backends/nixl.rst:19 msgid "" "**Model Access**: Valid Hugging Face token (HF_TOKEN) for Llama 3.1 8B " "Instruct" -msgstr "" +msgstr "**模型访问**：有效的 Hugging Face 令牌 (HF_TOKEN) 用于 Llama 3.1 8B Instruct" #: ../../source/kv_cache/storage_backends/nixl.rst:22 msgid "Ways to configure LMCache NIXL Offloading" -msgstr "" +msgstr "配置 LMCache NIXL 卸载的方法" #: ../../source/kv_cache/storage_backends/nixl.rst:24 msgid "**Configuration File**:" -msgstr "" +msgstr "**配置文件**：" #: ../../source/kv_cache/storage_backends/nixl.rst:26 msgid "Passed in through ``LMCACHE_CONFIG_FILE=lmcache-config.yaml``" -msgstr "" +msgstr "通过 ``LMCACHE_CONFIG_FILE=lmcache-config.yaml`` 传入" #: ../../source/kv_cache/storage_backends/nixl.rst:28 msgid "Example ``lmcache-config.yaml`` for POSIX backend:" -msgstr "" +msgstr "示例 ``lmcache-config.yaml`` 用于 POSIX 后端：" #: ../../source/kv_cache/storage_backends/nixl.rst:42 msgid "Key settings:" -msgstr "" +msgstr "关键设置：" #: ../../source/kv_cache/storage_backends/nixl.rst:44 msgid "``nixl_buffer_size``: buffer size for NIXL transfers." -msgstr "" +msgstr "``nixl_buffer_size``: NIXL 传输的缓冲区大小。" #: ../../source/kv_cache/storage_backends/nixl.rst:46 msgid "" "``nixl_pool_size``: number of descriptors opened at init time for nixl " "backend. Set to 0 for dynamic mode." -msgstr "" +msgstr "``nixl_pool_size``: 初始化时为 nixl 后端打开的描述符数量。设置为 0 以启用动态模式。" #: ../../source/kv_cache/storage_backends/nixl.rst:48 msgid "" "``nixl_path``: directory under which the storage files will be saved " "(e.g. /mnt/nixl/). Needed for NIXL backends that store to file." -msgstr "" +msgstr "``nixl_path``: 存储文件将保存的目录（例如 /mnt/nixl/）。对于将数据存储到文件的 NIXL 后端，这是必需的。" #: ../../source/kv_cache/storage_backends/nixl.rst:50 msgid "" @@ -105,48 +105,48 @@ msgid "" "be on. \"cpu\" or \"cuda\" is supported for \"GDS\", \"GDS_MT\", and " "\"OBJ\" backends - for \"POSIX\", \"HF3FS\" & \"AZURE_BLOB\", must be " "\"cpu\"." -msgstr "" +msgstr "``nixl_buffer_device``: 指定 NIXL 管理的内存应该位于何处。对于 \\\"GDS\\\"、\\\"GDS_MT\\\" 和 \\\"OBJ\\\" 后端，支持 \\\"cpu\\\" 或 \\\"cuda\\\"；对于 \\\"POSIX\\\"、\\\"HF3FS\\\" 和 \\\"AZURE_BLOB\\\"，必须为 \\\"cpu\\\"。" #: ../../source/kv_cache/storage_backends/nixl.rst:52 msgid "``nixl_backend``: configuration of which nixl backend to use for storage." -msgstr "" +msgstr "``nixl_backend``: 配置用于存储的 nixl 后端。" #: ../../source/kv_cache/storage_backends/nixl.rst:56 msgid "" "Supported backends are: [\"GDS\", \"GDS_MT\", \"POSIX\", \"HF3FS\", " "\"OBJ\", \"AZURE_BLOB\"]." -msgstr "" +msgstr "支持的后端包括：[\\\"GDS\\\", \\\"GDS_MT\\\", \\\"POSIX\\\", \\\"HF3FS\\\", \\\"OBJ\\\", \\\"AZURE_BLOB\\\"]。" #: ../../source/kv_cache/storage_backends/nixl.rst:58 msgid "" "Backend specific params should be provided via " "``extra_config.nixl_backend_params``. Please refer to NIXL documentation " "for specifics." -msgstr "" +msgstr "后端特定参数应通过 ``extra_config.nixl_backend_params`` 提供。有关具体信息，请参阅 NIXL 文档。" #: ../../source/kv_cache/storage_backends/nixl.rst:60 msgid "Example ``lmcache-config.yaml`` for OBJ backend using S3 API:" -msgstr "" +msgstr "示例 ``lmcache-config.yaml`` 用于使用 S3 API 的 OBJ 后端：" #: ../../source/kv_cache/storage_backends/nixl.rst:78 msgid "Example ``lmcache-config.yaml`` for POSIX backend using liburing:" -msgstr "" +msgstr "示例 ``lmcache-config.yaml`` 用于使用 liburing 的 POSIX 后端：" #: ../../source/kv_cache/storage_backends/nixl.rst:82 msgid "" "using POSIX backend with liburing requires NIXL to be built with liburing" " support." -msgstr "" +msgstr "使用带有 liburing 支持的 POSIX 后端需要将 NIXL 构建为支持 liburing。" #: ../../source/kv_cache/storage_backends/nixl.rst:98 msgid "" "Example ``lmcache-config.yaml`` for AZURE_BLOB backend to offload using " "Azure Blob Storage API:" -msgstr "" +msgstr "示例 ``lmcache-config.yaml`` 用于 AZURE_BLOB 后端，通过 Azure Blob Storage API 进行卸载：" #: ../../source/kv_cache/storage_backends/nixl.rst:115 msgid "Per-Worker Endpoint Distribution" -msgstr "" +msgstr "每个工作节点的端点分配" #: ../../source/kv_cache/storage_backends/nixl.rst:117 msgid "" @@ -155,49 +155,49 @@ msgid "" "providing a list of endpoints via ``nixl_endpoint_list``. Each worker " "selects an endpoint in round-robin order based on its ``local_worker_id``" " (the worker ID within its host)." -msgstr "" +msgstr "当使用 OBJ 后端与多个张量并行 (TP) 工作节点时，您可以通过提供一个端点列表来在多个对象存储端点之间分配工作节点，方法是使用 ``nixl_endpoint_list``。每个工作节点根据其 ``local_worker_id``（其主机内的工作节点 ID）以轮询方式选择一个端点。" #: ../../source/kv_cache/storage_backends/nixl.rst:141 msgid "" "When ``nixl_endpoint_list`` is set, any ``endpoint_override`` value in " "``nixl_backend_params`` is ignored (a warning is logged)." -msgstr "" +msgstr "当设置 ``nixl_endpoint_list`` 时，``nixl_backend_params`` 中的任何 ``endpoint_override`` 值将被忽略（会记录警告）。" #: ../../source/kv_cache/storage_backends/nixl.rst:145 msgid "Dynamic Mode" -msgstr "" +msgstr "动态模式" #: ../../source/kv_cache/storage_backends/nixl.rst:147 msgid "" "Nixl Storage Backend also supports a dynamic mode, which creates nixl " "storage descriptors on demand instead of at init time." -msgstr "" +msgstr "Nixl 存储后端还支持动态模式，该模式按需创建 nixl 存储描述符，而不是在初始化时创建。" #: ../../source/kv_cache/storage_backends/nixl.rst:149 msgid "" "In order to use dynamic mode, extra_config.nixl_pool_size should be set " "to 0." -msgstr "" +msgstr "为了使用动态模式，extra_config.nixl_pool_size 应设置为 0。" #: ../../source/kv_cache/storage_backends/nixl.rst:152 msgid "Restrictions" -msgstr "" +msgstr "限制" #: ../../source/kv_cache/storage_backends/nixl.rst:154 msgid "" "Dynamic mode is currently only supported for nixl OBJ and AZURE_BLOB " "backends." -msgstr "" +msgstr "动态模式目前仅支持 nixl OBJ 和 AZURE_BLOB 后端。" #: ../../source/kv_cache/storage_backends/nixl.rst:155 msgid "save_unfull_chunk must be set to False." -msgstr "" +msgstr "save_unfull_chunk 必须设置为 False。" #: ../../source/kv_cache/storage_backends/nixl.rst:157 msgid "Example ``lmcache-config.yaml`` for OBJ backend with dynamic mode:" -msgstr "" +msgstr "示例 ``lmcache-config.yaml`` 用于动态模式的 OBJ 后端：" #: ../../source/kv_cache/storage_backends/nixl.rst:184 msgid "Example ``lmcache-config.yaml`` for AZURE_BLOB backend with dynamic mode:" -msgstr "" +msgstr "示例 ``lmcache-config.yaml`` 用于动态模式的 AZURE_BLOB 后端：" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/redis.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/redis.po index c4b458f2b4e..05dbd96a1c0 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/redis.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/redis.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache/storage_backends/redis.rst:2 msgid "Redis" -msgstr "" +msgstr "Redis" #: ../../source/kv_cache/storage_backends/redis.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/redis.rst:9 msgid "" @@ -35,38 +35,38 @@ msgid "" ":doc:`InfiniStore <./infinistore>`. This guide will mainly focus on " "single-node Redis but also shows you how to set up Redis Sentinels and an" " LMCache Server." -msgstr "" +msgstr "Redis 是一个内存中的键值存储，是 LMCache 中支持的远程 KV Cache 卸载选项之一。其他一些远程后端包括 :doc:`Mooncake <./mooncake>`、:doc:`Valkey <./valkey>` 和 :doc:`InfiniStore <./infinistore>`。本指南将主要集中于单节点 Redis，但也会向您展示如何设置 Redis Sentinel 和 LMCache 服务器。" #: ../../source/kv_cache/storage_backends/redis.rst:14 msgid "Two ways to configure LMCache Redis Offloading:" -msgstr "" +msgstr "配置 LMCache Redis 卸载的两种方式：" #: ../../source/kv_cache/storage_backends/redis.rst:16 msgid "**1. Environment Variables:**" -msgstr "" +msgstr "**1. 环境变量：**" #: ../../source/kv_cache/storage_backends/redis.rst:32 msgid "**2. Configuration File**:" -msgstr "" +msgstr "**2. 配置文件**:" #: ../../source/kv_cache/storage_backends/redis.rst:34 msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" -msgstr "" +msgstr "通过 ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml`` 传入" #: ../../source/kv_cache/storage_backends/redis.rst:36 msgid "Example ``config.yaml``:" -msgstr "" +msgstr "示例 ``config.yaml``:" #: ../../source/kv_cache/storage_backends/redis.rst:53 msgid "Remote Storage Explanation:" -msgstr "" +msgstr "远程存储说明：" #: ../../source/kv_cache/storage_backends/redis.rst:55 msgid "" "LMCache's backend is obeys the natural memory hierarchy of prioritizing " "CPU RAM offloading, then Local Storage offloading, and finally remote " "offloading." -msgstr "" +msgstr "LMCache 的后端遵循自然内存层次结构，优先考虑 CPU RAM 卸载，然后是本地存储卸载，最后是远程卸载。" #: ../../source/kv_cache/storage_backends/redis.rst:58 msgid "" @@ -75,116 +75,116 @@ msgid "" "host:port pairs (depending on what connector type is used). If " "``remote_url`` is set to ``None``, LMCache will not use any remote " "storage." -msgstr "" +msgstr "为了让 LMCache 知道如何创建连接器以连接远程后端，您必须在 ``remote_url`` 中指定一个连接器类型，后面跟着一个或多个主机:端口对（具体取决于使用的连接器类型）。如果 ``remote_url`` 设置为 ``None``，LMCache 将不会使用任何远程存储。" #: ../../source/kv_cache/storage_backends/redis.rst:62 msgid "Examples of ``remote_url``'s:" -msgstr "" +msgstr "``remote_url`` 的示例：" #: ../../source/kv_cache/storage_backends/redis.rst:73 msgid "Remote Storage Example" -msgstr "" +msgstr "远程存储示例" #: ../../source/kv_cache/storage_backends/redis.rst:77 msgid "**Prerequisites:**" -msgstr "" +msgstr "**前提条件：**" #: ../../source/kv_cache/storage_backends/redis.rst:79 msgid "" "A Machine with at least one GPU. You can adjust the max model length of " "your vllm instance depending on your GPU memory." -msgstr "" +msgstr "一台至少配备一个 GPU 的机器。您可以根据您的显存调整 vllm 实例的最大模型长度。" #: ../../source/kv_cache/storage_backends/redis.rst:81 msgid "" "vllm and lmcache installed (:doc:`Installation Guide " "<../../getting_started/installation>`)" -msgstr "" +msgstr "安装了 vllm 和 LMCache (:doc:`安装指南 <../../getting_started/installation>`)" #: ../../source/kv_cache/storage_backends/redis.rst:83 msgid "Hugging Face access to ``meta-llama/Meta-Llama-3.1-8B-Instruct``" -msgstr "" +msgstr "Hugging Face 访问 ``meta-llama/Meta-Llama-3.1-8B-Instruct``" #: ../../source/kv_cache/storage_backends/redis.rst:89 msgid "**Step 0. Set up a directory for this example:**" -msgstr "" +msgstr "**第 0 步. 为此示例设置一个目录:**" #: ../../source/kv_cache/storage_backends/redis.rst:96 msgid "**Step 1. Start a Redis server:**" -msgstr "" +msgstr "**步骤 1. 启动 Redis 服务器：**" #: ../../source/kv_cache/storage_backends/redis.rst:104 msgid "Check if Redis is running:" -msgstr "" +msgstr "检查 Redis 是否正在运行：" #: ../../source/kv_cache/storage_backends/redis.rst:110 msgid "Expected Response:" -msgstr "" +msgstr "预期响应：" #: ../../source/kv_cache/storage_backends/redis.rst:116 msgid "**Optional: Setting up Sentinels:**" -msgstr "" +msgstr "**可选：设置哨兵：**" #: ../../source/kv_cache/storage_backends/redis.rst:118 msgid "" "To enable high availability with Redis, you can configure Redis sentinels" " to monitor the master and automatically fail over to a replica if " "needed." -msgstr "" +msgstr "要通过 Redis 实现高可用性，您可以配置 Redis 哨兵来监控主节点，并在需要时自动切换到副本。" #: ../../source/kv_cache/storage_backends/redis.rst:121 msgid "**Step 1a. Start a Redis replica:**" -msgstr "" +msgstr "**步骤 1a. 启动 Redis 副本：**" #: ../../source/kv_cache/storage_backends/redis.rst:127 msgid "**Step 1b. Create Sentinel configuration files:**" -msgstr "" +msgstr "**步骤 1b. 创建 Sentinel 配置文件：**" #: ../../source/kv_cache/storage_backends/redis.rst:129 msgid "" "Create three files: ``sentinel-26379.conf``, ``sentinel-26380.conf``, and" " ``sentinel-26381.conf``, with contents like this:" -msgstr "" +msgstr "创建三个文件：``sentinel-26379.conf``、``sentinel-26380.conf`` 和 ``sentinel-26381.conf``，内容如下：" #: ../../source/kv_cache/storage_backends/redis.rst:139 msgid "**Step 1c. Start each Sentinel:**" -msgstr "" +msgstr "**步骤 1c. 启动每个 Sentinel:**" #: ../../source/kv_cache/storage_backends/redis.rst:147 msgid "**Step 1d. Make sure the Sentinels are tracking the master:**" -msgstr "" +msgstr "**步骤 1d. 确保哨兵正在跟踪主节点：**" #: ../../source/kv_cache/storage_backends/redis.rst:155 msgid "**Step 1e. Verify everything is running:**" -msgstr "" +msgstr "**步骤 1e. 验证一切是否正常运行:**" #: ../../source/kv_cache/storage_backends/redis.rst:161 msgid "You should see something like this (without the comments):" -msgstr "" +msgstr "您应该看到类似这样的内容（不包括注释）：" #: ../../source/kv_cache/storage_backends/redis.rst:175 msgid "**Alternative: Starting an LMCache Server:**" -msgstr "" +msgstr "**替代方案：启动 LMCache 服务器：**" #: ../../source/kv_cache/storage_backends/redis.rst:177 msgid "" "The ``lmcache_server`` CLI entrypoint starts a remote LMCache server and " "comes with the ``lmcache`` package." -msgstr "" +msgstr "``lmcache_server`` CLI 入口点启动一个远程 LMCache 服务器，并附带 ``lmcache`` 包。" #: ../../source/kv_cache/storage_backends/redis.rst:186 msgid "" "Currently, the only supported device is \"cpu\" (which is the default, so" " you don't need to specify it)." -msgstr "" +msgstr "目前，唯一支持的设备是“cpu”（这是默认值，因此您无需指定它）。" #: ../../source/kv_cache/storage_backends/redis.rst:189 msgid "**Step 2. Start a vLLM server with remote offloading enabled:**" -msgstr "" +msgstr "**步骤 2. 启动一个启用远程卸载的 vLLM 服务器：**" #: ../../source/kv_cache/storage_backends/redis.rst:191 msgid "Create a an lmcache configuration file called: ``redis-offload.yaml``" -msgstr "" +msgstr "创建一个名为 ``redis-offload.yaml`` 的 lmcache 配置文件。" #: ../../source/kv_cache/storage_backends/redis.rst:202 #: ../../source/kv_cache/storage_backends/redis.rst:230 @@ -193,31 +193,31 @@ msgid "" "If you don't want to use a config file, uncomment the first three " "environment variables and then comment out the ``LMCACHE_CONFIG_FILE`` " "below:" -msgstr "" +msgstr "如果您不想使用配置文件，请取消注释前面三个环境变量，然后注释掉下面的 ``LMCACHE_CONFIG_FILE``：" #: ../../source/kv_cache/storage_backends/redis.rst:220 msgid "**Optional: Sentinels**" -msgstr "" +msgstr "**可选：哨兵**" #: ../../source/kv_cache/storage_backends/redis.rst:222 msgid "" "Create a an lmcache configuration file called: ``redis-sentinel-" "offload.yaml``" -msgstr "" +msgstr "创建一个名为 ``redis-sentinel-offload.yaml`` 的 lmcache 配置文件" #: ../../source/kv_cache/storage_backends/redis.rst:245 msgid "**Alternative: LMCache Server**" -msgstr "" +msgstr "**替代方案：LMCache 服务器**" #: ../../source/kv_cache/storage_backends/redis.rst:247 msgid "" "Create a an lmcache configuration file called: ``lmcache-server-" "offload.yaml``" -msgstr "" +msgstr "创建一个名为 ``lmcache-server-offload.yaml`` 的 lmcache 配置文件" #: ../../source/kv_cache/storage_backends/redis.rst:270 msgid "**Step 3. Viewing and Managing LMCache Entries in Redis:**" -msgstr "" +msgstr "**步骤 3. 在 Redis 中查看和管理 LMCache 条目:**" #: ../../source/kv_cache/storage_backends/redis.rst:272 msgid "" @@ -225,123 +225,123 @@ msgid "" "reuse, feel free to use the same ``query-twice.py`` script and ``man-" "bash.txt`` long context as in :doc:`CPU RAM <./cpu_ram>` and :doc:`Local " "Storage <./local_storage>`." -msgstr "" +msgstr "如果您想体验通过卸载和 KV Cache 重用来加速 TTFT 的速度，可以随意使用与 :doc:`CPU RAM <./cpu_ram>` 和 :doc:`Local Storage <./local_storage>` 中相同的 ``query-twice.py`` 脚本和 ``man-bash.txt`` 长上下文。" #: ../../source/kv_cache/storage_backends/redis.rst:275 msgid "" "Here, we are instead going to demonstrate how to search for and modify " "LMCache KV Chunk entries in Redis." -msgstr "" +msgstr "在这里，我们将演示如何在 Redis 中查找和修改 LMCache KV 块条目。" #: ../../source/kv_cache/storage_backends/redis.rst:277 msgid "" "Please note that the official LMCache way to achieve this redis-specific " "functionality of viewing and modifying LMCache KV Chunks is available in " ":doc:`LMCache Controller <../../kv_cache_management/index>`." -msgstr "" +msgstr "请注意，官方的 LMCache 方法来实现此 Redis 特定功能以查看和修改 LMCache KV 块可在 :doc:`LMCache Controller <../../kv_cache_management/index>` 中找到。" #: ../../source/kv_cache/storage_backends/redis.rst:279 msgid "Let's warm/populate LMCache first with ``curl`` this time:" -msgstr "" +msgstr "这次让我们先用 ``curl`` 预热/填充 LMCache：" #: ../../source/kv_cache/storage_backends/redis.rst:296 msgid "" "LMCache stores data in Redis using a structured key format. Each key " "contains the following information in a delimited format:" -msgstr "" +msgstr "LMCache 使用结构化键格式在 Redis 中存储数据。每个键包含以下以分隔格式的信息：" #: ../../source/kv_cache/storage_backends/redis.rst:302 msgid "`model_name`: Name of the language model" -msgstr "" +msgstr "`model_name`: 语言模型的名称" #: ../../source/kv_cache/storage_backends/redis.rst:303 msgid "`world_size`: Total number of workers in distributed deployment" -msgstr "" +msgstr "`world_size`: 分布式部署中工作节点的总数" #: ../../source/kv_cache/storage_backends/redis.rst:304 msgid "" "`worker_id`: ID of the worker that created this cache entry, in the range" " of [0, world_size - 1]" -msgstr "" +msgstr "`worker_id`: 创建此缓存条目的工作线程 ID，范围为 [0, world_size - 1]" #: ../../source/kv_cache/storage_backends/redis.rst:305 msgid "`chunk_hash`: Hash of the token chunk (SHA-256 based)" -msgstr "" +msgstr "`chunk_hash`: 令牌块的哈希值（基于 SHA-256）" #: ../../source/kv_cache/storage_backends/redis.rst:307 msgid "For example, a typical key might look like:" -msgstr "" +msgstr "例如，一个典型的键可能看起来像：" #: ../../source/kv_cache/storage_backends/redis.rst:313 msgid "**Using redis-cli to View LMCache Data**" -msgstr "" +msgstr "**使用 redis-cli 查看 LMCache 数据**" #: ../../source/kv_cache/storage_backends/redis.rst:315 msgid "To inspect and manage LMCache entries in Redis:" -msgstr "" +msgstr "要检查和管理 Redis 中的 LMCache 条目：" #: ../../source/kv_cache/storage_backends/redis.rst:321 msgid "**Optional: If you are using sentinels, first find the master port:**" -msgstr "" +msgstr "**可选：如果您使用的是哨兵，首先找到主端口：**" #: ../../source/kv_cache/storage_backends/redis.rst:329 msgid "**List LMCache keys:**" -msgstr "" +msgstr "**列出 LMCache 键：**" #: ../../source/kv_cache/storage_backends/redis.rst:331 msgid "" "Notice (from the suffixes of the keys) that each LMCache KV Chunk has two" " entries: ``kv_bytes`` and ``metadata``" -msgstr "" +msgstr "注意（从键的后缀中）每个 LMCache KV 块有两个条目：``kv_bytes`` 和 ``metadata``" #: ../../source/kv_cache/storage_backends/redis.rst:344 msgid "**Delete LMCache entries:**" -msgstr "" +msgstr "**删除 LMCache 条目：**" #: ../../source/kv_cache/storage_backends/redis.rst:350 msgid "Delete a specific LMCache entry:" -msgstr "" +msgstr "删除特定的 LMCache 条目：" #: ../../source/kv_cache/storage_backends/redis.rst:359 msgid "**Check if a key exists:**" -msgstr "" +msgstr "**检查键是否存在：**" #: ../../source/kv_cache/storage_backends/redis.rst:365 msgid "**View memory usage for a key:**" -msgstr "" +msgstr "**查看键的内存使用情况：**" #: ../../source/kv_cache/storage_backends/redis.rst:367 msgid "" "Notice that the ``kv_bytes`` entry is what is exactly holding the KV " "Chunk and is much larger than the ``metadata`` entry." -msgstr "" +msgstr "请注意，``kv_bytes`` 条目正是持有 KV 块的内容，并且比 ``metadata`` 条目大得多。" #: ../../source/kv_cache/storage_backends/redis.rst:377 msgid "**Delete specific keys:**" -msgstr "" +msgstr "**删除特定键:**" #: ../../source/kv_cache/storage_backends/redis.rst:391 msgid "**Monitor Redis in real-time:**" -msgstr "" +msgstr "**实时监控 Redis:**" #: ../../source/kv_cache/storage_backends/redis.rst:397 msgid "**Get Redis stats for LMCache:**" -msgstr "" +msgstr "**获取 LMCache 的 Redis 统计信息：**" #: ../../source/kv_cache/storage_backends/redis.rst:407 msgid "" "This tutorial utilized the ``redis-cli`` to directly peak into a remote " "backend and manipualte KV Chunks." -msgstr "" +msgstr "本教程使用 ``redis-cli`` 直接查看远程后端并操作 KV 块。" #: ../../source/kv_cache/storage_backends/redis.rst:410 msgid "" "Once again, please refer to the :doc:`LMCache Controller " "<../../kv_cache_management/index>` for the official LMCache way of " "controlling and routing your KV Caches in your LMCache instances." -msgstr "" +msgstr "请再次参考 :doc:`LMCache Controller <../../kv_cache_management/index>` 以获取在您的 LMCache 实例中控制和路由 KV Cache 的官方 LMCache 方法。" #: ../../source/kv_cache/storage_backends/redis.rst:413 msgid "**Step 4. Clean up:**" -msgstr "" +msgstr "**步骤 4. 清理：**" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/resp.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/resp.po index 5ad4b077ecb..bbe9baae1e3 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/resp.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/resp.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache/storage_backends/resp.rst:2 msgid "RESP (Native Redis/Valkey)" -msgstr "" +msgstr "RESP (原生 Redis/Valkey)" #: ../../source/kv_cache/storage_backends/resp.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/resp.rst:9 msgid "" @@ -33,36 +33,36 @@ msgid "" "Redis and Valkey servers, using the RESP2 wire protocol over TCP. It is " "designed for maximum throughput on KV cache store and retrieval " "operations, achieving **6+ GB/s** on reads with optimal configuration." -msgstr "" +msgstr "RESP 后端是一个高性能的原生 C++ 存储连接器，适用于 Redis 和 Valkey 服务器，使用 TCP 上的 RESP2 传输协议。它旨在实现 KV Cache 存储和检索操作的最大吞吐量，在最佳配置下读取速度可达到 **6+ GB/s**。" #: ../../source/kv_cache/storage_backends/resp.rst:13 msgid "Key advantages over the standard Redis connector:" -msgstr "" +msgstr "相较于标准 Redis 连接器的主要优势：" #: ../../source/kv_cache/storage_backends/resp.rst:15 msgid "" "**Multi-threaded C++ I/O**: Worker threads operate in parallel with zero-" "copy buffer passing and full GIL release" -msgstr "" +msgstr "**多线程 C++ I/O**：工作线程并行操作，采用零拷贝缓冲区传递和完全释放全局解释器锁（GIL）" #: ../../source/kv_cache/storage_backends/resp.rst:16 msgid "" "**Batched tiling**: Large batch operations are automatically split across" " worker threads for maximum parallelism" -msgstr "" +msgstr "**批处理切片**：大型批处理操作会自动在工作线程之间拆分，以实现最大并行性" #: ../../source/kv_cache/storage_backends/resp.rst:17 msgid "" "**eventfd-based completions**: The kernel wakes Python on completion -- " "no polling overhead" -msgstr "" +msgstr "**基于 eventfd 的完成**：内核在完成时唤醒 Python——没有轮询开销" #: ../../source/kv_cache/storage_backends/resp.rst:18 msgid "" "**Dual-mode support**: The same C++ connector works in both non-MP mode " "(via ``ConnectorClientBase``) and MP mode (via " "``NativeConnectorL2Adapter`` as an L2 adapter)" -msgstr "" +msgstr "**双模式支持**：相同的 C++ 连接器在非 MP 模式（通过 ``ConnectorClientBase``）和 MP 模式（通过 ``NativeConnectorL2Adapter`` 作为 L2 适配器）下均可工作。" #: ../../source/kv_cache/storage_backends/resp.rst:20 msgid "" @@ -70,295 +70,295 @@ msgid "" ":doc:`Adding Native Connectors " "<../../developer_guide/extending_lmcache/native_connectors>` for the full" " architecture." -msgstr "" +msgstr "本地 C++ 源代码位于 ``csrc/storage_backends/redis/``。有关完整架构，请参见 :doc:`添加本地连接器 <../../developer_guide/extending_lmcache/native_connectors>`。" #: ../../source/kv_cache/storage_backends/resp.rst:24 msgid "Prerequisites" -msgstr "" +msgstr "先决条件" #: ../../source/kv_cache/storage_backends/resp.rst:26 msgid "" "LMCache installed from source (``pip install -e .``) to compile the C++ " "extension" -msgstr "" +msgstr "从源代码安装的 LMCache (``pip install -e .``) 以编译 C++ 扩展" #: ../../source/kv_cache/storage_backends/resp.rst:27 msgid "" "A Redis 8.2+ or Valkey server (Redis 8.2 recommended for IO threads " "support)" -msgstr "" +msgstr "支持 IO 线程的 Redis 8.2+ 或 Valkey 服务器（推荐使用 Redis 8.2）" #: ../../source/kv_cache/storage_backends/resp.rst:28 msgid "A machine with at least one GPU for vLLM inference" -msgstr "" +msgstr "一台至少配备一个 GPU 的机器用于 vLLM 推理" #: ../../source/kv_cache/storage_backends/resp.rst:32 msgid "Redis Server Setup" -msgstr "" +msgstr "Redis 服务器设置" #: ../../source/kv_cache/storage_backends/resp.rst:35 msgid "" "Redis version and server configuration have a **major** impact on " "throughput. Using Redis 8.2 with IO threads yields ~6 GB/s reads vs ~1.5 " "GB/s with Redis 6.0 defaults." -msgstr "" +msgstr "Redis 版本和服务器配置对吞吐量有 **重大** 影响。使用 IO 线程的 Redis 8.2 读取速度约为 6 GB/s，而使用 Redis 6.0 默认配置时约为 1.5 GB/s。" #: ../../source/kv_cache/storage_backends/resp.rst:38 msgid "**Build Redis 8.2 from source (recommended):**" -msgstr "" +msgstr "**从源代码构建 Redis 8.2（推荐）：**" #: ../../source/kv_cache/storage_backends/resp.rst:47 msgid "**Start the server with IO threads enabled:**" -msgstr "" +msgstr "**启用 IO 线程启动服务器：**" #: ../../source/kv_cache/storage_backends/resp.rst:58 msgid "Recommended Server Flags" -msgstr "" +msgstr "推荐的服务器标志" #: ../../source/kv_cache/storage_backends/resp.rst:62 msgid "Flag" -msgstr "" +msgstr "标志" #: ../../source/kv_cache/storage_backends/resp.rst:63 msgid "Why" -msgstr "" +msgstr "为什么" #: ../../source/kv_cache/storage_backends/resp.rst:64 msgid "``--protected-mode no``" -msgstr "" +msgstr "``--protected-mode no``" #: ../../source/kv_cache/storage_backends/resp.rst:65 msgid "Allow connections from other hosts (use auth in production)" -msgstr "" +msgstr "允许来自其他主机的连接（在生产环境中使用认证）" #: ../../source/kv_cache/storage_backends/resp.rst:66 msgid "``--save '' --appendonly no``" -msgstr "" +msgstr "``--save '' --appendonly no``" #: ../../source/kv_cache/storage_backends/resp.rst:67 msgid "Disable persistence -- KV cache is ephemeral, persistence wastes bandwidth" -msgstr "" +msgstr "禁用持久性 -- KV Cache 是短暂的，持久性会浪费带宽" #: ../../source/kv_cache/storage_backends/resp.rst:68 msgid "``--io-threads 4``" -msgstr "" +msgstr "``--io-threads 4``" #: ../../source/kv_cache/storage_backends/resp.rst:69 msgid "Enable multi-threaded I/O for parallel read/write handling" -msgstr "" +msgstr "启用多线程 I/O 以实现并行读/写处理" #: ../../source/kv_cache/storage_backends/resp.rst:70 msgid "``--port 6379``" -msgstr "" +msgstr "``--port 6379``" #: ../../source/kv_cache/storage_backends/resp.rst:71 msgid "Default port (adjust if running multiple instances)" -msgstr "" +msgstr "默认端口（如果运行多个实例，请调整）" #: ../../source/kv_cache/storage_backends/resp.rst:74 msgid "" "The number of ``--io-threads`` should roughly match the number of " "physical cores available to the Redis process. 4 is a good starting " "point; benchmark with your hardware to find the optimum." -msgstr "" +msgstr "``--io-threads`` 的数量应大致与 Redis 进程可用的物理核心数量相匹配。4 是一个不错的起点；请根据您的硬件进行基准测试以找到最佳值。" #: ../../source/kv_cache/storage_backends/resp.rst:80 msgid "Chunk Size Selection and Throughput Tuning" -msgstr "" +msgstr "块大小选择与吞吐量调优" #: ../../source/kv_cache/storage_backends/resp.rst:82 msgid "" "The chunk size (in tokens) determines how many bytes each Redis key-value" " pair holds. This is the **single most important parameter** for " "throughput." -msgstr "" +msgstr "块大小（以令牌为单位）决定了每个 Redis 键值对所占用的字节数。这是 **吞吐量最重要的参数**。" #: ../../source/kv_cache/storage_backends/resp.rst:85 msgid "" "**The sweet spot is ~4 MB per chunk.** Both smaller and larger chunks " "degrade throughput:" -msgstr "" +msgstr "**最佳块大小约为 4 MB。** 较小或较大的块都会降低吞吐量：" #: ../../source/kv_cache/storage_backends/resp.rst:87 msgid "Chunk Size vs Throughput (Redis 8.2, 8 workers)" -msgstr "" +msgstr "块大小与吞吐量（Redis 8.2，8 个工作线程）" #: ../../source/kv_cache/storage_backends/resp.rst:91 msgid "Chunk Size" -msgstr "" +msgstr "块大小" #: ../../source/kv_cache/storage_backends/resp.rst:92 msgid "Total Data" -msgstr "" +msgstr "总数据" #: ../../source/kv_cache/storage_backends/resp.rst:93 msgid "SET Throughput" -msgstr "" +msgstr "设置吞吐量" #: ../../source/kv_cache/storage_backends/resp.rst:94 msgid "GET Throughput" -msgstr "" +msgstr "获取吞吐量" #: ../../source/kv_cache/storage_backends/resp.rst:95 msgid "1 MB (500 keys)" -msgstr "" +msgstr "1 MB (500 个键)" #: ../../source/kv_cache/storage_backends/resp.rst:96 msgid "500 MB" -msgstr "" +msgstr "500 MB" #: ../../source/kv_cache/storage_backends/resp.rst:97 msgid "~3.5 GB/s" -msgstr "" +msgstr "约 3.5 GB/s" #: ../../source/kv_cache/storage_backends/resp.rst:98 msgid "~5.2 GB/s" -msgstr "" +msgstr "~5.2 GB/s" #: ../../source/kv_cache/storage_backends/resp.rst:99 msgid "**4 MB (500 keys)**" -msgstr "" +msgstr "**4 MB (500 个键)**" #: ../../source/kv_cache/storage_backends/resp.rst:100 msgid "**2 GB**" -msgstr "" +msgstr "**2 GB**" #: ../../source/kv_cache/storage_backends/resp.rst:101 msgid "**~4.4 GB/s**" -msgstr "" +msgstr "**~4.4 GB/s**" #: ../../source/kv_cache/storage_backends/resp.rst:102 msgid "**~5.9 GB/s**" -msgstr "" +msgstr "**~5.9 GB/s**" #: ../../source/kv_cache/storage_backends/resp.rst:103 msgid "8 MB (200 keys)" -msgstr "" +msgstr "8 MB (200 个键)" #: ../../source/kv_cache/storage_backends/resp.rst:104 msgid "1.6 GB" -msgstr "" +msgstr "1.6 GB" #: ../../source/kv_cache/storage_backends/resp.rst:105 msgid "~4.2 GB/s" -msgstr "" +msgstr "约 4.2 GB/s" #: ../../source/kv_cache/storage_backends/resp.rst:106 msgid "~1.4 GB/s" -msgstr "" +msgstr "~1.4 GB/s" #: ../../source/kv_cache/storage_backends/resp.rst:108 msgid "**Why 4 MB?**" -msgstr "" +msgstr "**为什么是 4 MB？**" #: ../../source/kv_cache/storage_backends/resp.rst:110 msgid "Below ~2 MB, per-key overhead (RESP framing, TCP round-trips) dominates" -msgstr "" +msgstr "在 ~2 MB 以下，每个键的开销（RESP 帧、TCP 往返时间）占主导地位" #: ../../source/kv_cache/storage_backends/resp.rst:111 msgid "" "Above ~4 MB, Redis server-side memory allocation and TCP window sizes " "become bottlenecks" -msgstr "" +msgstr "超过 ~4 MB 时，Redis 服务器端内存分配和 TCP 窗口大小成为瓶颈。" #: ../../source/kv_cache/storage_backends/resp.rst:112 msgid "" "At 4 MB, the balance between amortized overhead and memory pressure is " "optimal" -msgstr "" +msgstr "在 4 MB 时，摊销开销与内存压力之间的平衡是最佳的。" #: ../../source/kv_cache/storage_backends/resp.rst:114 msgid "**Calculating chunk size in tokens:**" -msgstr "" +msgstr "**计算块大小（以 token 为单位）：**" #: ../../source/kv_cache/storage_backends/resp.rst:116 msgid "" "The chunk size in bytes depends on the model's hidden dimension, number " "of KV heads, number of layers, and dtype:" -msgstr "" +msgstr "块大小（以字节为单位）取决于模型的隐藏维度、KV 头的数量、层数和数据类型：" #: ../../source/kv_cache/storage_backends/resp.rst:123 msgid "For ``meta-llama/Llama-3.1-8B-Instruct`` with BFloat16:" -msgstr "" +msgstr "对于 ``meta-llama/Llama-3.1-8B-Instruct`` 使用 BFloat16：" #: ../../source/kv_cache/storage_backends/resp.rst:135 msgid "" "The bytes-per-token calculation varies by model architecture. Larger " "models (e.g., 70B) have more layers and larger hidden dimensions, so " "fewer tokens are needed per chunk to reach the 4 MB sweet spot." -msgstr "" +msgstr "每个令牌的字节计算因模型架构而异。较大的模型（例如，70B）具有更多的层和更大的隐藏维度，因此每个块所需的令牌数量较少，以达到 4 MB 的最佳点。" #: ../../source/kv_cache/storage_backends/resp.rst:141 msgid "Throughput Sweep" -msgstr "" +msgstr "吞吐量扫描" #: ../../source/kv_cache/storage_backends/resp.rst:143 msgid "" "To find the optimal configuration for your hardware, use the included " "benchmark:" -msgstr "" +msgstr "要找到适合您硬件的最佳配置，请使用随附的基准测试：" #: ../../source/kv_cache/storage_backends/resp.rst:165 msgid "Expected output:" -msgstr "" +msgstr "预期输出：" #: ../../source/kv_cache/storage_backends/resp.rst:181 msgid "Environment Variable Configuration" -msgstr "" +msgstr "环境变量配置" #: ../../source/kv_cache/storage_backends/resp.rst:183 msgid "" "Sensitive credentials (and optionally host/port) can be provided via " "environment variables instead of config files or CLI arguments. This " "prevents secrets from appearing in logged configuration at startup." -msgstr "" +msgstr "敏感凭据（可选的主机/端口）可以通过环境变量提供，而不是配置文件或 CLI 参数。这可以防止在启动时将秘密记录到配置日志中。" #: ../../source/kv_cache/storage_backends/resp.rst:191 msgid "Variable" -msgstr "" +msgstr "变量" #: ../../source/kv_cache/storage_backends/resp.rst:192 #: ../../source/kv_cache/storage_backends/resp.rst:322 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/kv_cache/storage_backends/resp.rst:193 msgid "``LMCACHE_RESP_USERNAME``" -msgstr "" +msgstr "``LMCACHE_RESP_USERNAME``" #: ../../source/kv_cache/storage_backends/resp.rst:194 msgid "" "Redis ACL username. Used as default when ``username`` is not set in " "config/JSON." -msgstr "" +msgstr "Redis ACL 用户名。当 ``username`` 在 config/JSON 中未设置时使用该默认值。" #: ../../source/kv_cache/storage_backends/resp.rst:195 msgid "``LMCACHE_RESP_PASSWORD``" -msgstr "" +msgstr "``LMCACHE_RESP_PASSWORD``" #: ../../source/kv_cache/storage_backends/resp.rst:196 msgid "" "Redis AUTH password. Used as default when ``password`` is not set in " "config/JSON." -msgstr "" +msgstr "Redis AUTH 密码。当 config/JSON 中未设置 ``password`` 时使用该密码作为默认值。" #: ../../source/kv_cache/storage_backends/resp.rst:197 msgid "``LMCACHE_RESP_HOST``" -msgstr "" +msgstr "``LMCACHE_RESP_HOST``" #: ../../source/kv_cache/storage_backends/resp.rst:198 msgid "" "Redis hostname or IP. Used as default when ``host`` is not set in " "config/JSON/URL." -msgstr "" +msgstr "Redis 主机名或 IP。当配置/JSON/URL 中未设置 ``host`` 时使用此默认值。" #: ../../source/kv_cache/storage_backends/resp.rst:199 msgid "``LMCACHE_RESP_PORT``" -msgstr "" +msgstr "``LMCACHE_RESP_PORT``" #: ../../source/kv_cache/storage_backends/resp.rst:200 msgid "Redis port. Used as default when ``port`` is not set in config/JSON/URL." -msgstr "" +msgstr "Redis 端口。当 ``port`` 在 config/JSON/URL 中未设置时使用此端口作为默认值。" #: ../../source/kv_cache/storage_backends/resp.rst:202 msgid "" @@ -368,203 +368,203 @@ msgid "" "read at adapter creation time inside the adapter itself, so they are " "**never stored in the config object** and **never printed in startup " "logs**." -msgstr "" +msgstr "配置文件（非多租户）和 ``--l2-adapter`` JSON（多租户）优先于环境变量。环境变量作为默认值使用——当相应的配置值为空或未设置时使用。它们在适配器创建时被读取，因此**从不存储在配置对象中**，并且**从不在启动日志中打印**。" #: ../../source/kv_cache/storage_backends/resp.rst:208 msgid "**Example — MP mode with env vars:**" -msgstr "" +msgstr "**示例 — MP 模式与环境变量：**" #: ../../source/kv_cache/storage_backends/resp.rst:222 msgid "**Example — Non-MP mode with env vars:**" -msgstr "" +msgstr "**示例 — 非 MP 模式与环境变量：**" #: ../../source/kv_cache/storage_backends/resp.rst:236 msgid "" "For production deployments, always use environment variables for " "credentials rather than embedding them in config files or CLI arguments." -msgstr "" +msgstr "在生产环境中，始终使用环境变量来存储凭据，而不是将其嵌入配置文件或命令行参数中。" #: ../../source/kv_cache/storage_backends/resp.rst:241 msgid "Non-MP Mode (Single Process)" -msgstr "" +msgstr "非 MP 模式（单进程）" #: ../../source/kv_cache/storage_backends/resp.rst:243 msgid "" "In non-MP mode, the RESP connector is used directly as a remote storage " "backend via the ``RESPClient`` asyncio wrapper." -msgstr "" +msgstr "在非多进程模式下，RESP 连接器通过 ``RESPClient`` asyncio 包装器直接用作远程存储后端。" #: ../../source/kv_cache/storage_backends/resp.rst:246 msgid "**Configuration file** (``resp-config.yaml``):" -msgstr "" +msgstr "**配置文件** (``resp-config.yaml``):" #: ../../source/kv_cache/storage_backends/resp.rst:254 msgid "" "Credentials can be set via environment variables (recommended) or in the " "config file under ``extra_config`` (see `Environment Variable " "Configuration`_ above)." -msgstr "" +msgstr "凭据可以通过环境变量（推荐）或在配置文件的 ``extra_config`` 中设置（请参见上面的 `环境变量配置`_）。" #: ../../source/kv_cache/storage_backends/resp.rst:257 msgid "**Launch vLLM:**" -msgstr "" +msgstr "**启动 vLLM:**" #: ../../source/kv_cache/storage_backends/resp.rst:268 msgid "" "``save_unfull_chunk`` must be off (default) and chunk metadata saving " "must be disabled for optimal throughput with the native RESP connector." -msgstr "" +msgstr "``save_unfull_chunk`` 必须关闭（默认）并且必须禁用块元数据保存，以便在使用原生 RESP 连接器时实现最佳吞吐量。" #: ../../source/kv_cache/storage_backends/resp.rst:273 msgid "MP Mode (Multiprocess)" -msgstr "" +msgstr "MP 模式（多进程）" #: ../../source/kv_cache/storage_backends/resp.rst:275 msgid "" "In MP mode, LMCache runs as a separate server process communicating with " "vLLM over ZMQ. The RESP connector serves as an L2 adapter with variable-" "size chunk support." -msgstr "" +msgstr "在 MP 模式下，LMCache 作为一个独立的服务器进程运行，通过 ZMQ 与 vLLM 进行通信。RESP 连接器作为一个具有可变大小块支持的 L2 适配器。" #: ../../source/kv_cache/storage_backends/resp.rst:278 msgid "**Step 1: Start Redis** (see `Redis Server Setup`_ above)" -msgstr "" +msgstr "**步骤 1：启动 Redis**（请参见上面的 `Redis Server Setup`_）" #: ../../source/kv_cache/storage_backends/resp.rst:280 msgid "**Step 2: Launch LMCache MP Server:**" -msgstr "" +msgstr "**步骤 2：启动 LMCache MP 服务器：**" #: ../../source/kv_cache/storage_backends/resp.rst:291 msgid "**Step 3: Launch vLLM with LMCache MP Connector:**" -msgstr "" +msgstr "**步骤 3：启动带有 LMCache MP 连接器的 vLLM：**" #: ../../source/kv_cache/storage_backends/resp.rst:311 msgid "L2 Adapter Configuration" -msgstr "" +msgstr "L2 适配器配置" #: ../../source/kv_cache/storage_backends/resp.rst:313 msgid "The ``--l2-adapter`` JSON accepts these fields:" -msgstr "" +msgstr "``--l2-adapter`` JSON 接受以下字段：" #: ../../source/kv_cache/storage_backends/resp.rst:319 msgid "Field" -msgstr "" +msgstr "字段" #: ../../source/kv_cache/storage_backends/resp.rst:320 msgid "Type" -msgstr "" +msgstr "类型" #: ../../source/kv_cache/storage_backends/resp.rst:321 msgid "Default" -msgstr "" +msgstr "默认" #: ../../source/kv_cache/storage_backends/resp.rst:323 msgid "``type``" -msgstr "" +msgstr "``type``" #: ../../source/kv_cache/storage_backends/resp.rst:324 #: ../../source/kv_cache/storage_backends/resp.rst:328 #: ../../source/kv_cache/storage_backends/resp.rst:340 #: ../../source/kv_cache/storage_backends/resp.rst:344 msgid "str" -msgstr "" +msgstr "字符串" #: ../../source/kv_cache/storage_backends/resp.rst:325 #: ../../source/kv_cache/storage_backends/resp.rst:329 #: ../../source/kv_cache/storage_backends/resp.rst:333 msgid "(required)" -msgstr "" +msgstr "（必需）" #: ../../source/kv_cache/storage_backends/resp.rst:326 msgid "Must be ``\"resp\"``" -msgstr "" +msgstr "必须是 ``\"resp\"``" #: ../../source/kv_cache/storage_backends/resp.rst:327 msgid "``host``" -msgstr "" +msgstr "``host``" #: ../../source/kv_cache/storage_backends/resp.rst:330 msgid "Redis/Valkey hostname or IP" -msgstr "" +msgstr "Redis/Valkey 主机名或 IP" #: ../../source/kv_cache/storage_backends/resp.rst:331 msgid "``port``" -msgstr "" +msgstr "``port``" #: ../../source/kv_cache/storage_backends/resp.rst:332 #: ../../source/kv_cache/storage_backends/resp.rst:336 msgid "int" -msgstr "" +msgstr "整数" #: ../../source/kv_cache/storage_backends/resp.rst:334 msgid "Redis/Valkey port" -msgstr "" +msgstr "Redis/Valkey 端口" #: ../../source/kv_cache/storage_backends/resp.rst:335 msgid "``num_workers``" -msgstr "" +msgstr "``num_workers``" #: ../../source/kv_cache/storage_backends/resp.rst:337 msgid "8" -msgstr "" +msgstr "8" #: ../../source/kv_cache/storage_backends/resp.rst:338 msgid "C++ worker threads for parallel I/O" -msgstr "" +msgstr "用于并行 I/O 的 C++ 工作线程" #: ../../source/kv_cache/storage_backends/resp.rst:339 msgid "``username``" -msgstr "" +msgstr "``username``" #: ../../source/kv_cache/storage_backends/resp.rst:341 #: ../../source/kv_cache/storage_backends/resp.rst:345 msgid "``\"\"``" -msgstr "" +msgstr "``\"\"``" #: ../../source/kv_cache/storage_backends/resp.rst:342 msgid "" "Redis ACL username (leave empty for no auth). Falls back to " "``LMCACHE_RESP_USERNAME`` env var if empty." -msgstr "" +msgstr "Redis ACL 用户名（留空以不进行身份验证）。如果为空，将回退到 ``LMCACHE_RESP_USERNAME`` 环境变量。" #: ../../source/kv_cache/storage_backends/resp.rst:343 msgid "``password``" -msgstr "" +msgstr "``密码``" #: ../../source/kv_cache/storage_backends/resp.rst:346 msgid "" "Redis AUTH password (leave empty for no auth). Falls back to " "``LMCACHE_RESP_PASSWORD`` env var if empty." -msgstr "" +msgstr "Redis AUTH 密码（留空表示不进行身份验证）。如果为空，将回退到 ``LMCACHE_RESP_PASSWORD`` 环境变量。" #: ../../source/kv_cache/storage_backends/resp.rst:347 msgid "``max_capacity_gb``" -msgstr "" +msgstr "``max_capacity_gb``" #: ../../source/kv_cache/storage_backends/resp.rst:348 msgid "float" -msgstr "" +msgstr "浮点数" #: ../../source/kv_cache/storage_backends/resp.rst:349 msgid "0" -msgstr "" +msgstr "0" #: ../../source/kv_cache/storage_backends/resp.rst:350 msgid "" "Maximum L2 storage capacity in GB for client-side usage tracking. " "Required for L2 eviction. Set to 0 (default) to disable usage tracking." -msgstr "" +msgstr "用于客户端使用跟踪的最大 L2 存储容量（以 GB 为单位）。这是 L2 逐出的必要条件。设置为 0（默认值）以禁用使用跟踪。" #: ../../source/kv_cache/storage_backends/resp.rst:353 msgid "L2 Eviction" -msgstr "" +msgstr "L2 逐出" #: ../../source/kv_cache/storage_backends/resp.rst:355 msgid "" "To enable automatic eviction of least-recently-used keys when the Redis " "backend fills up, set ``max_capacity_gb`` and add an ``\"eviction\"`` " "block:" -msgstr "" +msgstr "要启用在 Redis 后端填满时自动逐出最近最少使用的键，请设置 ``max_capacity_gb`` 并添加一个 ``\"eviction\"`` 块：" #: ../../source/kv_cache/storage_backends/resp.rst:378 #, python-format @@ -573,7 +573,7 @@ msgid "" "(``trigger_watermark``), the eviction controller will delete the least-" "recently-used ~20% of stored keys (``eviction_ratio``) using the Redis " "``DEL`` command." -msgstr "" +msgstr "这配置了 10 GB 的容量限制。当使用量超过 80%（``trigger_watermark``）时，逐出控制器将使用 Redis ``DEL`` 命令删除最近最少使用的 ~20% 存储的键（``eviction_ratio``）。" #: ../../source/kv_cache/storage_backends/resp.rst:383 msgid "" @@ -581,136 +581,136 @@ msgid "" "configure the Redis server's ``maxmemory`` setting. You should set " "``max_capacity_gb`` to match or be slightly below your Redis server's " "available memory." -msgstr "" +msgstr "``max_capacity_gb`` 启用 **客户端** 大小跟踪。它并不配置 Redis 服务器的 ``maxmemory`` 设置。您应该将 ``max_capacity_gb`` 设置为与 Redis 服务器的可用内存相匹配或略低于该值。" #: ../../source/kv_cache/storage_backends/resp.rst:389 msgid "Testing the Setup" -msgstr "" +msgstr "测试设置" #: ../../source/kv_cache/storage_backends/resp.rst:391 msgid "" "Send the same prompt twice. The first request stores KV cache to Redis; " "the second retrieves it." -msgstr "" +msgstr "两次发送相同的提示。第一次请求将 KV Cache 存储到 Redis；第二次请求检索它。" #: ../../source/kv_cache/storage_backends/resp.rst:408 msgid "Verify data was stored:" -msgstr "" +msgstr "验证数据是否已存储：" #: ../../source/kv_cache/storage_backends/resp.rst:414 msgid "Clear state between runs:" -msgstr "" +msgstr "在运行之间清除状态：" #: ../../source/kv_cache/storage_backends/resp.rst:422 msgid "Best Practices" -msgstr "" +msgstr "最佳实践" #: ../../source/kv_cache/storage_backends/resp.rst:424 msgid "**Server deployment:**" -msgstr "" +msgstr "**服务器部署：**" #: ../../source/kv_cache/storage_backends/resp.rst:426 msgid "Use Redis 8.2+ with ``--io-threads 4`` (or more, matching available cores)" -msgstr "" +msgstr "使用 Redis 8.2+ 和 ``--io-threads 4``（或更多，匹配可用核心）" #: ../../source/kv_cache/storage_backends/resp.rst:427 msgid "Disable persistence (``--save '' --appendonly no``) for KV cache workloads" -msgstr "" +msgstr "禁用持久化（``--save '' --appendonly no``）以支持 KV Cache 工作负载" #: ../../source/kv_cache/storage_backends/resp.rst:428 msgid "Pin Redis to its own NUMA node if running on multi-socket systems" -msgstr "" +msgstr "在多插槽系统上将 Redis 固定到其自己的 NUMA 节点" #: ../../source/kv_cache/storage_backends/resp.rst:429 msgid "" "For production, enable authentication with ``--requirepass`` and supply " "credentials via ``LMCACHE_RESP_USERNAME`` / ``LMCACHE_RESP_PASSWORD`` " "environment variables to keep them out of logs" -msgstr "" +msgstr "在生产环境中，使用 ``--requirepass`` 启用身份验证，并通过 ``LMCACHE_RESP_USERNAME`` / ``LMCACHE_RESP_PASSWORD`` 环境变量提供凭据，以避免将其记录到日志中。" #: ../../source/kv_cache/storage_backends/resp.rst:431 msgid "**Client tuning:**" -msgstr "" +msgstr "**客户端调优:**" #: ../../source/kv_cache/storage_backends/resp.rst:433 msgid "" "Start with ``num_workers: 8`` and increase if the server has spare CPU " "and you're not saturating the network" -msgstr "" +msgstr "从 ``num_workers: 8`` 开始，如果服务器有多余的 CPU 并且网络没有饱和，则可以增加该值。" #: ../../source/kv_cache/storage_backends/resp.rst:434 msgid "" "More workers help when chunk sizes are smaller (more keys per batch = " "more parallelism needed)" -msgstr "" +msgstr "当块大小较小时，更多的工作线程会有所帮助（每批次更多的键 = 需要更多的并行性）" #: ../../source/kv_cache/storage_backends/resp.rst:435 msgid "" "On NUMA systems, ensure the LMCache process runs on the same NUMA node as" " the NIC" -msgstr "" +msgstr "在 NUMA 系统上，确保 LMCache 进程与 NIC 运行在同一个 NUMA 节点上。" #: ../../source/kv_cache/storage_backends/resp.rst:437 msgid "**Chunk size:**" -msgstr "" +msgstr "**块大小:**" #: ../../source/kv_cache/storage_backends/resp.rst:439 msgid "Target ~4 MB per chunk for maximum throughput" -msgstr "" +msgstr "每个块目标约 4 MB，以实现最大吞吐量" #: ../../source/kv_cache/storage_backends/resp.rst:440 msgid "" "Calculate the token count using your model's per-token byte size (see " "formula above)" -msgstr "" +msgstr "使用模型的每个 token 字节大小计算 token 数量（请参见上面的公式）" #: ../../source/kv_cache/storage_backends/resp.rst:441 msgid "" "If unsure, run the benchmark sweep to find the optimum for your specific " "hardware" -msgstr "" +msgstr "如果不确定，请运行基准测试以找到适合您特定硬件的最佳值。" #: ../../source/kv_cache/storage_backends/resp.rst:443 msgid "**Network:**" -msgstr "" +msgstr "**网络：**" #: ../../source/kv_cache/storage_backends/resp.rst:445 msgid "Use localhost or loopback for single-machine deployments" -msgstr "" +msgstr "在单机部署中使用 localhost 或回环地址" #: ../../source/kv_cache/storage_backends/resp.rst:446 msgid "" "For cross-machine setups, ensure low-latency networking (ideally <100 us " "RTT)" -msgstr "" +msgstr "对于跨机器设置，确保低延迟网络（理想情况下 <100 微秒 RTT）" #: ../../source/kv_cache/storage_backends/resp.rst:447 msgid "" "The RESP connector uses TCP; RDMA is not currently supported (consider " ":doc:`Mooncake <./mooncake>` for RDMA)" -msgstr "" +msgstr "RESP 连接器使用 TCP；目前不支持 RDMA（考虑使用 :doc:`Mooncake <./mooncake>` 进行 RDMA）。" #: ../../source/kv_cache/storage_backends/resp.rst:451 msgid "Additional Resources" -msgstr "" +msgstr "附加资源" #: ../../source/kv_cache/storage_backends/resp.rst:453 msgid "" "Benchmark script: " "``examples/kv_cache_reuse/remote_backends/resp/benchmark_resp_client.py``" -msgstr "" +msgstr "基准测试脚本：``examples/kv_cache_reuse/remote_backends/resp/benchmark_resp_client.py``" #: ../../source/kv_cache/storage_backends/resp.rst:454 msgid "C++ source: ``csrc/storage_backends/redis/``" -msgstr "" +msgstr "C++ 源码: ``csrc/storage_backends/redis/``" #: ../../source/kv_cache/storage_backends/resp.rst:455 msgid "Native connector architecture: ``csrc/storage_backends/README.md``" -msgstr "" +msgstr "本地连接器架构: ``csrc/storage_backends/README.md``" #: ../../source/kv_cache/storage_backends/resp.rst:456 msgid "" "Developer guide for adding new native connectors: :doc:`Adding Native " "Connectors <../../developer_guide/extending_lmcache/native_connectors>`" -msgstr "" +msgstr "添加新原生连接器的开发者指南： :doc:`添加原生连接器 <../../developer_guide/extending_lmcache/native_connectors>`" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/s3.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/s3.po index 2893d24920a..b5a2370d744 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/s3.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/s3.po @@ -21,67 +21,67 @@ msgstr "" #: ../../source/kv_cache/storage_backends/s3.rst:2 msgid "S3 Backend" -msgstr "" +msgstr "S3 后端" #: ../../source/kv_cache/storage_backends/s3.rst:5 msgid "Example Configurations" -msgstr "" +msgstr "示例配置" #: ../../source/kv_cache/storage_backends/s3.rst:8 msgid "Basic S3 Configuration" -msgstr "" +msgstr "基本 S3 配置" #: ../../source/kv_cache/storage_backends/s3.rst:24 msgid "S3 Express One Zone" -msgstr "" +msgstr "S3 Express One Zone" #: ../../source/kv_cache/storage_backends/s3.rst:42 msgid "CoreWeave (S3-compatible)" -msgstr "" +msgstr "CoreWeave (S3 兼容)" #: ../../source/kv_cache/storage_backends/s3.rst:64 msgid "" "**Note**: `cwlota.com` is CoreWeave's S3-compatible Cloud Storage that " "caches for GPU locality. You can set `disable_tls: True` for non-AWS " "services." -msgstr "" +msgstr "**注意**：`cwlota.com` 是 CoreWeave 的 S3 兼容云存储，旨在为 GPU 本地性提供缓存。您可以将 `disable_tls: True` 设置为非 AWS 服务。" #: ../../source/kv_cache/storage_backends/s3.rst:66 msgid "" "Check out the blog post between LMCache, Cohere, and CoreWeave: " "https://blog.lmcache.ai/en/2025/10/29/breaking-the-memory-barrier-how-" "lmcache-and-coreweave-power-efficient-llm-inference-for-cohere/" -msgstr "" +msgstr "查看 LMCache、Cohere 和 CoreWeave 之间的博客文章：https://blog.lmcache.ai/en/2025/10/29/breaking-the-memory-barrier-how-lmcache-and-coreweave-power-efficient-llm-inference-for-cohere/" #: ../../source/kv_cache/storage_backends/s3.rst:70 msgid "Configuration Parameters" -msgstr "" +msgstr "配置参数" #: ../../source/kv_cache/storage_backends/s3.rst:72 msgid "**remote_url**: S3 bucket URL (`s3://bucket-name`)" -msgstr "" +msgstr "**remote_url**: S3 存储桶 URL (`s3://bucket-name`)" #: ../../source/kv_cache/storage_backends/s3.rst:73 msgid "" "**save_unfull_chunk**: Save partial chunks (default: False, **must be " "False for S3**)" -msgstr "" +msgstr "**save_unfull_chunk**: 保存部分块（默认值：False，**对于 S3 必须为 False**）" #: ../../source/kv_cache/storage_backends/s3.rst:74 msgid "**enable_async_loading**: Async loading (default: False)" -msgstr "" +msgstr "**enable_async_loading**: 异步加载（默认值：False）" #: ../../source/kv_cache/storage_backends/s3.rst:75 msgid "**blocking_timeout_secs**: Timeout seconds (default: 10)" -msgstr "" +msgstr "**blocking_timeout_secs**: 超时秒数（默认值：10）" #: ../../source/kv_cache/storage_backends/s3.rst:78 msgid "S3-Specific (in extra_config)" -msgstr "" +msgstr "S3 特定（在 extra_config 中）" #: ../../source/kv_cache/storage_backends/s3.rst:80 msgid "**s3_region**: AWS region for S3 client (required)" -msgstr "" +msgstr "**s3_region**: S3 客户端的 AWS 区域（必需）" #: ../../source/kv_cache/storage_backends/s3.rst:81 msgid "" @@ -89,49 +89,49 @@ msgid "" "spawn. Benefits taper out after exceeding the number of CPU cores. This " "is also a way to restrict the number of outgoing requests in case your " "S3-compatible object store has a rate-limiting gateway." -msgstr "" +msgstr "**s3_num_io_threads**: AWS CRT 客户端生成的 IO 线程数量。超过 CPU 核心数量后，收益会逐渐减少。这也是限制外发请求数量的一种方式，以防您的 S3 兼容对象存储具有速率限制网关。" #: ../../source/kv_cache/storage_backends/s3.rst:82 msgid "" "**s3_prefer_http2**: Enable HTTP/2 with ALPN negotiation ([\"h2\", " "\"http/1.1\"])" -msgstr "" +msgstr "**s3_prefer_http2**: 启用带有 ALPN 协商的 HTTP/2 ([\\\"h2\\\", \\\"http/1.1\\\"])" #: ../../source/kv_cache/storage_backends/s3.rst:83 msgid "" "**s3_enable_s3express**: Enable S3 Express One Zone support in AWS CRT " "client" -msgstr "" +msgstr "**s3_enable_s3express**: 在 AWS CRT 客户端中启用 S3 Express 单区支持" #: ../../source/kv_cache/storage_backends/s3.rst:84 msgid "" "**save_chunk_meta**: Whether to save chunk metadata in the object store " "along with your data (False required for S3)" -msgstr "" +msgstr "**save_chunk_meta**: 是否在对象存储中与您的数据一起保存块元数据（对于 S3，需设置为 False）" #: ../../source/kv_cache/storage_backends/s3.rst:85 msgid "" "**aws_access_key_id**: AWS access key ID (or log in with `aws configure` " "in your environment)" -msgstr "" +msgstr "**aws_access_key_id**: AWS 访问密钥 ID（或在您的环境中使用 `aws configure` 登录）" #: ../../source/kv_cache/storage_backends/s3.rst:86 msgid "" "**aws_secret_access_key**: AWS secret access key (or log in with `aws " "configure` in your environment)" -msgstr "" +msgstr "**aws_secret_access_key**: AWS 秘密访问密钥（或在您的环境中使用 `aws configure` 登录）" #: ../../source/kv_cache/storage_backends/s3.rst:88 msgid "**Tips:**::" -msgstr "" +msgstr "**提示:**::" #: ../../source/kv_cache/storage_backends/s3.rst:92 msgid "Consider S3 Express One Zone for less redundancy but better performance" -msgstr "" +msgstr "考虑使用 S3 Express One Zone，以获得更少的冗余但更好的性能。" #: ../../source/kv_cache/storage_backends/s3.rst:96 msgid "MP Mode Configuration" -msgstr "" +msgstr "MP 模式配置" #: ../../source/kv_cache/storage_backends/s3.rst:98 msgid "" @@ -139,54 +139,54 @@ msgid "" "spec passed to the LMCache server, rather than through ``remote_url`` + " "``extra_config``. Each ``--l2-adapter`` argument takes a JSON object " "whose ``\"type\": \"s3\"`` field selects the S3 adapter." -msgstr "" +msgstr "在多进程（MP）模式下，S3 通过传递给 LMCache 服务器的 JSON 规范配置为 L2 适配器，而不是通过 ``remote_url`` + ``extra_config``。每个 ``--l2-adapter`` 参数接受一个 JSON 对象，其 ``\\\"type\\\": \\\"s3\\\"`` 字段选择 S3 适配器。" #: ../../source/kv_cache/storage_backends/s3.rst:122 msgid "S3 L2 Adapter Fields" -msgstr "" +msgstr "S3 L2 适配器字段" #: ../../source/kv_cache/storage_backends/s3.rst:124 msgid "**type** (required): must be ``\"s3\"``." -msgstr "" +msgstr "**类型**（必需）：必须是 ``\"s3\"``。" #: ../../source/kv_cache/storage_backends/s3.rst:125 msgid "" "**s3_endpoint** (required): bucket URL. Accepts either ``s3://bucket`` or" " the bare host form (e.g. ``bucket.s3.us-east-1.amazonaws.com``)." -msgstr "" +msgstr "**s3_endpoint** (必需): 存储桶 URL。接受 ``s3://bucket`` 或裸主机形式（例如 ``bucket.s3.us-east-1.amazonaws.com``）。" #: ../../source/kv_cache/storage_backends/s3.rst:127 msgid "**s3_region** (required): AWS region for the S3 client." -msgstr "" +msgstr "**s3_region** (必需): S3 客户端的 AWS 区域。" #: ../../source/kv_cache/storage_backends/s3.rst:128 msgid "**s3_num_io_threads**: number of CRT IO threads (default ``64``)." -msgstr "" +msgstr "**s3_num_io_threads**: CRT IO 线程的数量（默认 ``64``）。" #: ../../source/kv_cache/storage_backends/s3.rst:129 msgid "" "**s3_prefer_http2**: attempt HTTP/2 via ALPN negotiation (default " "``true``)." -msgstr "" +msgstr "**s3_prefer_http2**: 通过 ALPN 协商尝试 HTTP/2（默认 ``true``）。" #: ../../source/kv_cache/storage_backends/s3.rst:130 msgid "" "**s3_enable_s3express**: enable S3 Express One Zone signing (default " "``false``)." -msgstr "" +msgstr "**s3_enable_s3express**: 启用 S3 Express 单区域签名（默认 ``false``）。" #: ../../source/kv_cache/storage_backends/s3.rst:131 msgid "" "**disable_tls**: bypass TLS, for non-AWS HTTP endpoints (default " "``false``)." -msgstr "" +msgstr "**disable_tls**: 跳过 TLS，用于非 AWS HTTP 端点（默认 ``false``）。" #: ../../source/kv_cache/storage_backends/s3.rst:132 msgid "" "**aws_access_key_id**, **aws_secret_access_key**: optional static " "credentials. When omitted the adapter uses the AWS default credentials " "chain (``aws configure``, environment variables, IRSA, etc.)." -msgstr "" +msgstr "**aws_access_key_id**, **aws_secret_access_key**: 可选的静态凭证。当省略时，适配器使用 AWS 默认凭证链（``aws configure``、环境变量、IRSA 等）。" #: ../../source/kv_cache/storage_backends/s3.rst:135 msgid "" @@ -194,7 +194,7 @@ msgid "" " L2 eviction. Set to ``0`` (default) to disable usage tracking — " "``get_usage()`` then returns ``(-1.0, -1.0)`` and no automatic eviction " "is triggered." -msgstr "" +msgstr "**max_capacity_gb**: ``get_usage()`` 用于基于水印的 L2 逐出所使用的容量。设置为 ``0``（默认）以禁用使用跟踪 — ``get_usage()`` 将返回 ``(-1.0, -1.0)``，并且不会触发自动逐出。" #: ../../source/kv_cache/storage_backends/s3.rst:138 msgid "" @@ -203,18 +203,18 @@ msgid "" " for the full schema. When present, keys that are currently being loaded " "(reference-counted by the lookup-and-lock path) are skipped by " "``delete()``." -msgstr "" +msgstr "**逐出**：可选的子字典，启用此适配器的 L2 逐出控制器。有关完整的架构，请参见 :class:`L2AdapterConfigBase` ``_parse_eviction_config``。当存在时，当前正在加载的键（通过查找和锁定路径进行引用计数）将被 ``delete()`` 跳过。" #: ../../source/kv_cache/storage_backends/s3.rst:145 msgid "Differences vs Non-MP S3" -msgstr "" +msgstr "与非多租户 S3 的区别" #: ../../source/kv_cache/storage_backends/s3.rst:147 msgid "" "The MP adapter honors first-class eviction: it implements ``delete()`` " "(real S3 ``DeleteObject``), refcounted ``submit_unlock``, and " "``get_usage()`` driven by ``max_capacity_gb``." -msgstr "" +msgstr "MP 适配器支持一流的逐出：它实现了 ``delete()``（真实的 S3 ``DeleteObject``）、引用计数的 ``submit_unlock`` 和由 ``max_capacity_gb`` 驱动的 ``get_usage()``。" #: ../../source/kv_cache/storage_backends/s3.rst:150 msgid "" @@ -223,9 +223,9 @@ msgid "" "name is ``@@``, which is **not** " "compatible with the non-MP naming. A bucket populated by non-MP LMCache " "cannot be read directly by MP LMCache and vice versa." -msgstr "" +msgstr "键通过 ``ObjectKey`` (``model_name`` + ``kv_rank`` + ``chunk_hash``) 而不是 ``CacheEngineKey`` 来识别。线格式对象名称为 ``@@``，这与非 MP 命名 **不** 兼容。由非 MP LMCache 填充的桶不能被 MP LMCache 直接读取，反之亦然。" #: ../../source/kv_cache/storage_backends/s3.rst:155 msgid "Unfull chunks are rejected (same constraint as non-MP)." -msgstr "" +msgstr "未填充的块会被拒绝（与非 MP 的约束相同）。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/sagemaker_hyperpod.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/sagemaker_hyperpod.po index fbaa6786101..c528e749f00 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/sagemaker_hyperpod.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/sagemaker_hyperpod.po @@ -21,53 +21,53 @@ msgstr "" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:2 msgid "SageMaker Hyperpod" -msgstr "" +msgstr "SageMaker Hyperpod" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:5 msgid "Prerequisites" -msgstr "" +msgstr "前提条件" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:7 msgid "" "Create an Amazon SageMaker HyperPod cluster with tiered storage enabled " "by following the instructions at:" -msgstr "" +msgstr "按照以下说明创建一个启用分层存储的 Amazon SageMaker HyperPod 集群：" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:9 msgid "" "https://docs.aws.amazon.com/sagemaker/latest/dg/managed-tier-" "checkpointing-setup.html" -msgstr "" +msgstr "https://docs.aws.amazon.com/sagemaker/latest/dg/managed-tier-checkpointing-setup.html" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:11 msgid "" "This enables the ai-toolkit daemon that provides shared memory access for" " LMCache." -msgstr "" +msgstr "这启用提供 LMCache 共享内存访问的 ai-toolkit 守护进程。" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:14 msgid "Example Configuration" -msgstr "" +msgstr "示例配置" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:24 msgid "Configuration Parameters" -msgstr "" +msgstr "配置参数" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:27 msgid "SageMaker Hyperpod-Specific (in extra_config)" -msgstr "" +msgstr "SageMaker Hyperpod 特定（在 extra_config 中）" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:29 msgid "" "**sagemaker_hyperpod_bucket**: Bucket name for KV storage namespace " "(default: \"lmcache\")" -msgstr "" +msgstr "**sagemaker_hyperpod_bucket**: KV 存储命名空间的桶名称（默认: \"lmcache\"）" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:30 msgid "" "**sagemaker_hyperpod_shared_memory_name**: Name of shared memory segment " "(default: \"shared_memory\"). Set to None to disable shared memory." -msgstr "" +msgstr "**sagemaker_hyperpod_shared_memory_name**: 共享内存段的名称（默认: \"shared_memory\"）。设置为 None 以禁用共享内存。" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:31 msgid "" @@ -76,7 +76,7 @@ msgid "" "default: 100, minimum: 1). This limit is per LMCache engine instance. " "With multiple workers (e.g., high TP), each worker creates its own engine" " with separate limits." -msgstr "" +msgstr "**sagemaker_hyperpod_max_concurrent_requests**: 每时每刻允许的最大并发 HTTP 请求数（应用级限流，默认值：100，最小值：1）。此限制适用于每个 LMCache 引擎实例。对于多个工作进程（例如，高 TP），每个工作进程创建自己的引擎并具有独立的限制。" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:32 msgid "" @@ -85,7 +85,7 @@ msgid "" "minimum: 1). For typical single-daemon setups, this effectively limits " "connections from one engine to one daemon. With N workers per node, total" " connections to the daemon = N × this value." -msgstr "" +msgstr "**sagemaker_hyperpod_max_connections**: 每个 LMCache 引擎在所有守护进程中的连接池中允许的最大 TCP 连接总数（默认：256，最小：1）。对于典型的单守护进程设置，这实际上限制了一个引擎与一个守护进程之间的连接。每个节点的 N 个工作线程与守护进程的总连接数 = N × 此值。" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:33 msgid "" @@ -98,60 +98,60 @@ msgid "" "balancing. With N workers per node connecting to the same daemon, total " "connections = N × this value. Reduce proportionally for high TP setups " "(e.g., set to 16 for 8 workers to achieve ~128 total connections)." -msgstr "" +msgstr "**sagemaker_hyperpod_max_connections_per_host**: 每个 LMCache 引擎到单个守护进程地址（IP:端口）的最大 TCP 连接数（默认：128，最小：1）。“主机”指的是守护进程的网络地址，而不是客户端机器。对于今天典型的单守护进程设置，这与 max_connections 的效果相似。此参数支持未来的多守护进程配置，其中一个引擎可以连接多个守护进程以实现负载均衡。对于每个节点连接到同一守护进程的 N 个工作者，总连接数 = N × 此值。对于高吞吐量设置，按比例减少（例如，对于 8 个工作者设置为 16，以实现 ~128 个总连接）。" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:34 msgid "" "**sagemaker_hyperpod_timeout_ms**: Timeout for lease acquisition requests" " in milliseconds (default: 5000, minimum: 100)" -msgstr "" +msgstr "**sagemaker_hyperpod_timeout_ms**: 租约获取请求的超时时间（毫秒）（默认值：5000，最小值：100）" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:35 msgid "" "**sagemaker_hyperpod_lease_ttl_s**: Server-side lease timeout in seconds " "(default: 30.0)" -msgstr "" +msgstr "**sagemaker_hyperpod_lease_ttl_s**: 服务器端租约超时时间（单位：秒，默认值：30.0）" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:36 msgid "" "**sagemaker_hyperpod_put_stream_chunk_bytes**: Chunk size for streaming " "PUT requests in bytes (default: 65536, minimum: 1024)" -msgstr "" +msgstr "**sagemaker_hyperpod_put_stream_chunk_bytes**: 流式 PUT 请求的块大小（以字节为单位）（默认: 65536，最小: 1024）" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:37 msgid "" "**sagemaker_hyperpod_use_https**: Enable HTTPS instead of HTTP (default: " "False). **Note**: Ignored if ``remote_url`` already contains ``http://`` " "or ``https://`` protocol." -msgstr "" +msgstr "**sagemaker_hyperpod_use_https**: 启用 HTTPS 而不是 HTTP（默认: False）。**注意**: 如果 ``remote_url`` 已经包含 ``http://`` 或 ``https://`` 协议，则忽略此设置。" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:38 msgid "" "**save_chunk_meta**: Whether to save chunk metadata with data (set False " "for performance)" -msgstr "" +msgstr "**save_chunk_meta**: 是否将块元数据与数据一起保存（为提高性能设置为 False）" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:41 msgid "Kubernetes Deployment Requirements" -msgstr "" +msgstr "Kubernetes 部署要求" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:44 msgid "Environment Variable for Node IP" -msgstr "" +msgstr "节点 IP 的环境变量" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:46 msgid "" "Add the ``NODE_IP`` environment variable to resolve the local node's IP " "address:" -msgstr "" +msgstr "添加 ``NODE_IP`` 环境变量以解析本地节点的 IP 地址：" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:57 msgid "/dev/shm Volume Configuration" -msgstr "" +msgstr "/dev/shm 卷配置" #: ../../source/kv_cache/storage_backends/sagemaker_hyperpod.rst:59 msgid "" "SageMaker Hyperpod requires /dev/shm for high-performance shared memory " "operations:" -msgstr "" +msgstr "SageMaker Hyperpod 需要 /dev/shm 以进行高性能共享内存操作：" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/valkey.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/valkey.po index ca886813b97..8338380e0aa 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/valkey.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/valkey.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache/storage_backends/valkey.rst:2 msgid "Valkey" -msgstr "" +msgstr "Valkey" #: ../../source/kv_cache/storage_backends/valkey.rst:5 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/valkey.rst:7 msgid "" @@ -33,214 +33,214 @@ msgid "" "is a supported option for remote KV Cache offloading in LMCache. Some " "other remote backends are :doc:`Mooncake <./mooncake>`, :doc:`Redis " "<./redis>`, and :doc:`InfiniStore <./infinistore>`." -msgstr "" +msgstr "Valkey 是一个开源（BSD）高性能键值数据存储，并且是 LMCache 中支持的远程 KV Cache 卸载选项之一。其他一些远程后端包括 :doc:`Mooncake <./mooncake>`、:doc:`Redis <./redis>` 和 :doc:`InfiniStore <./infinistore>`。" #: ../../source/kv_cache/storage_backends/valkey.rst:11 msgid "Prerequisites" -msgstr "" +msgstr "先决条件" #: ../../source/kv_cache/storage_backends/valkey.rst:13 msgid "To use this connector, you need valkey-glide 2.0 or higher." -msgstr "" +msgstr "要使用此连接器，您需要 valkey-glide 2.0 或更高版本。" #: ../../source/kv_cache/storage_backends/valkey.rst:21 msgid "Configuration Reference" -msgstr "" +msgstr "配置参考" #: ../../source/kv_cache/storage_backends/valkey.rst:23 msgid "The following ``extra_config`` keys are supported:" -msgstr "" +msgstr "支持以下 ``extra_config`` 键：" #: ../../source/kv_cache/storage_backends/valkey.rst:29 msgid "Key" -msgstr "" +msgstr "键" #: ../../source/kv_cache/storage_backends/valkey.rst:30 msgid "Default" -msgstr "" +msgstr "默认" #: ../../source/kv_cache/storage_backends/valkey.rst:31 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/kv_cache/storage_backends/valkey.rst:32 msgid "``valkey_num_workers``" -msgstr "" +msgstr "``valkey_num_workers``" #: ../../source/kv_cache/storage_backends/valkey.rst:33 msgid "``8``" -msgstr "" +msgstr "``8``" #: ../../source/kv_cache/storage_backends/valkey.rst:34 msgid "Number of worker threads, each with its own GLIDE client connection." -msgstr "" +msgstr "每个 GLIDE 客户端连接的工作线程数量。" #: ../../source/kv_cache/storage_backends/valkey.rst:35 msgid "``valkey_mode``" -msgstr "" +msgstr "``valkey_mode``" #: ../../source/kv_cache/storage_backends/valkey.rst:36 msgid "``\"standalone\"``" -msgstr "" +msgstr "``\"standalone\"``" #: ../../source/kv_cache/storage_backends/valkey.rst:37 msgid "" "``\"standalone\"`` or ``\"cluster\"``. Cluster mode auto-discovers " "topology from a seed node." -msgstr "" +msgstr "``\"standalone\"`` 或 ``\"cluster\"``。集群模式从种子节点自动发现拓扑。" #: ../../source/kv_cache/storage_backends/valkey.rst:38 msgid "``tls_enable``" -msgstr "" +msgstr "``tls_enable``" #: ../../source/kv_cache/storage_backends/valkey.rst:39 msgid "``false``" -msgstr "" +msgstr "``false``" #: ../../source/kv_cache/storage_backends/valkey.rst:40 msgid "Enable TLS. Required for ElastiCache Serverless." -msgstr "" +msgstr "启用 TLS。ElastiCache Serverless 需要此项。" #: ../../source/kv_cache/storage_backends/valkey.rst:41 msgid "``valkey_username``" -msgstr "" +msgstr "``valkey_username``" #: ../../source/kv_cache/storage_backends/valkey.rst:42 #: ../../source/kv_cache/storage_backends/valkey.rst:45 msgid "``\"\"``" -msgstr "" +msgstr "``\"\"``" #: ../../source/kv_cache/storage_backends/valkey.rst:43 msgid "Authentication username." -msgstr "" +msgstr "认证用户名。" #: ../../source/kv_cache/storage_backends/valkey.rst:44 msgid "``valkey_password``" -msgstr "" +msgstr "``valkey_password``" #: ../../source/kv_cache/storage_backends/valkey.rst:46 msgid "Authentication password." -msgstr "" +msgstr "认证密码。" #: ../../source/kv_cache/storage_backends/valkey.rst:47 msgid "``valkey_database``" -msgstr "" +msgstr "``valkey_database``" #: ../../source/kv_cache/storage_backends/valkey.rst:48 msgid "None" -msgstr "" +msgstr "无" #: ../../source/kv_cache/storage_backends/valkey.rst:49 msgid "Database ID (standalone mode only, ignored in cluster mode)." -msgstr "" +msgstr "数据库 ID（仅在独立模式下使用，在集群模式下被忽略）。" #: ../../source/kv_cache/storage_backends/valkey.rst:50 msgid "``request_timeout``" -msgstr "" +msgstr "``request_timeout``" #: ../../source/kv_cache/storage_backends/valkey.rst:51 msgid "``5.0``" -msgstr "" +msgstr "``5.0``" #: ../../source/kv_cache/storage_backends/valkey.rst:52 msgid "" "GLIDE request timeout in seconds. Also used as the Python-side Future " "timeout." -msgstr "" +msgstr "GLIDE 请求超时时间（以秒为单位）。也用于 Python 端的 Future 超时。" #: ../../source/kv_cache/storage_backends/valkey.rst:53 msgid "``connection_timeout``" -msgstr "" +msgstr "``connection_timeout``" #: ../../source/kv_cache/storage_backends/valkey.rst:54 msgid "``10.0``" -msgstr "" +msgstr "``10.0``" #: ../../source/kv_cache/storage_backends/valkey.rst:55 msgid "GLIDE connection timeout in seconds for initial client connections." -msgstr "" +msgstr "初始客户端连接的 GLIDE 连接超时时间（以秒为单位）。" #: ../../source/kv_cache/storage_backends/valkey.rst:58 msgid "Example Configurations" -msgstr "" +msgstr "示例配置" #: ../../source/kv_cache/storage_backends/valkey.rst:61 msgid "Standalone-mode" -msgstr "" +msgstr "独立模式" #: ../../source/kv_cache/storage_backends/valkey.rst:64 msgid "Basic Valkey Configuration (Standalone-mode)" -msgstr "" +msgstr "基本 Valkey 配置（独立模式）" #: ../../source/kv_cache/storage_backends/valkey.rst:76 msgid "Standalone-mode Valkey Configuration with database" -msgstr "" +msgstr "带数据库的独立模式 Valkey 配置" #: ../../source/kv_cache/storage_backends/valkey.rst:89 msgid "Cluster-mode" -msgstr "" +msgstr "集群模式" #: ../../source/kv_cache/storage_backends/valkey.rst:92 msgid "Cluster-mode Valkey Configuration (Endpoint)" -msgstr "" +msgstr "集群模式 Valkey 配置（端点）" #: ../../source/kv_cache/storage_backends/valkey.rst:94 msgid "For example, the configuration endpoint in ElastiCache is as follows:" -msgstr "" +msgstr "例如，ElastiCache 中的配置端点如下：" #: ../../source/kv_cache/storage_backends/valkey.rst:96 msgid "..clustercfg..cache.amazonaws.com" -msgstr "" +msgstr "..clustercfg..cache.amazonaws.com" #: ../../source/kv_cache/storage_backends/valkey.rst:98 msgid "You need to add this DNS name in the ." -msgstr "" +msgstr "您需要在 中添加此 DNS 名称。" #: ../../source/kv_cache/storage_backends/valkey.rst:113 msgid "Cluster-mode Valkey Configuration (Nodes)" -msgstr "" +msgstr "集群模式 Valkey 配置（节点）" #: ../../source/kv_cache/storage_backends/valkey.rst:115 msgid "" "Nodes are deployed directly and configured in cluster mode without " "connecting them via DNS names (CNAME)." -msgstr "" +msgstr "节点直接部署并配置为集群模式，而无需通过 DNS 名称 (CNAME) 连接它们。" #: ../../source/kv_cache/storage_backends/valkey.rst:117 msgid "In this scenario, you simply input multiple IP hosts and ports." -msgstr "" +msgstr "在这种情况下，您只需输入多个 IP 主机和端口。" #: ../../source/kv_cache/storage_backends/valkey.rst:119 msgid "Example: 172.0.0.1:7001, 172.0.0.2:7002 ... 172.0.0.6:7006" -msgstr "" +msgstr "示例：172.0.0.1:7001, 172.0.0.2:7002 ... 172.0.0.6:7006" #: ../../source/kv_cache/storage_backends/valkey.rst:132 msgid "" "Cluster-mode Valkey Configuration with numbered databases (Valkey 9.0+ " "and Valkey-GLIDE 2.1+)" -msgstr "" +msgstr "带编号数据库的集群模式 Valkey 配置（Valkey 9.0+ 和 Valkey-GLIDE 2.1+）" #: ../../source/kv_cache/storage_backends/valkey.rst:134 msgid "" "Valkey connector supports numbered databases in both the Endpoint using " "DNS and the Nodes method using IP and port pairs." -msgstr "" +msgstr "Valkey 连接器支持在使用 DNS 的端点和使用 IP 和端口对的节点方法中使用编号数据库。" #: ../../source/kv_cache/storage_backends/valkey.rst:148 msgid "TLS / ElastiCache Serverless" -msgstr "" +msgstr "TLS / ElastiCache Serverless" #: ../../source/kv_cache/storage_backends/valkey.rst:150 msgid "ElastiCache Serverless requires TLS. Set ``tls_enable: true``:" -msgstr "" +msgstr "ElastiCache Serverless 需要 TLS。设置 ``tls_enable: true``：" #: ../../source/kv_cache/storage_backends/valkey.rst:163 msgid "Performance Tuning" -msgstr "" +msgstr "性能调优" #: ../../source/kv_cache/storage_backends/valkey.rst:165 msgid "" "For large models (e.g., 70B with TP=8), increase the worker count for " "higher throughput:" -msgstr "" +msgstr "对于大型模型（例如，70B，TP=8），增加工作线程数量以提高吞吐量：" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/weka.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/weka.po index a4813b3efe5..c7501191cac 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/weka.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/weka.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache/storage_backends/weka.rst:2 msgid "Weka" -msgstr "" +msgstr "Weka" #: ../../source/kv_cache/storage_backends/weka.rst:7 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache/storage_backends/weka.rst:9 msgid "" @@ -34,31 +34,31 @@ msgid "" "filesystem backend can work with a WekaFS mount, this particular backend " "is optimized for Weka's characteristics. It leverages GPUDirect Storage " "for I/O and it allows data-sharing between multiple LMCache instances." -msgstr "" +msgstr "WekaFS 是一个高性能的分布式文件系统，是 LMCache 中 KV Cache 卸载的支持选项。尽管本地文件系统后端可以与 WekaFS 挂载一起工作，但该后端针对 Weka 的特性进行了优化。它利用 GPUDirect Storage 进行 I/O，并允许多个 LMCache 实例之间的数据共享。" #: ../../source/kv_cache/storage_backends/weka.rst:15 msgid "Ways to configure LMCache WEKA Offloading" -msgstr "" +msgstr "配置 LMCache WEKA 卸载的方法" #: ../../source/kv_cache/storage_backends/weka.rst:17 msgid "**1. Environment Variables:**" -msgstr "" +msgstr "**1. 环境变量：**" #: ../../source/kv_cache/storage_backends/weka.rst:33 msgid "**2. Configuration File**:" -msgstr "" +msgstr "**2. 配置文件**:" #: ../../source/kv_cache/storage_backends/weka.rst:35 msgid "Passed in through ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml``" -msgstr "" +msgstr "通过 ``LMCACHE_CONFIG_FILE=your-lmcache-config.yaml`` 传入" #: ../../source/kv_cache/storage_backends/weka.rst:37 msgid "Example ``config.yaml``:" -msgstr "" +msgstr "示例 ``config.yaml``:" #: ../../source/kv_cache/storage_backends/weka.rst:54 msgid "GDS Buffer Size Explanation" -msgstr "" +msgstr "GDS 缓冲区大小说明" #: ../../source/kv_cache/storage_backends/weka.rst:56 msgid "" @@ -69,64 +69,64 @@ msgid "" "has 80GiBs of VRAM would be to start with 8GiB and set ``--gpu-memory-" "utilization 0.85`` and depending on your workflow fine-tune it from " "there." -msgstr "" +msgstr "后端目前预先注册缓冲区空间以加速 GDS 操作。此缓冲区空间在显存中注册，因此在设置时应考虑 ``--gpu-memory-utilization`` 选项，例如来自 ``vllm`` 的选项。例如，对于通常具有 80GiB 显存的 H100，一个好的经验法则是从 8GiB 开始，并设置 ``--gpu-memory-utilization 0.85``，然后根据您的工作流程进行微调。" #: ../../source/kv_cache/storage_backends/weka.rst:64 msgid "Setup Example" -msgstr "" +msgstr "设置示例" #: ../../source/kv_cache/storage_backends/weka.rst:68 msgid "**Prerequisites:**" -msgstr "" +msgstr "**前提条件：**" #: ../../source/kv_cache/storage_backends/weka.rst:70 msgid "" "A Machine with at least one GPU. You can adjust the max model length of " "your vllm instance depending on your GPU memory." -msgstr "" +msgstr "一台至少配备一个 GPU 的机器。您可以根据您的显存调整 vllm 实例的最大模型长度。" #: ../../source/kv_cache/storage_backends/weka.rst:72 msgid "Weka already installed and mounted." -msgstr "" +msgstr "Weka 已经安装并挂载。" #: ../../source/kv_cache/storage_backends/weka.rst:74 msgid "" "vllm and lmcache installed (:doc:`Installation Guide " "<../../getting_started/installation>`)" -msgstr "" +msgstr "vllm 和 lmcache 已安装 (:doc:`安装指南 <../../getting_started/installation>`)" #: ../../source/kv_cache/storage_backends/weka.rst:76 msgid "Hugging Face access to ``meta-llama/Llama-3.1-8B-Instruct``" -msgstr "" +msgstr "Hugging Face 访问 ``meta-llama/Llama-3.1-8B-Instruct``" #: ../../source/kv_cache/storage_backends/weka.rst:82 msgid "**Step 1. Create cache directory under your Weka mount:**" -msgstr "" +msgstr "**步骤 1. 在您的 Weka 挂载下创建缓存目录：**" #: ../../source/kv_cache/storage_backends/weka.rst:84 msgid "To find all your WekaFS mounts run:" -msgstr "" +msgstr "要查找所有 WekaFS 挂载，请运行：" #: ../../source/kv_cache/storage_backends/weka.rst:90 msgid "For the sake of this example let's say that the above returns:" -msgstr "" +msgstr "为了这个例子，我们假设上面的返回结果是：" #: ../../source/kv_cache/storage_backends/weka.rst:96 msgid "Then create a directory under it (the name here is arbitrary):" -msgstr "" +msgstr "然后在其下创建一个目录（这里的名称是任意的）：" #: ../../source/kv_cache/storage_backends/weka.rst:102 msgid "**Step 2. Start a vLLM server with Weka offloading enabled:**" -msgstr "" +msgstr "**步骤 2. 启动一个启用 Weka 卸载的 vLLM 服务器：**" #: ../../source/kv_cache/storage_backends/weka.rst:104 msgid "Create a an lmcache configuration file called: ``weka-offload.yaml``" -msgstr "" +msgstr "创建一个名为 ``weka-offload.yaml`` 的 lmcache 配置文件" #: ../../source/kv_cache/storage_backends/weka.rst:115 msgid "" "If you don't want to use a config file, uncomment the first three " "environment variables and then comment out the ``LMCACHE_CONFIG_FILE`` " "below:" -msgstr "" +msgstr "如果您不想使用配置文件，请取消注释前面三个环境变量，然后注释掉下面的 ``LMCACHE_CONFIG_FILE``：" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/check_finish.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/check_finish.po index c95ce151b41..e1b52fa0782 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/check_finish.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/check_finish.po @@ -21,9 +21,9 @@ msgstr "" #: ../../source/kv_cache_management/check_finish.rst:4 msgid "Check finish of a control event" -msgstr "" +msgstr "检查控制事件的完成情况" #: ../../source/kv_cache_management/check_finish.rst:6 msgid "Coming soon..." -msgstr "" +msgstr "敬请期待..." diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/clear.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/clear.po index 30408c83f55..8226e12af3a 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/clear.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/clear.po @@ -21,59 +21,59 @@ msgstr "" #: ../../source/kv_cache_management/clear.rst:4 msgid "Clear the KV cache" -msgstr "" +msgstr "清除 KV Cache" #: ../../source/kv_cache_management/clear.rst:6 msgid "The ``clear`` interface is defined as the following:" -msgstr "" +msgstr "``clear`` 接口定义如下：" #: ../../source/kv_cache_management/clear.rst:12 msgid "" "The function removes the KV cache stored at ``location`` for the " "specified ``instance_id``. It returns an ``event_id`` and the number of " "tokens scheduled for clearing." -msgstr "" +msgstr "该函数会移除指定 ``instance_id`` 在 ``location`` 存储的 KV Cache。它返回一个 ``event_id`` 和计划清除的令牌数量。" #: ../../source/kv_cache_management/clear.rst:17 msgid "Example usage:" -msgstr "" +msgstr "示例用法：" #: ../../source/kv_cache_management/clear.rst:19 msgid "" "First, create a yaml file ``example.yaml`` to configure the lmcache " "instance:" -msgstr "" +msgstr "首先，创建一个 yaml 文件 ``example.yaml`` 来配置 lmcache 实例：" #: ../../source/kv_cache_management/clear.rst:37 msgid "Start the vllm/lmcache instance at port 8000:" -msgstr "" +msgstr "在 8000 端口启动 vllm/lmcache 实例：" #: ../../source/kv_cache_management/clear.rst:44 msgid "Start the lmcache controller at port 9000 and the monitor at port 9001:" -msgstr "" +msgstr "在9000端口启动lmcache控制器，在9001端口启动监视器：" #: ../../source/kv_cache_management/clear.rst:50 msgid "Send a request to vllm:" -msgstr "" +msgstr "发送请求到 vllm：" #: ../../source/kv_cache_management/clear.rst:62 msgid "Clear the KV cache in the system:" -msgstr "" +msgstr "清除系统中的 KV Cache：" #: ../../source/kv_cache_management/clear.rst:74 msgid "The controller responds with a message similar to:" -msgstr "" +msgstr "控制器会回复类似于以下消息：" #: ../../source/kv_cache_management/clear.rst:80 msgid "" "This indicates that the KV cache for 12 tokens has been scheduled for " "clearing. We can verify the cache has been cleared by performing a " "lookup:" -msgstr "" +msgstr "这表明 12 个令牌的 KV Cache 已被安排清除。我们可以通过执行查找来验证缓存是否已被清除：" #: ../../source/kv_cache_management/clear.rst:91 msgid "" "The lookup should return an empty result, confirming that the KV cache " "has been cleared for the given tokens." -msgstr "" +msgstr "查找应该返回一个空结果，确认给定的令牌的 KV Cache 已被清除。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/compress.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/compress.po index 6c50ee49847..f0609241095 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/compress.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/compress.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache_management/compress.rst:4 msgid "Compress and Decompress the KV cache" -msgstr "" +msgstr "压缩和解压缩 KV Cache" #: ../../source/kv_cache_management/compress.rst:6 msgid "The ``compress`` interface is defined as the following:" -msgstr "" +msgstr "``compress`` 接口定义如下：" #: ../../source/kv_cache_management/compress.rst:13 msgid "" @@ -33,62 +33,62 @@ msgid "" "by ``tokens`` using the given ``method`` in the storage ``location``. The" " controller returns an ``event_id`` and the number of tokens scheduled " "for compression or decompression." -msgstr "" +msgstr "这两个函数使用存储位置中的给定方法对由 ``tokens`` 指定的 KV Cache 块进行压缩/解压缩。控制器返回一个 ``event_id`` 和计划进行压缩或解压缩的令牌数量。" #: ../../source/kv_cache_management/compress.rst:17 msgid "Example usage:" -msgstr "" +msgstr "示例用法：" #: ../../source/kv_cache_management/compress.rst:19 msgid "" "First, we need a yaml file ``example.yaml`` to properly configure the " "lmcache instance:" -msgstr "" +msgstr "首先，我们需要一个 yaml 文件 ``example.yaml`` 来正确配置 lmcache 实例：" #: ../../source/kv_cache_management/compress.rst:37 msgid "Second, we need to start the vllm/lmcache instance at port 8000:" -msgstr "" +msgstr "第二，我们需要在 8000 端口启动 vllm/lmcache 实例：" #: ../../source/kv_cache_management/compress.rst:43 msgid "" "Third, we need to start the lmcache controller at port 9000 and the " "monitor at port 9001:" -msgstr "" +msgstr "第三，我们需要在 9000 端口启动 lmcache 控制器，在 9001 端口启动监控器：" #: ../../source/kv_cache_management/compress.rst:49 msgid "Then we can send a request to vllm to see if it works properly:" -msgstr "" +msgstr "然后我们可以向 vllm 发送请求，以查看它是否正常工作：" #: ../../source/kv_cache_management/compress.rst:61 msgid "Now we send a request to tokenize the prompt:" -msgstr "" +msgstr "现在我们发送请求以对提示进行标记：" #: ../../source/kv_cache_management/compress.rst:72 msgid "We should be able to see token ids in response:" -msgstr "" +msgstr "我们应该能够在响应中看到令牌 ID：" #: ../../source/kv_cache_management/compress.rst:78 msgid "After all, we issue a ``compress`` request:" -msgstr "" +msgstr "毕竟，我们发出了一个 ``compress`` 请求：" #: ../../source/kv_cache_management/compress.rst:91 #: ../../source/kv_cache_management/compress.rst:112 msgid "The controller responds with a message similar to:" -msgstr "" +msgstr "控制器的响应消息类似于：" #: ../../source/kv_cache_management/compress.rst:97 msgid "" "This indicates that 12 tokens are being compressed. The ``event_id`` can " "be used to query the status of the operation." -msgstr "" +msgstr "这表明正在压缩 12 个 token。``event_id`` 可用于查询操作的状态。" #: ../../source/kv_cache_management/compress.rst:99 msgid "Once the kv cache is compressed, we can use cachegen to decompress" -msgstr "" +msgstr "一旦 KV Cache 被压缩，我们可以使用 cachegen 进行解压。" #: ../../source/kv_cache_management/compress.rst:118 msgid "" "This indicates that 12 tokens are being decompressed. The ``event_id`` " "can be used to query the status of the operation." -msgstr "" +msgstr "这表示正在解压缩 12 个令牌。``event_id`` 可用于查询操作的状态。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/health.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/health.po index 60da02dcf7e..ac3ff5689d4 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/health.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/health.po @@ -21,40 +21,40 @@ msgstr "" #: ../../source/kv_cache_management/health.rst:4 msgid "Check controller health" -msgstr "" +msgstr "检查控制器健康状态" #: ../../source/kv_cache_management/health.rst:6 msgid "The ``health`` interface is defined as the following:" -msgstr "" +msgstr "``health`` 接口定义如下：" #: ../../source/kv_cache_management/health.rst:12 msgid "" "The function returns an ``event_id`` and a dictionary mapping " "``worker_id`` to ``error_code``. A value of ``0`` indicates a healthy " "worker, while a non-zero value signals an error." -msgstr "" +msgstr "该函数返回一个 ``event_id`` 和一个将 ``worker_id`` 映射到 ``error_code`` 的字典。值为 ``0`` 表示工作正常，而非零值则表示出现错误。" #: ../../source/kv_cache_management/health.rst:17 msgid "Example usage:" -msgstr "" +msgstr "示例用法：" #: ../../source/kv_cache_management/health.rst:19 msgid "" "First, start the lmcache controller at port 9000 and the monitor at port " "9001:" -msgstr "" +msgstr "首先，在 9000 端口启动 lmcache 控制器，在 9001 端口启动监视器：" #: ../../source/kv_cache_management/health.rst:25 msgid "Then send a health check request:" -msgstr "" +msgstr "然后发送健康检查请求：" #: ../../source/kv_cache_management/health.rst:33 msgid "The controller responds with a message similar to the following:" -msgstr "" +msgstr "控制器会返回类似于以下内容的消息：" #: ../../source/kv_cache_management/health.rst:39 msgid "" "Here ``error_codes`` lists each worker's ``error_code``. ``0`` represents" " a healthy worker, while non-zero values indicate an error." -msgstr "" +msgstr "这里 ``error_codes`` 列出了每个工作者的 ``error_code``。 ``0`` 表示工作者健康，而非零值则表示出现错误。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/index.po index 7fbe7d24754..63d3d35f238 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/index.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/index.po @@ -21,74 +21,74 @@ msgstr "" #: ../../source/kv_cache_management/index.rst:2 msgid "LMCache Controller" -msgstr "" +msgstr "LMCache 控制器" #: ../../source/kv_cache_management/index.rst:5 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/kv_cache_management/index.rst:6 msgid "" "The overall architecture of the LMCache Controller is shown in the " "figure, mainly consisting of two parts: the Controller Manager and " "LMCache Worker." -msgstr "" +msgstr "LMCache 控制器的整体架构如图所示，主要由两个部分组成：控制器管理器和 LMCache 工作线程。" #: ../../source/kv_cache_management/index.rst:9 msgid "" "The Controller Manager mainly consists of KV Controller, Reg Controller, " "and Cluster Executor." -msgstr "" +msgstr "控制器管理器主要由 KV 控制器、注册控制器和集群执行器组成。" #: ../../source/kv_cache_management/index.rst:11 msgid "" "KV Controller: The KV Controller handles the chunk information reported " "by LMCache Workers, and lookup requests query chunk information from the " "KV Controller." -msgstr "" +msgstr "KV Controller：KV Controller 处理由 LMCache Workers 报告的块信息，并且查找请求从 KV Controller 查询块信息。" #: ../../source/kv_cache_management/index.rst:12 msgid "" "Reg Controller: The Reg Controller is responsible for handling " "register/deregister/heartbeat requests from LMCache Workers." -msgstr "" +msgstr "Reg Controller: Reg Controller 负责处理来自 LMCache Workers 的注册/注销/心跳请求。" #: ../../source/kv_cache_management/index.rst:13 msgid "" "Cluster Executor: When the Controller Manager receives user requests, " "such as Clear or Move, it sends the corresponding commands to LMCache " "Workers through the Cluster Executor." -msgstr "" +msgstr "集群执行器：当控制器管理器接收到用户请求，例如清除或移动时，它通过集群执行器将相应的命令发送给 LMCache 工作线程。" #: ../../source/kv_cache_management/index.rst:15 msgid "" "The LMCache Worker is a thread within a rank process, which is " "responsible for the following tasks:" -msgstr "" +msgstr "LMCache Worker 是一个在排名进程中的线程，负责以下任务：" #: ../../source/kv_cache_management/index.rst:17 msgid "sends register, deregister, heartbeat to the Reg Controller." -msgstr "" +msgstr "向注册控制器发送注册、注销和心跳。" #: ../../source/kv_cache_management/index.rst:18 msgid "" "send chunk information to the KV Controller, which include admit and " "evict message." -msgstr "" +msgstr "将块信息发送到 KV 控制器，包括接纳和逐出消息。" #: ../../source/kv_cache_management/index.rst:19 msgid "" "listens on a port to receive commands from the Cluster Executor and " "performs corresponding processing." -msgstr "" +msgstr "在一个端口上监听以接收来自集群执行器的命令，并执行相应的处理。" #: ../../source/kv_cache_management/index.rst:21 msgid "LMCache Controller Architecture Diagram" -msgstr "" +msgstr "LMCache 控制器架构图" #: ../../source/kv_cache_management/index.rst:25 msgid "P2P Related" -msgstr "" +msgstr "P2P 相关" #: ../../source/kv_cache_management/index.rst:27 msgid "" @@ -96,119 +96,119 @@ msgid "" "enabled. The LMCache Controller serves as the central node and stores " "information for each chunk. The ``P2PBackend`` queries chunk information " "from the LMCache Controller and performs data transmission through NIXL." -msgstr "" +msgstr "如果启用 ``enable_p2p``，则必须启用 LMCache 控制器。LMCache 控制器作为中央节点，存储每个块的信息。``P2PBackend`` 从 LMCache 控制器查询块信息，并通过 NIXL 执行数据传输。" #: ../../source/kv_cache_management/index.rst:33 msgid "Key Features" -msgstr "" +msgstr "关键特性" #: ../../source/kv_cache_management/index.rst:35 msgid "Exposes a set of APIs for users and orchestrators to manage the KV cache." -msgstr "" +msgstr "暴露一组 API 供用户和调度器管理 KV Cache。" #: ../../source/kv_cache_management/index.rst:37 msgid "Currently, the controller provides the following APIs:" -msgstr "" +msgstr "目前，控制器提供以下 API：" #: ../../source/kv_cache_management/index.rst:39 msgid ":ref:`Clear `: Clear the KV caches." -msgstr "" +msgstr ":ref:`清除 `: 清除 KV 缓存。" #: ../../source/kv_cache_management/index.rst:40 msgid ":ref:`Compress `: Compress the KV cache." -msgstr "" +msgstr ":ref:`压缩 `: 压缩 KV Cache。" #: ../../source/kv_cache_management/index.rst:41 msgid ":ref:`Health `: Check the health status of cache workers." -msgstr "" +msgstr "`:ref:`Health `: 检查缓存工作线程的健康状态。`" #: ../../source/kv_cache_management/index.rst:42 msgid ":ref:`Lookup `: Lookup the KV cache for a given list of tokens." -msgstr "" +msgstr ":ref:`查找 `: 在 KV Cache 中查找给定的令牌列表。" #: ../../source/kv_cache_management/index.rst:43 msgid ":ref:`Move `: Move the KV cache to a different location." -msgstr "" +msgstr ":ref:`移动 `: 将 KV Cache 移动到不同的位置。" #: ../../source/kv_cache_management/index.rst:44 msgid ":ref:`Pin `: Persist the KV cache to prevent it from being evicted." -msgstr "" +msgstr ":ref:`Pin `: 持久化 KV Cache 以防止其被逐出。" #: ../../source/kv_cache_management/index.rst:45 msgid "" ":ref:`CheckFinish `: Check whether a (non-blocking) control" " event has finished or not." -msgstr "" +msgstr "`:ref:`CheckFinish `: 检查一个（非阻塞）控制事件是否已经完成。`" #: ../../source/kv_cache_management/index.rst:46 msgid ":ref:`QueryWorkerInfo `: Query the worker info." -msgstr "" +msgstr ":ref:`QueryWorkerInfo `: 查询工作者信息。" #: ../../source/kv_cache_management/index.rst:48 msgid "Interacts with the LMCache worker." -msgstr "" +msgstr "与 LMCache 工作线程交互。" #: ../../source/kv_cache_management/index.rst:50 msgid "Currently, the LMCache worker supports the following functions:" -msgstr "" +msgstr "目前，LMCache 工作线程支持以下功能：" #: ../../source/kv_cache_management/index.rst:52 msgid "register with the controller" -msgstr "" +msgstr "向控制器注册" #: ../../source/kv_cache_management/index.rst:53 msgid "deregister from the controller" -msgstr "" +msgstr "从控制器注销" #: ../../source/kv_cache_management/index.rst:54 msgid "heartbeat" -msgstr "" +msgstr "心跳" #: ../../source/kv_cache_management/index.rst:55 msgid "admit or evict chunk information(LocalCPUBackend or LocalDiskBackend)" -msgstr "" +msgstr "接受或逐出块信息（LocalCPUBackend 或 LocalDiskBackend）" #: ../../source/kv_cache_management/index.rst:59 msgid "Quick Start" -msgstr "" +msgstr "快速开始" #: ../../source/kv_cache_management/index.rst:61 msgid "**Start the Controller**" -msgstr "" +msgstr "**启动控制器**" #: ../../source/kv_cache_management/index.rst:67 msgid "Expected output:" -msgstr "" +msgstr "预期输出：" #: ../../source/kv_cache_management/index.rst:82 msgid "**Controller Configuration**" -msgstr "" +msgstr "**控制器配置**" #: ../../source/kv_cache_management/index.rst:84 msgid "--host: default is 0.0.0.0" -msgstr "" +msgstr "--host: 默认值为 0.0.0.0" #: ../../source/kv_cache_management/index.rst:85 msgid "" "--port: default is 9000, the externally exposed port through which " "interfaces like lookup can be accessed via this port." -msgstr "" +msgstr "--port: 默认值为 9000，通过此端口可以访问诸如查找等接口。" #: ../../source/kv_cache_management/index.rst:86 msgid "" "--monitor-port: default is 9001, the port through which LMCache Worker " "communicates with Controller Manager (deprecated, indicates the pull port" " in --monitor-ports, reply port is None)." -msgstr "" +msgstr "--monitor-port: 默认值为 9001，LMCache Worker 与 Controller Manager 通信的端口（已弃用，表示 --monitor-ports 中的拉取端口，回复端口为 None）。" #: ../../source/kv_cache_management/index.rst:87 #, python-brace-format msgid "" "--monitor-ports: default is None, if configured, requires a JSON format " "string input such as ``{\"pull\": 8300, \"reply\": 8400}``." -msgstr "" +msgstr "--monitor-ports: 默认值为 None，如果配置，则需要输入 JSON 格式的字符串，例如 ``{\\\"pull\\\": 8300, \\\"reply\\\": 8400}``。" #: ../../source/kv_cache_management/index.rst:89 msgid "**YAML Configuration**" -msgstr "" +msgstr "**YAML 配置**" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/lookup.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/lookup.po index c8c59154996..8737f3b558b 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/lookup.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/lookup.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache_management/lookup.rst:4 msgid "Lookup the KV cache" -msgstr "" +msgstr "查找 KV Cache" #: ../../source/kv_cache_management/lookup.rst:6 msgid "The ``lookup`` interface is defined as the following:" -msgstr "" +msgstr "``lookup`` 接口定义如下：" #: ../../source/kv_cache_management/lookup.rst:12 msgid "" @@ -33,47 +33,47 @@ msgid "" "dictionary containing the layout information for each token. The layout " "information is represented as a mapping between ``instance_id`` and a " "tuple of ``(location, matched_prefix_length)``." -msgstr "" +msgstr "该函数接受一个令牌列表作为输入，并返回 event_id 以及一个包含每个令牌布局信息的字典。布局信息表示为 ``instance_id`` 与 ``(location, matched_prefix_length)`` 元组之间的映射。" #: ../../source/kv_cache_management/lookup.rst:16 msgid "Example usage:" -msgstr "" +msgstr "示例用法：" #: ../../source/kv_cache_management/lookup.rst:18 msgid "" "First, we need a yaml file ``example.yaml`` to properly configure the " "lmcache instance:" -msgstr "" +msgstr "首先，我们需要一个 yaml 文件 ``example.yaml`` 来正确配置 lmcache 实例：" #: ../../source/kv_cache_management/lookup.rst:36 msgid "Second, we need to start the vllm/lmcache instance at port 8000:" -msgstr "" +msgstr "第二，我们需要在 8000 端口启动 vllm/lmcache 实例：" #: ../../source/kv_cache_management/lookup.rst:42 msgid "" "Third, we need to start the lmcache controller at port 9000 and the " "monitor at port 9001:" -msgstr "" +msgstr "第三，我们需要在 9000 端口启动 lmcache 控制器，在 9001 端口启动监视器：" #: ../../source/kv_cache_management/lookup.rst:48 msgid "Then we can send a request to vllm to see if it works properly:" -msgstr "" +msgstr "然后我们可以向 vllm 发送请求，以查看它是否正常工作：" #: ../../source/kv_cache_management/lookup.rst:60 msgid "Now we send a request to tokenize the prompt:" -msgstr "" +msgstr "现在我们发送请求以对提示进行标记化：" #: ../../source/kv_cache_management/lookup.rst:71 msgid "We should be able to see token ids in response:" -msgstr "" +msgstr "我们应该能够在响应中看到令牌 ID：" #: ../../source/kv_cache_management/lookup.rst:77 msgid "Finally, we can send a ``lookup`` request to the lmcache controller:" -msgstr "" +msgstr "最后，我们可以向 lmcache 控制器发送一个 ``lookup`` 请求：" #: ../../source/kv_cache_management/lookup.rst:87 msgid "We should be able to see the response like this:" -msgstr "" +msgstr "我们应该能够看到这样的响应：" #: ../../source/kv_cache_management/lookup.rst:93 msgid "" @@ -81,5 +81,5 @@ msgid "" " a tuple of ``(location, matched_prefix_length)`` indicating the cache " "location within that instance and matched prefix length. ``event_id`` is " "an identifier of the controller operation and can typically be ignored." -msgstr "" +msgstr "字段 ``lmcache_default_instance`` 显示实例 ID，后面是一个元组 ``(location, matched_prefix_length)``，指示该实例内的缓存位置和匹配前缀长度。 ``event_id`` 是控制器操作的标识符，通常可以忽略。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/move.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/move.po index 043ddc6c4d5..84db45b6dbb 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/move.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/move.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/kv_cache_management/move.rst:4 msgid "Move the KV cache" -msgstr "" +msgstr "移动 KV Cache" #: ../../source/kv_cache_management/move.rst:6 msgid "The ``move`` interface is defined as the following:" -msgstr "" +msgstr "``move`` 接口定义如下：" #: ../../source/kv_cache_management/move.rst:13 msgid "" @@ -33,53 +33,53 @@ msgid "" "``old_position`` to ``new_position``. Each position is a tuple of " "``(instance_id, location)``. Setting ``copy`` to ``True`` copies the KV " "cache instead of moving it." -msgstr "" +msgstr "该函数将由 ``tokens`` 标识的 KV Cache 块从 ``old_position`` 移动到 ``new_position``。每个位置是一个 ``(instance_id, location)`` 的元组。将 ``copy`` 设置为 ``True`` 会复制 KV Cache，而不是移动它。" #: ../../source/kv_cache_management/move.rst:18 msgid "" "Note that NIXL is required to be installed for P2P transfer. We'll " "support other transports later such as Python socket and Mooncake." -msgstr "" +msgstr "请注意，必须安装 NIXL 才能进行 P2P 传输。我们稍后将支持其他传输方式，例如 Python 套接字和 Mooncake。" #: ../../source/kv_cache_management/move.rst:22 msgid "Example usage:" -msgstr "" +msgstr "示例用法：" #: ../../source/kv_cache_management/move.rst:24 msgid "" "First, prepare two yaml files ``instance1.yaml`` and ``instance2.yaml`` " "to configure two lmcache instances:" -msgstr "" +msgstr "首先，准备两个 yaml 文件 ``instance1.yaml`` 和 ``instance2.yaml`` 来配置两个 lmcache 实例：" #: ../../source/kv_cache_management/move.rst:70 msgid "Start two vllm engines:" -msgstr "" +msgstr "启动两个 vllm 引擎：" #: ../../source/kv_cache_management/move.rst:82 msgid "Start the lmcache controller at port 9000 and the monitor at port 9001:" -msgstr "" +msgstr "在9000端口启动 lmcache 控制器，在9001端口启动监视器：" #: ../../source/kv_cache_management/move.rst:88 msgid "Send a request to vllm engine 1:" -msgstr "" +msgstr "向 vllm 引擎 1 发送请求：" #: ../../source/kv_cache_management/move.rst:100 msgid "Tokenize the prompt to obtain token ids:" -msgstr "" +msgstr "将提示进行分词以获取令牌 ID：" #: ../../source/kv_cache_management/move.rst:111 msgid "" "Move the KV cache from engine 1's CPU to engine 2's CPU using the token " "ids:" -msgstr "" +msgstr "将 KV Cache 从引擎 1 的 CPU 移动到引擎 2 的 CPU，使用的 token ids 为：" #: ../../source/kv_cache_management/move.rst:123 msgid "The controller responds with a message similar to:" -msgstr "" +msgstr "控制器会返回类似于以下内容的消息：" #: ../../source/kv_cache_management/move.rst:129 msgid "" "``num_tokens`` indicates how many tokens' KV cache are being moved. The " "returned ``event_id`` can be used to query the status of the operation." -msgstr "" +msgstr "``num_tokens`` 表示正在移动多少个 token 的 KV Cache。返回的 ``event_id`` 可用于查询操作的状态。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/pin.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/pin.po index 44902c1dd6c..82158226947 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/pin.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/pin.po @@ -21,56 +21,56 @@ msgstr "" #: ../../source/kv_cache_management/pin.rst:4 msgid "Pin the KV cache" -msgstr "" +msgstr "固定 KV Cache" #: ../../source/kv_cache_management/pin.rst:6 msgid "The ``pin`` interface is defined as the following:" -msgstr "" +msgstr "``pin`` 接口定义如下：" #: ../../source/kv_cache_management/pin.rst:12 msgid "" "The function pins (persists) the KV cache chunks specified by ``tokens`` " "in the given ``location`` of the ``instance_id``. The controller returns " "an ``event_id`` and the number of tokens scheduled for pinning." -msgstr "" +msgstr "该函数将指定的 ``tokens`` 在给定的 ``instance_id`` 的 ``location`` 中固定（持久化）KV Cache 块。控制器返回一个 ``event_id`` 和计划固定的令牌数量。" #: ../../source/kv_cache_management/pin.rst:17 msgid "Example usage:" -msgstr "" +msgstr "示例用法：" #: ../../source/kv_cache_management/pin.rst:19 msgid "" "First, create a yaml file ``example.yaml`` to configure the lmcache " "instance:" -msgstr "" +msgstr "首先，创建一个 yaml 文件 ``example.yaml`` 来配置 lmcache 实例：" #: ../../source/kv_cache_management/pin.rst:38 msgid "Start the vllm/lmcache instance at port 8000:" -msgstr "" +msgstr "在 8000 端口启动 vllm/lmcache 实例：" #: ../../source/kv_cache_management/pin.rst:45 msgid "Start the lmcache controller at port 9000 and the monitor at port 9001:" -msgstr "" +msgstr "在9000端口启动lmcache控制器，在9001端口启动监视器：" #: ../../source/kv_cache_management/pin.rst:51 msgid "Send a request to vllm:" -msgstr "" +msgstr "发送请求到 vllm：" #: ../../source/kv_cache_management/pin.rst:63 msgid "Tokenize the prompt to obtain token ids:" -msgstr "" +msgstr "将提示进行分词以获取令牌 ID：" #: ../../source/kv_cache_management/pin.rst:74 msgid "Pin the KV cache in the system:" -msgstr "" +msgstr "在系统中固定 KV Cache：" #: ../../source/kv_cache_management/pin.rst:86 msgid "The controller responds with a message similar to:" -msgstr "" +msgstr "控制器会回复类似于以下内容的消息：" #: ../../source/kv_cache_management/pin.rst:92 msgid "" "``num_tokens`` indicates how many tokens' KV cache are pinned. The " "returned ``event_id`` can be used to query the status of the operation." -msgstr "" +msgstr "``num_tokens`` 指示有多少个 token 的 KV Cache 被固定。返回的 ``event_id`` 可用于查询操作的状态。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/query_worker_info.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/query_worker_info.po index c405fcfb04a..811d951a560 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/query_worker_info.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/query_worker_info.po @@ -21,48 +21,48 @@ msgstr "" #: ../../source/kv_cache_management/query_worker_info.rst:4 msgid "Query Worker Info" -msgstr "" +msgstr "查询工作者信息" #: ../../source/kv_cache_management/query_worker_info.rst:6 msgid "The ``query_worker_info`` interface is defined as the following:" -msgstr "" +msgstr "``query_worker_info`` 接口定义如下：" #: ../../source/kv_cache_management/query_worker_info.rst:12 msgid "" "The function get the info of the workers which specified by " "``instance_id`` and ``worker_ids``. The controller returns an " "``event_id`` and the worker infos." -msgstr "" +msgstr "该函数获取由 ``instance_id`` 和 ``worker_ids`` 指定的工作者的信息。控制器返回一个 ``event_id`` 和工作者信息。" #: ../../source/kv_cache_management/query_worker_info.rst:16 msgid "Example usage:" -msgstr "" +msgstr "示例用法：" #: ../../source/kv_cache_management/query_worker_info.rst:18 msgid "" "First, create a yaml file ``example.yaml`` to configure the lmcache " "instance:" -msgstr "" +msgstr "首先，创建一个 yaml 文件 ``example.yaml`` 来配置 lmcache 实例：" #: ../../source/kv_cache_management/query_worker_info.rst:37 msgid "Start the vllm/lmcache instance at port 8000:" -msgstr "" +msgstr "在端口 8000 启动 vllm/lmcache 实例：" #: ../../source/kv_cache_management/query_worker_info.rst:44 msgid "Start the lmcache controller at port 9000 and the monitor at port 9001:" -msgstr "" +msgstr "在9000端口启动lmcache控制器，在9001端口启动监视器：" #: ../../source/kv_cache_management/query_worker_info.rst:50 msgid "Send a request to controller:" -msgstr "" +msgstr "发送请求到控制器：" #: ../../source/kv_cache_management/query_worker_info.rst:61 msgid "The controller responds with a message similar to:" -msgstr "" +msgstr "控制器会回复类似于以下内容的消息：" #: ../../source/kv_cache_management/query_worker_info.rst:67 msgid "" "``worker_infos`` contains the queried worker information. returned " "``event_id`` can be used to query the status of the operation." -msgstr "" +msgstr "``worker_infos`` 包含查询到的工作者信息。返回的 ``event_id`` 可用于查询操作的状态。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/blending.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/blending.po index 38ca824a692..15db547f0f2 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/blending.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/blending.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/kv_cache_optimizations/blending.rst:2 msgid "Blending" -msgstr "" +msgstr "混合" #: ../../source/kv_cache_optimizations/blending.rst:4 msgid "" @@ -29,21 +29,21 @@ msgid "" " a subset of tokens at non-prefix positions. For example, CacheBlend can " "combine multiple (pre-)computed KV caches, when their corresponding texts" " are concatenated in the LLM input" -msgstr "" +msgstr "CacheBlend 通过在非前缀位置重计算一部分令牌来实现 KV Cache 的重用。例如，当相应的文本在 LLM 输入中连接时，CacheBlend 可以组合多个（预）计算的 KV Cache。" #: ../../source/kv_cache_optimizations/blending.rst:8 msgid "Configuring CacheBlend in RAG scenarios" -msgstr "" +msgstr "在 RAG 场景中配置 CacheBlend" #: ../../source/kv_cache_optimizations/blending.rst:10 msgid "" "Here, we will explain the code in our end-to-end `example " "`_>." -msgstr "" +msgstr "在这里，我们将解释我们端到端的 `example `_ 中的代码。" #: ../../source/kv_cache_optimizations/blending.rst:12 msgid "Below are some blending-related configurations (and explanations):" -msgstr "" +msgstr "以下是一些与混合相关的配置（及其解释）：" #: ../../source/kv_cache_optimizations/blending.rst:37 msgid "" @@ -52,24 +52,24 @@ msgid "" "tokenizing each string individually. For example, assume we have a system" " prompt and three text chunks. We need to preprocess them into tokens " "before sending to the LLM:" -msgstr "" +msgstr "首先，我们将文本预处理为标记，因为将连接的字符串进行标记化可能会产生与分别对每个字符串进行标记化后再连接的结果不同的标记。例如，假设我们有一个系统提示和三个文本块。在将它们发送到 LLM 之前，我们需要将它们预处理为标记：" #: ../../source/kv_cache_optimizations/blending.rst:59 msgid "" "Then, we can send the tokenized prompt to vLLM. Meanwhile, LMCache will " "store the KV caches of different chunks according to the " "``BLEND_SPECIAL_STR``." -msgstr "" +msgstr "然后，我们可以将标记化的提示发送到 vLLM。同时，LMCache 将根据 ``BLEND_SPECIAL_STR`` 存储不同块的 KV 缓存。" #: ../../source/kv_cache_optimizations/blending.rst:65 msgid "" "Similarly, we build another prompt using the same chunks but with " "different orders." -msgstr "" +msgstr "同样，我们使用相同的块但以不同的顺序构建另一个提示。" #: ../../source/kv_cache_optimizations/blending.rst:82 msgid "" "Even though the second prompt has a different order of chunks, LMCache " "can still reuse the KV caches of chunk1, chunk2, and chunk3." -msgstr "" +msgstr "尽管第二个提示的块顺序不同，LMCache 仍然可以重用 chunk1、chunk2 和 chunk3 的 KV 缓存。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/cachegen.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/cachegen.po index 104f559e68a..f3689fc2405 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/cachegen.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/cachegen.po @@ -21,33 +21,33 @@ msgstr "" #: ../../source/kv_cache_optimizations/compression/cachegen.rst:4 msgid "CacheGen" -msgstr "" +msgstr "CacheGen" #: ../../source/kv_cache_optimizations/compression/cachegen.rst:6 msgid "" "Cachegen leverages KV cache's distributional properties to encode a KV " "cache into more compact bitstream representations with negligible " "decoding overhead." -msgstr "" +msgstr "Cachegen 利用 KV Cache 的分布特性，将 KV Cache 编码为更紧凑的比特流表示，解码开销可以忽略不计。" #: ../../source/kv_cache_optimizations/compression/cachegen.rst:10 msgid "Configuring CacheGen in LMCache" -msgstr "" +msgstr "在 LMCache 中配置 CacheGen" #: ../../source/kv_cache_optimizations/compression/cachegen.rst:12 msgid "" "The settings should be very similar to :ref:`naive KV cache sharing " "`. Only minor configurations need to be done to enable " "CacheGen." -msgstr "" +msgstr "设置应该与 :ref:`naive KV cache sharing ` 非常相似。只需进行少量配置即可启用 CacheGen。" #: ../../source/kv_cache_optimizations/compression/cachegen.rst:15 msgid "To enable CacheGen in offline inference, we need to set:" -msgstr "" +msgstr "要在离线推理中启用 CacheGen，我们需要设置：" #: ../../source/kv_cache_optimizations/compression/cachegen.rst:22 msgid "" "To enable CacheGen in online inference, we need to set the " "``remote_serde`` in the configuration yaml:" -msgstr "" +msgstr "要在在线推理中启用 CacheGen，我们需要在配置 yaml 中设置 ``remote_serde``：" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/index.po index e4d65cccc18..66ebd05d1b6 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/index.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/index.po @@ -21,19 +21,19 @@ msgstr "" #: ../../source/kv_cache_optimizations/compression/index.rst:2 msgid "Compression" -msgstr "" +msgstr "压缩" #: ../../source/kv_cache_optimizations/compression/index.rst:4 msgid "" "KV cache compression can greatly reduces the size of the cache, which can" " be beneficial for both storage/memory usage and loading speed. " "Currently, we support the following compression algorithms:" -msgstr "" +msgstr "KV Cache 压缩可以大大减少缓存的大小，这对存储/内存使用和加载速度都有好处。目前，我们支持以下压缩算法：" #: ../../source/kv_cache_optimizations/compression/index.rst:7 msgid "" ":ref:`CacheGen `: `CacheGen: KV Cache Compression and Streaming" " for Fast Large Language Model Serving " "`_" -msgstr "" +msgstr ":ref:`CacheGen `: `CacheGen: KV Cache 压缩与流式传输以快速服务大型语言模型 `_" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/layerwise.po b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/layerwise.po index f8a96a41898..6d17dc821ad 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/layerwise.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/layerwise.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/kv_cache_optimizations/layerwise.rst:2 msgid "Layerwise KV Transfer" -msgstr "" +msgstr "逐层 KV 传输" #: ../../source/kv_cache_optimizations/layerwise.rst:4 msgid "" @@ -29,271 +29,271 @@ msgid "" "optimization that allows for forward pass to \"stagger\" through its " "computation as each layer's KV Cache is received instead of only waiting " "to begin after the entire loading" -msgstr "" +msgstr "在逐层粒度上存储和加载 KV Cache 是一个关键优化，它允许前向传播在计算过程中“错开”，因为每一层的 KV Cache 在接收后就可以开始，而不必等到整个加载完成后才开始。" #: ../../source/kv_cache_optimizations/layerwise.rst:6 msgid "" "CacheBlend is implemented on top of the layerwise codepath in order to " "pipeline recompute and loading to mask the latency of loading KV Cache." -msgstr "" +msgstr "CacheBlend 是在逐层代码路径之上实现的，目的是将重计算和加载进行流水线处理，以掩盖加载 KV Cache 的延迟。" #: ../../source/kv_cache_optimizations/layerwise.rst:8 msgid "Basic Codepath" -msgstr "" +msgstr "基本代码路径" #: ../../source/kv_cache_optimizations/layerwise.rst:25 msgid "Architecture Overview" -msgstr "" +msgstr "架构概述" #: ../../source/kv_cache_optimizations/layerwise.rst:27 msgid "**CacheEngine**" -msgstr "" +msgstr "**CacheEngine**" #: ../../source/kv_cache_optimizations/layerwise.rst:28 msgid "The main orchestrator containing two primary generators:" -msgstr "" +msgstr "主要协调器，包含两个主要生成器：" #: ../../source/kv_cache_optimizations/layerwise.rst:30 msgid "" "**Retrieval Generator** (N + 2 yields): Handles layer-by-layer KV cache " "loading with on-demand memory allocation" -msgstr "" +msgstr "**检索生成器** (N + 2 生成): 处理逐层 KV Cache 加载，按需分配内存" #: ../../source/kv_cache_optimizations/layerwise.rst:31 msgid "" "**Storage Generator** (N + 1 yields): Manages layer-by-layer KV cache " "saving with upfront CPU memory allocation" -msgstr "" +msgstr "**存储生成器** (N + 1 yields): 管理逐层 KV Cache 保存，并提前分配 CPU 内存" #: ../../source/kv_cache_optimizations/layerwise.rst:33 msgid "**LayerwiseGPUConnector**" -msgstr "" +msgstr "**LayerwiseGPUConnector**" #: ../../source/kv_cache_optimizations/layerwise.rst:34 msgid "Manages GPU-CPU memory transfers with dedicated CUDA streams:" -msgstr "" +msgstr "管理使用专用 CUDA 流的 GPU-CPU 内存传输：" #: ../../source/kv_cache_optimizations/layerwise.rst:36 msgid "" "**Load GPU Buffer**: Temporary GPU memory for CPU→GPU transfers " "(``use_gpu: true``)" -msgstr "" +msgstr "**加载 GPU 缓冲区**：用于 CPU→GPU 传输的临时显存（``use_gpu: true``）" #: ../../source/kv_cache_optimizations/layerwise.rst:37 msgid "" "**Store GPU Buffer**: Temporary GPU memory for GPU→CPU transfers " "(``use_gpu: true``)" -msgstr "" +msgstr "**存储 GPU 缓冲区**：用于 GPU→CPU 传输的临时显存（``use_gpu: true``）" #: ../../source/kv_cache_optimizations/layerwise.rst:38 msgid "" "**Nested Generators**: ``batched_to_gpu()`` and ``batched_from_gpu()`` " "handle actual memory operations" -msgstr "" +msgstr "**嵌套生成器**: ``batched_to_gpu()`` 和 ``batched_from_gpu()`` 处理实际的内存操作" #: ../../source/kv_cache_optimizations/layerwise.rst:40 msgid "**StorageManager**" -msgstr "" +msgstr "**存储管理器**" #: ../../source/kv_cache_optimizations/layerwise.rst:41 msgid "Handles persistent storage operations:" -msgstr "" +msgstr "处理持久存储操作：" #: ../../source/kv_cache_optimizations/layerwise.rst:43 msgid "" "``layerwise_batched_get()``: Asynchronous retrieval with ``.result()`` " "for request-level concurrency" -msgstr "" +msgstr "``layerwise_batched_get()``: 使用 ``.result()`` 进行请求级并发的异步检索" #: ../../source/kv_cache_optimizations/layerwise.rst:44 msgid "``batched_put()``: Stores memory objects to persistent backends" -msgstr "" +msgstr "``batched_put()``: 将内存对象存储到持久后端" #: ../../source/kv_cache_optimizations/layerwise.rst:47 msgid "Execution Flow" -msgstr "" +msgstr "执行流程" #: ../../source/kv_cache_optimizations/layerwise.rst:49 msgid "The layerwise pipeline follows a numbered execution sequence:" -msgstr "" +msgstr "逐层管道遵循编号的执行顺序：" #: ../../source/kv_cache_optimizations/layerwise.rst:51 msgid "**1. start_load_kv()**" -msgstr "" +msgstr "**1. start_load_kv()**" #: ../../source/kv_cache_optimizations/layerwise.rst:52 msgid "Initializes Retrieval Generator via ``lmcache_engine.retrieve_layer()``" -msgstr "" +msgstr "通过 ``lmcache_engine.retrieve_layer()`` 初始化检索生成器" #: ../../source/kv_cache_optimizations/layerwise.rst:53 msgid "Performs setup (1st ``next()``) and loads layer 0 (2nd ``next()``)" -msgstr "" +msgstr "执行设置（第一次 ``next()``）并加载第 0 层（第二次 ``next()``）" #: ../../source/kv_cache_optimizations/layerwise.rst:54 msgid "Creates ``layerwise_retrievers`` list for ongoing layer processing" -msgstr "" +msgstr "创建 ``layerwise_retrievers`` 列表以进行持续的层处理" #: ../../source/kv_cache_optimizations/layerwise.rst:56 msgid "**2. wait_for_layer_load()** (repeated for each layer)" -msgstr "" +msgstr "**2. wait_for_layer_load()** (对每一层重复执行)" #: ../../source/kv_cache_optimizations/layerwise.rst:57 msgid "Advances Retrieval Generator via ``next()`` to process layer i" -msgstr "" +msgstr "通过 ``next()`` 使检索生成器向前推进以处理第 i 层" #: ../../source/kv_cache_optimizations/layerwise.rst:58 msgid "" "Triggers ``StorageManager.layerwise_batched_get()`` for async cache " "retrieval" -msgstr "" +msgstr "触发 ``StorageManager.layerwise_batched_get()`` 进行异步缓存查找" #: ../../source/kv_cache_optimizations/layerwise.rst:59 msgid "" "Calls GPU Load Generator's ``batched_to_gpu()`` to transfer memory " "objects to GPU" -msgstr "" +msgstr "调用 GPU Load Generator 的 ``batched_to_gpu()`` 将内存对象传输到 GPU" #: ../../source/kv_cache_optimizations/layerwise.rst:60 msgid "" "**Last request in batch**: Synchronizes " "``current_stream.wait_stream(load_stream)``" -msgstr "" +msgstr "**批处理中的最后请求**：同步 ``current_stream.wait_stream(load_stream)``" #: ../../source/kv_cache_optimizations/layerwise.rst:62 msgid "**3. save_kv_layer()** (repeated for each layer)" -msgstr "" +msgstr "**3. save_kv_layer()**（对每一层重复执行）" #: ../../source/kv_cache_optimizations/layerwise.rst:63 msgid "" "**First call only**: Creates Storage Generator with upfront CPU memory " "allocation" -msgstr "" +msgstr "**首次调用仅**：创建具有预先分配 CPU 内存的存储生成器" #: ../../source/kv_cache_optimizations/layerwise.rst:64 msgid "Advances Storage Generator via ``next()`` to process layer i" -msgstr "" +msgstr "通过 ``next()`` 进展存储生成器以处理第 i 层" #: ../../source/kv_cache_optimizations/layerwise.rst:65 msgid "" "Calls GPU Store Generator's ``batched_from_gpu()`` to transfer GPU data " "to CPU" -msgstr "" +msgstr "调用 GPU 存储生成器的 ``batched_from_gpu()`` 将 GPU 数据传输到 CPU" #: ../../source/kv_cache_optimizations/layerwise.rst:66 msgid "" "**First request in batch**: Synchronizes " "``store_stream.wait_stream(current_stream)``" -msgstr "" +msgstr "**批处理中的第一个请求**：同步 ``store_stream.wait_stream(current_stream)``" #: ../../source/kv_cache_optimizations/layerwise.rst:68 msgid "**4. wait_for_save()**" -msgstr "" +msgstr "**4. wait_for_save()**" #: ../../source/kv_cache_optimizations/layerwise.rst:69 msgid "Finalizes Storage Generator with last ``next()`` call" -msgstr "" +msgstr "通过最后一次 ``next()`` 调用来完成存储生成器" #: ../../source/kv_cache_optimizations/layerwise.rst:70 msgid "Completes all ``StorageManager.batched_put()`` operations" -msgstr "" +msgstr "完成所有 ``StorageManager.batched_put()`` 操作" #: ../../source/kv_cache_optimizations/layerwise.rst:71 msgid "Performs GPU Store Generator cleanup" -msgstr "" +msgstr "执行 GPU 存储生成器清理" #: ../../source/kv_cache_optimizations/layerwise.rst:74 msgid "Key Optimizations" -msgstr "" +msgstr "关键优化" #: ../../source/kv_cache_optimizations/layerwise.rst:76 msgid "**Pipelined Memory Operations**" -msgstr "" +msgstr "**流水线内存操作**" #: ../../source/kv_cache_optimizations/layerwise.rst:77 msgid "The system overlaps layer N+1 computation with layer N storage." -msgstr "" +msgstr "系统将第 N+1 层的计算与第 N 层的存储重叠。" #: ../../source/kv_cache_optimizations/layerwise.rst:79 msgid "**Stream Synchronization**" -msgstr "" +msgstr "**流同步**" #: ../../source/kv_cache_optimizations/layerwise.rst:80 msgid "Three CUDA streams coordinate operations:" -msgstr "" +msgstr "三个 CUDA 流协调操作：" #: ../../source/kv_cache_optimizations/layerwise.rst:82 msgid "``current_stream``: vLLM's forward pass computation" -msgstr "" +msgstr "``current_stream``: vLLM 的前向传播计算" #: ../../source/kv_cache_optimizations/layerwise.rst:83 msgid "``load_stream``: KV cache loading operations" -msgstr "" +msgstr "``load_stream``: KV Cache 加载操作" #: ../../source/kv_cache_optimizations/layerwise.rst:84 msgid "``store_stream``: KV cache storing operations" -msgstr "" +msgstr "``store_stream``: KV Cache 存储操作" #: ../../source/kv_cache_optimizations/layerwise.rst:86 msgid "**Batch-Level Coordination**" -msgstr "" +msgstr "**批量级协调**" #: ../../source/kv_cache_optimizations/layerwise.rst:87 msgid "Multiple requests are processed together with specialized synchronization:" -msgstr "" +msgstr "多个请求一起处理，并使用专门的同步机制：" #: ../../source/kv_cache_optimizations/layerwise.rst:89 msgid "" "**First request**: Provides store stream synchronization to prevent GPU " "buffer corruption" -msgstr "" +msgstr "**第一次请求**：提供存储流同步以防止 GPU 缓冲区损坏" #: ../../source/kv_cache_optimizations/layerwise.rst:90 msgid "" "**Last request**: Provides load stream synchronization to ensure KV cache" " availability" -msgstr "" +msgstr "**最后请求**：提供加载流同步以确保 KV Cache 可用性" #: ../../source/kv_cache_optimizations/layerwise.rst:92 msgid "**Memory Allocation Strategies**" -msgstr "" +msgstr "**内存分配策略**" #: ../../source/kv_cache_optimizations/layerwise.rst:93 msgid "**Retrieval**: Layer-by-layer allocation" -msgstr "" +msgstr "**检索**：逐层分配" #: ../../source/kv_cache_optimizations/layerwise.rst:94 msgid "**Storage**: Upfront allocation for all layers" -msgstr "" +msgstr "**存储**：为所有层的预先分配" #: ../../source/kv_cache_optimizations/layerwise.rst:96 msgid "**Cache Key Management**" -msgstr "" +msgstr "**缓存键管理**" #: ../../source/kv_cache_optimizations/layerwise.rst:97 msgid "" "Multi-layer cache engine keys use ``split_layers(N)`` to create per-layer" " kubernetes_deployment" -msgstr "" +msgstr "多层缓存引擎密钥使用 ``split_layers(N)`` 来创建每层的 kubernetes_deployment" #: ../../source/kv_cache_optimizations/layerwise.rst:100 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/kv_cache_optimizations/layerwise.rst:102 msgid "Enable layerwise caching by setting:" -msgstr "" +msgstr "通过设置启用逐层缓存：" #: ../../source/kv_cache_optimizations/layerwise.rst:108 msgid "" "The system automatically selects appropriate layerwise GPU connectors " "based on configuration:" -msgstr "" +msgstr "系统会根据配置自动选择适当的逐层显卡连接器：" #: ../../source/kv_cache_optimizations/layerwise.rst:110 msgid "``VLLMPagedMemLayerwiseGPUConnector``: For standard layerwise operations" -msgstr "" +msgstr "``VLLMPagedMemLayerwiseGPUConnector``: 用于标准逐层操作" #: ../../source/kv_cache_optimizations/layerwise.rst:111 msgid "``VLLMBufferLayerwiseGPUConnector``: When blending is enabled" -msgstr "" +msgstr "``VLLMBufferLayerwiseGPUConnector``: 当启用混合时" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/architecture.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/architecture.po index 41f0e04fb4f..d345fdb0645 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/architecture.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/architecture.po @@ -21,35 +21,35 @@ msgstr "" #: ../../source/mp/architecture.rst:2 msgid "Architecture & Developer Guide" -msgstr "" +msgstr "架构与开发者指南" #: ../../source/mp/architecture.rst:4 msgid "" "This page describes the internal architecture of LMCache multiprocess " "mode. It is aimed at developers who want to understand, debug, or extend " "the system." -msgstr "" +msgstr "本页描述了 LMCache 多进程模式的内部架构。旨在帮助希望理解、调试或扩展系统的开发人员。" #: ../../source/mp/architecture.rst:12 msgid "High-Level Architecture" -msgstr "" +msgstr "高层架构" #: ../../source/mp/architecture.rst:43 msgid "Server Variants" -msgstr "" +msgstr "服务器变体" #: ../../source/mp/architecture.rst:45 msgid "" "All three server entry points share the same ``MPCacheEngine`` and " "``StorageManager`` core." -msgstr "" +msgstr "所有三个服务器入口点共享相同的 ``MPCacheEngine`` 和 ``StorageManager`` 核心。" #: ../../source/mp/architecture.rst:48 msgid "" "**``server.py``** -- The default ZMQ-only server. Creates an " "``MPCacheEngine`` and a ``MessageQueueServer``, registers handlers for " "all core ``RequestType`` values, and blocks in a keep-alive loop." -msgstr "" +msgstr "**``server.py``** -- 默认的仅 ZMQ 服务器。创建一个 ``MPCacheEngine`` 和一个 ``MessageQueueServer``，为所有核心 ``RequestType`` 值注册处理程序，并在保持活动循环中阻塞。" #: ../../source/mp/architecture.rst:52 msgid "" @@ -59,7 +59,7 @@ msgid "" "``CB_STORE_PRE_COMPUTED``, ``CB_RETRIEVE_PRE_COMPUTED``, " "``CB_STORE_FINAL``). Enables non-prefix KV cache reuse across document " "paragraphs." -msgstr "" +msgstr "**``blend_server_v2.py``** -- 扩展了 ``MPCacheEngine``，增加了 ``BlendEngineV2``，该引擎添加了 CacheBlend 操作 (``CB_REGISTER_KV_CACHE``, ``CB_LOOKUP_PRE_COMPUTED``, ``CB_STORE_PRE_COMPUTED``, ``CB_RETRIEVE_PRE_COMPUTED``, ``CB_STORE_FINAL``)。 使得在文档段落之间能够重用非前缀 KV Cache。" #: ../../source/mp/architecture.rst:58 msgid "" @@ -72,57 +72,57 @@ msgid "" "state. The ZMQ server runs as part of the same process, and any " "configured runtime plugins are spawned by ``MPRuntimePluginLauncher`` " "during FastAPI startup." -msgstr "" +msgstr "**``http_server.py``** -- 在 FastAPI 应用程序中封装 ``run_cache_server()`` (来自 ``server.py``)。 端点由 ``http_apis/`` 下的模块贡献，并通过 ``HTTPAPIRegistry`` 自动注册：``GET /`` (基本存活检查)、``GET /healthcheck`` 用于 Kubernetes 探针、``POST /clear-cache`` 用于清除 L1 (CPU) 内存中的所有 KV 缓存数据，以及 ``GET /status`` 用于检查详细的内部状态。 ZMQ 服务器作为同一进程的一部分运行，任何配置的运行时插件在 FastAPI 启动期间由 ``MPRuntimePluginLauncher`` 生成。" #: ../../source/mp/architecture.rst:68 msgid "ZMQ Protocol" -msgstr "" +msgstr "ZMQ 协议" #: ../../source/mp/architecture.rst:70 msgid "Communication between vLLM and LMCache uses ZMQ (DEALER/ROUTER pattern)." -msgstr "" +msgstr "vLLM 和 LMCache 之间的通信使用 ZMQ（DEALER/ROUTER 模式）。" #: ../../source/mp/architecture.rst:72 msgid "**RequestType enum** (defined in ``protocols/base.py``):" -msgstr "" +msgstr "**RequestType 枚举**（定义在 ``protocols/base.py``）：" #: ../../source/mp/architecture.rst:78 msgid "Request Type" -msgstr "" +msgstr "请求类型" #: ../../source/mp/architecture.rst:79 msgid "Handler Type" -msgstr "" +msgstr "处理程序类型" #: ../../source/mp/architecture.rst:80 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/mp/architecture.rst:81 msgid "``REGISTER_KV_CACHE``" -msgstr "" +msgstr "``REGISTER_KV_CACHE``" #: ../../source/mp/architecture.rst:82 ../../source/mp/architecture.rst:85 #: ../../source/mp/architecture.rst:117 ../../source/mp/architecture.rst:127 #: ../../source/mp/architecture.rst:130 ../../source/mp/architecture.rst:133 msgid "SYNC" -msgstr "" +msgstr "同步" #: ../../source/mp/architecture.rst:83 msgid "Register GPU KV cache tensors for a vLLM instance." -msgstr "" +msgstr "为 vLLM 实例注册 GPU KV Cache 张量。" #: ../../source/mp/architecture.rst:84 msgid "``UNREGISTER_KV_CACHE``" -msgstr "" +msgstr "``UNREGISTER_KV_CACHE``" #: ../../source/mp/architecture.rst:86 msgid "Unregister KV cache tensors." -msgstr "" +msgstr "注销 KV Cache 张量。" #: ../../source/mp/architecture.rst:87 msgid "``STORE``" -msgstr "" +msgstr "``STORE``" #: ../../source/mp/architecture.rst:88 ../../source/mp/architecture.rst:91 #: ../../source/mp/architecture.rst:94 ../../source/mp/architecture.rst:98 @@ -133,308 +133,308 @@ msgstr "" #: ../../source/mp/architecture.rst:142 ../../source/mp/architecture.rst:145 #: ../../source/mp/architecture.rst:148 ../../source/mp/architecture.rst:153 msgid "BLOCKING" -msgstr "" +msgstr "阻塞" #: ../../source/mp/architecture.rst:89 msgid "Store KV cache chunks from GPU to L1 (CPU)." -msgstr "" +msgstr "将 KV Cache 块从 GPU 存储到 L1 (CPU)。" #: ../../source/mp/architecture.rst:90 msgid "``RETRIEVE``" -msgstr "" +msgstr "``RETRIEVE``" #: ../../source/mp/architecture.rst:92 msgid "Copy KV cache chunks from L1 (CPU) back to GPU." -msgstr "" +msgstr "将 KV Cache 块从 L1 (CPU) 复制回 GPU。" #: ../../source/mp/architecture.rst:93 msgid "``LOOKUP``" -msgstr "" +msgstr "``LOOKUP``" #: ../../source/mp/architecture.rst:95 msgid "" "Submit a prefix lookup; the prefetch job is tracked server-side by " "request_id." -msgstr "" +msgstr "提交前缀查找；预取作业由 request_id 在服务器端进行跟踪。" #: ../../source/mp/architecture.rst:97 msgid "``QUERY_PREFETCH_STATUS``" -msgstr "" +msgstr "``QUERY_PREFETCH_STATUS``" #: ../../source/mp/architecture.rst:99 msgid "" "Poll a prefetch job by request_id. Returns the loaded chunk count when " "done, or ``None`` while the prefetch is still in progress." -msgstr "" +msgstr "通过 request_id 轮询预取作业。完成时返回加载的块数，预取仍在进行时返回 ``None``。" #: ../../source/mp/architecture.rst:101 msgid "``QUERY_PREFETCH_LOOKUP_HITS``" -msgstr "" +msgstr "``QUERY_PREFETCH_LOOKUP_HITS``" #: ../../source/mp/architecture.rst:103 msgid "" "Query the lookup-phase hit chunk count by request_id, before the prefetch" " finishes. Returns ``None`` while the lookup is still running." -msgstr "" +msgstr "在预取完成之前，通过 request_id 查询查找阶段的命中块计数。当查找仍在运行时返回 ``None``。" #: ../../source/mp/architecture.rst:106 msgid "``FREE_LOOKUP_LOCKS``" -msgstr "" +msgstr "``FREE_LOOKUP_LOCKS``" #: ../../source/mp/architecture.rst:108 msgid "Release read locks from a cancelled lookup without doing a full RETRIEVE." -msgstr "" +msgstr "从取消的查找中释放读取锁，而无需执行完整的 RETRIEVE。" #: ../../source/mp/architecture.rst:110 msgid "``END_SESSION``" -msgstr "" +msgstr "``END_SESSION``" #: ../../source/mp/architecture.rst:112 msgid "Remove session state for a finished request." -msgstr "" +msgstr "移除已完成请求的会话状态。" #: ../../source/mp/architecture.rst:113 msgid "``CLEAR``" -msgstr "" +msgstr "``CLEAR``" #: ../../source/mp/architecture.rst:115 msgid "Clear all cached data." -msgstr "" +msgstr "清除所有缓存数据。" #: ../../source/mp/architecture.rst:116 msgid "``GET_CHUNK_SIZE``" -msgstr "" +msgstr "``GET_CHUNK_SIZE``" #: ../../source/mp/architecture.rst:118 msgid "Return the server's chunk size." -msgstr "" +msgstr "返回服务器的块大小。" #: ../../source/mp/architecture.rst:119 msgid "``PING``" -msgstr "" +msgstr "``PING``" #: ../../source/mp/architecture.rst:121 msgid "Liveness ping; the handler always returns ``True``." -msgstr "" +msgstr "存活探测；处理程序始终返回 ``True``。" #: ../../source/mp/architecture.rst:122 msgid "``REPORT_BLOCK_ALLOCATION``" -msgstr "" +msgstr "``REPORT_BLOCK_ALLOCATION``" #: ../../source/mp/architecture.rst:124 msgid "" "Fire-and-forget channel for the vLLM scheduler to report GPU block " "allocation events to the observability subsystem." -msgstr "" +msgstr "vLLM 调度器的火忘通道，用于向可观察性子系统报告 GPU 块分配事件。" #: ../../source/mp/architecture.rst:126 msgid "``NOOP``" -msgstr "" +msgstr "``NOOP``" #: ../../source/mp/architecture.rst:128 msgid "Debug heartbeat -- returns a confirmation string." -msgstr "" +msgstr "调试心跳 -- 返回确认字符串。" #: ../../source/mp/architecture.rst:129 msgid "``CB_REGISTER_KV_CACHE``" -msgstr "" +msgstr "``CB_REGISTER_KV_CACHE``" #: ../../source/mp/architecture.rst:131 msgid "(Blend) Register CacheBlend KV buffer." -msgstr "" +msgstr "(Blend) 注册 CacheBlend KV 缓冲区。" #: ../../source/mp/architecture.rst:132 msgid "``CB_UNREGISTER_KV_CACHE``" -msgstr "" +msgstr "``CB_UNREGISTER_KV_CACHE``" #: ../../source/mp/architecture.rst:134 msgid "(Blend) Unregister CacheBlend KV buffer." -msgstr "" +msgstr "(Blend) 取消注册 CacheBlend KV 缓冲区。" #: ../../source/mp/architecture.rst:135 msgid "``CB_STORE_PRE_COMPUTED``" -msgstr "" +msgstr "``CB_STORE_PRE_COMPUTED``" #: ../../source/mp/architecture.rst:137 msgid "(Blend) Store pre-computed paragraph chunks." -msgstr "" +msgstr "(Blend) 存储预计算的段落块。" #: ../../source/mp/architecture.rst:138 msgid "``CB_LOOKUP_PRE_COMPUTED``" -msgstr "" +msgstr "``CB_LOOKUP_PRE_COMPUTED``" #: ../../source/mp/architecture.rst:140 msgid "(Blend) Lookup pre-computed paragraph chunks." -msgstr "" +msgstr "(Blend) 查找预计算的段落块。" #: ../../source/mp/architecture.rst:141 msgid "``CB_RETRIEVE_PRE_COMPUTED``" -msgstr "" +msgstr "``CB_RETRIEVE_PRE_COMPUTED``" #: ../../source/mp/architecture.rst:143 msgid "(Blend) Retrieve pre-computed paragraph chunks to GPU." -msgstr "" +msgstr "(Blend) 将预计算的段落块检索到 GPU。" #: ../../source/mp/architecture.rst:144 msgid "``CB_STORE_FINAL``" -msgstr "" +msgstr "``CB_STORE_FINAL``" #: ../../source/mp/architecture.rst:146 msgid "(Blend) Store final blended chunks." -msgstr "" +msgstr "(Blend) 存储最终混合块。" #: ../../source/mp/architecture.rst:147 msgid "``CB_LOOKUP_PRE_COMPUTED_V2``" -msgstr "" +msgstr "``CB_LOOKUP_PRE_COMPUTED_V2``" #: ../../source/mp/architecture.rst:149 msgid "" "(Blend V2) Lookup pre-computed chunks; returns ``CBMatchResult`` entries " "(with old/cur ranges and per-chunk hashes) so the retrieve step can skip " "re-hashing." -msgstr "" +msgstr "（Blend V2）查找预计算的块；返回 ``CBMatchResult`` 条目（包含旧范围/当前范围和每块哈希），以便检索步骤可以跳过重新哈希。" #: ../../source/mp/architecture.rst:152 msgid "``CB_RETRIEVE_PRE_COMPUTED_V2``" -msgstr "" +msgstr "``CB_RETRIEVE_PRE_COMPUTED_V2``" #: ../../source/mp/architecture.rst:154 msgid "" "(Blend V2) Retrieve pre-computed chunks using the ``CBMatchResult`` list " "returned by ``CB_LOOKUP_PRE_COMPUTED_V2``." -msgstr "" +msgstr "（Blend V2）使用 ``CB_LOOKUP_PRE_COMPUTED_V2`` 返回的 ``CBMatchResult`` 列表检索预计算块。" #: ../../source/mp/architecture.rst:157 msgid "**Handler types:**" -msgstr "" +msgstr "**处理程序类型:**" #: ../../source/mp/architecture.rst:159 msgid "**SYNC** -- Runs directly in the ZMQ main loop (fast, non-blocking)." -msgstr "" +msgstr "**同步** -- 直接在 ZMQ 主循环中运行（快速，非阻塞）。" #: ../../source/mp/architecture.rst:160 msgid "" "**BLOCKING** -- Dispatched to a thread pool (may involve GPU copies or " "I/O)." -msgstr "" +msgstr "**阻塞** -- 分配到线程池（可能涉及 GPU 复制或 I/O）。" #: ../../source/mp/architecture.rst:163 msgid "Config System" -msgstr "" +msgstr "配置系统" #: ../../source/mp/architecture.rst:165 msgid "Each config module exposes a composable triple:" -msgstr "" +msgstr "每个配置模块都暴露一个可组合的三元组：" #: ../../source/mp/architecture.rst:171 msgid "``server.py:parse_args()`` composes them:" -msgstr "" +msgstr "``server.py:parse_args()`` 组合它们：" #: ../../source/mp/architecture.rst:184 msgid "" "Both ``blend_server_v2.py`` and ``http_server.py`` reuse this pattern, " "adding ``add_http_frontend_args()`` for the HTTP variant." -msgstr "" +msgstr "``blend_server_v2.py`` 和 ``http_server.py`` 都重用了这个模式，为 HTTP 变体添加了 ``add_http_frontend_args()``。" #: ../../source/mp/architecture.rst:188 msgid "Distributed Storage" -msgstr "" +msgstr "分布式存储" #: ../../source/mp/architecture.rst:191 msgid "StorageManager" -msgstr "" +msgstr "StorageManager" #: ../../source/mp/architecture.rst:193 ../../source/mp/architecture.rst:423 msgid "``lmcache/v1/distributed/storage_manager.py``" -msgstr "" +msgstr "``lmcache/v1/distributed/storage_manager.py``" #: ../../source/mp/architecture.rst:195 msgid "" "The top-level manager that wires together L1, L2, and all controllers. " "Key methods:" -msgstr "" +msgstr "将 L1、L2 和所有控制器连接在一起的顶级管理器。关键方法：" #: ../../source/mp/architecture.rst:198 msgid "``reserve_write()`` / ``finish_write()`` -- Two-phase write into L1." -msgstr "" +msgstr "``reserve_write()`` / ``finish_write()`` -- L1 的两阶段写入。" #: ../../source/mp/architecture.rst:199 msgid "" "``submit_prefetch_task()`` / ``query_prefetch_status()`` -- Async lookup " "+ L2 prefetch." -msgstr "" +msgstr "``submit_prefetch_task()`` / ``query_prefetch_status()`` -- 异步查找 + L2 预取。" #: ../../source/mp/architecture.rst:201 msgid "" "``read_prefetched_results()`` / ``finish_read_prefetched()`` -- Read " "prefetched data from L1 with automatic lock management." -msgstr "" +msgstr "``read_prefetched_results()`` / ``finish_read_prefetched()`` -- 从 L1 读取预取的数据，并自动管理锁。" #: ../../source/mp/architecture.rst:205 msgid "L1Manager" -msgstr "" +msgstr "L1Manager" #: ../../source/mp/architecture.rst:207 ../../source/mp/architecture.rst:427 msgid "``lmcache/v1/distributed/l1_manager.py``" -msgstr "" +msgstr "``lmcache/v1/distributed/l1_manager.py``" #: ../../source/mp/architecture.rst:209 msgid "Manages objects in CPU memory with a state machine:" -msgstr "" +msgstr "在 CPU 内存中使用状态机管理对象：" #: ../../source/mp/architecture.rst:219 msgid "" "Each object has two ``TTLLock`` instances (read and write) with " "configurable timeouts to prevent deadlocks from crashed clients." -msgstr "" +msgstr "每个对象都有两个 ``TTLLock`` 实例（读和写），并具有可配置的超时，以防止因客户端崩溃而导致的死锁。" #: ../../source/mp/architecture.rst:222 msgid "" "The ``L1MemoryManager`` handles the underlying memory allocation (lazy " "growth up to ``--l1-size-gb``)." -msgstr "" +msgstr "``L1MemoryManager`` 处理底层内存分配（懒惰增长至 ``--l1-size-gb``）。" #: ../../source/mp/architecture.rst:226 msgid "L2 Adapters" -msgstr "" +msgstr "L2 适配器" #: ../../source/mp/architecture.rst:228 msgid "``lmcache/v1/distributed/l2_adapters/``" -msgstr "" +msgstr "``lmcache/v1/distributed/l2_adapters/``" #: ../../source/mp/architecture.rst:230 msgid "" "The ``L2AdapterInterface`` (in ``base.py``) defines three async task " "methods:" -msgstr "" +msgstr "``L2AdapterInterface``（在 ``base.py`` 中）定义了三个异步任务方法：" #: ../../source/mp/architecture.rst:232 msgid "``submit_store_task(key, data)`` -- Push data to L2." -msgstr "" +msgstr "``submit_store_task(key, data)`` -- 将数据推送到 L2." #: ../../source/mp/architecture.rst:233 msgid "``submit_lookup_and_lock_task(keys)`` -- Check if keys exist in L2." -msgstr "" +msgstr "``submit_lookup_and_lock_task(keys)`` -- 检查 keys 是否存在于 L2 中。" #: ../../source/mp/architecture.rst:234 msgid "``submit_load_task(keys, layout_desc)`` -- Load data from L2 into L1." -msgstr "" +msgstr "``submit_load_task(keys, layout_desc)`` -- 从 L2 加载数据到 L1。" #: ../../source/mp/architecture.rst:236 msgid "" "The factory function ``create_l2_adapter()`` (in ``__init__.py``) uses " "``isinstance()`` on the config type to instantiate the correct adapter." -msgstr "" +msgstr "工厂函数 ``create_l2_adapter()`` （在 ``__init__.py`` 中）使用 ``isinstance()`` 对配置类型进行检查，以实例化正确的适配器。" #: ../../source/mp/architecture.rst:239 msgid "" "New adapter types are registered via ``register_l2_adapter_type()`` in " "``config.py``." -msgstr "" +msgstr "新的适配器类型通过 ``register_l2_adapter_type()`` 在 ``config.py`` 中注册。" #: ../../source/mp/architecture.rst:243 msgid "Controllers" -msgstr "" +msgstr "控制器" #: ../../source/mp/architecture.rst:245 msgid "" @@ -443,7 +443,7 @@ msgid "" "and adapter store eventfds. When new objects appear in L1 (signaled via " "``StoreListener``), it submits async store tasks to each L2 adapter based" " on the ``StorePolicy``." -msgstr "" +msgstr "**StoreController** (``storage_controllers/store_controller.py``)：事件驱动的后台线程，使用 ``select.poll()`` 监听事件文件描述符和适配器存储事件文件描述符。当 L1 中出现新对象时（通过 ``StoreListener`` 发出信号），它根据 ``StorePolicy`` 向每个 L2 适配器提交异步存储任务。" #: ../../source/mp/architecture.rst:251 msgid "" @@ -453,7 +453,7 @@ msgid "" "``IsolatedLRU``, or ``noop``) until usage drops below the target. " "``IsolatedLRU`` evicts per ``cache_salt`` against limits registered " "through the ``/quota`` HTTP endpoints; see :ref:`mp-http-quota-api`." -msgstr "" +msgstr "**逐出控制器** (``storage_controllers/eviction_controller.py``)：定期检查 L1 内存使用情况与水位线阈值的关系。当触发时，使用配置的策略（``LRU``、``IsolatedLRU`` 或 ``noop``）逐出对象，直到使用量降到目标以下。``IsolatedLRU`` 根据通过 ``/quota`` HTTP 端点注册的限制，针对 ``cache_salt`` 进行逐出；请参见 :ref:`mp-http-quota-api`。" #: ../../source/mp/architecture.rst:258 msgid "" @@ -461,27 +461,27 @@ msgid "" "Handles L2 lookup and load requests submitted by ``StorageManager`` " "during ``LOOKUP`` RPCs. When keys are not in L1, it queries L2 adapters " "and loads found data back into L1." -msgstr "" +msgstr "**预取控制器** (``storage_controllers/prefetch_controller.py``): 处理 ``StorageManager`` 在 ``LOOKUP`` RPC 中提交的 L2 查找和加载请求。当键不在 L1 中时，它会查询 L2 适配器并将找到的数据加载回 L1。" #: ../../source/mp/architecture.rst:264 msgid "Request Flows" -msgstr "" +msgstr "请求流程" #: ../../source/mp/architecture.rst:267 msgid "LOOKUP Flow" -msgstr "" +msgstr "查找流程" #: ../../source/mp/architecture.rst:285 msgid "STORE Flow" -msgstr "" +msgstr "存储流程" #: ../../source/mp/architecture.rst:304 msgid "RETRIEVE Flow" -msgstr "" +msgstr "获取流程" #: ../../source/mp/architecture.rst:320 msgid "Observability Internals" -msgstr "" +msgstr "可观察性内部实现" #: ../../source/mp/architecture.rst:322 msgid "" @@ -491,7 +491,7 @@ msgid "" "objects to a bounded queue (``--event-bus-queue-size``, default 10000, " "tail-drop on overflow). A background drain thread dispatches each event " "to all registered subscribers." -msgstr "" +msgstr "**EventBus** (``lmcache/v1/mp_observability/event_bus.py``) 是一个在服务器启动时由 ``init_observability()`` 初始化的全局单例。生产者（L1Manager、StorageManager、MPCacheEngine）将 ``Event`` 对象发布到一个有界队列中 (``--event-bus-queue-size``, 默认 10000，溢出时尾部丢弃)。一个后台排空线程将每个事件分发给所有注册的订阅者。" #: ../../source/mp/architecture.rst:329 msgid "" @@ -501,7 +501,7 @@ msgid "" "and ``tracing/`` (OTel spans built from START/END event pairs). " "``init_observability()`` registers the set selected by CLI flags " "(``--disable-metrics``, ``--disable-logging``, ``--enable-tracing``)." -msgstr "" +msgstr "**订阅者** 位于 ``lmcache/v1/mp_observability/subscribers/`` 目录下，按关注点分组：``metrics/``（OTel 计数器和生命周期直方图）、``logging/``（Python 日志处理程序、查找哈希 JSONL）和 ``tracing/``（由 START/END 事件对构建的 OTel 跨度）。``init_observability()`` 根据 CLI 标志（``--disable-metrics``、``--disable-logging``、``--enable-tracing``）注册所选的集合。" #: ../../source/mp/architecture.rst:336 msgid "" @@ -510,15 +510,15 @@ msgid "" "bind to the real provider. Metrics are exported both to an in-process " "Prometheus ``/metrics`` endpoint (``--prometheus-port``, default 9090) " "and, when ``--otlp-endpoint`` is set, pushed to an OTel collector." -msgstr "" +msgstr "**OTel 提供者**在构造订阅者之前通过 ``otel_init.py`` 进行设置，因此模块级的 ``get_meter()`` / ``get_tracer()`` 调用绑定到真实的提供者。指标同时导出到进程内的 Prometheus ``/metrics`` 端点（``--prometheus-port``, 默认 9090），并且在设置了 ``--otlp-endpoint`` 时，推送到 OTel 收集器。" #: ../../source/mp/architecture.rst:344 msgid "How to Extend" -msgstr "" +msgstr "如何扩展" #: ../../source/mp/architecture.rst:347 msgid "Adding a new L2 adapter" -msgstr "" +msgstr "添加新的 L2 适配器" #: ../../source/mp/architecture.rst:349 msgid "" @@ -526,33 +526,33 @@ msgid "" "``lmcache/v1/distributed/l2_adapters/`` — ``__init__.py`` auto-discovers " "modules matching that suffix via ``pkgutil`` and imports them lazily on " "first use, so no other files need to be modified." -msgstr "" +msgstr "在 ``lmcache/v1/distributed/l2_adapters/`` 下创建一个新的 ``*_l2_adapter.py`` 模块 — ``__init__.py`` 通过 ``pkgutil`` 自动发现匹配该后缀的模块，并在首次使用时懒加载导入，因此无需修改其他文件。" #: ../../source/mp/architecture.rst:354 msgid "" "Create a config class subclassing ``L2AdapterConfigBase`` with " "``from_dict()`` and ``help()`` methods." -msgstr "" +msgstr "创建一个配置类，继承自 ``L2AdapterConfigBase``，并实现 ``from_dict()`` 和 ``help()`` 方法。" #: ../../source/mp/architecture.rst:356 msgid "" "Create an adapter class implementing ``L2AdapterInterface``, and a small " "factory function ``(config, l1_memory_desc) -> L2AdapterInterface``." -msgstr "" +msgstr "创建一个实现 ``L2AdapterInterface`` 的适配器类，以及一个小型工厂函数 ``(config, l1_memory_desc) -> L2AdapterInterface``。" #: ../../source/mp/architecture.rst:359 msgid "At module level, self-register both the config and the factory:" -msgstr "" +msgstr "在模块级别，自我注册配置和工厂：" #: ../../source/mp/architecture.rst:366 msgid "" "See ``mock_l2_adapter.py`` or ``s3_l2_adapter.py`` for reference " "implementations." -msgstr "" +msgstr "请参阅 ``mock_l2_adapter.py`` 或 ``s3_l2_adapter.py`` 以获取参考实现。" #: ../../source/mp/architecture.rst:370 msgid "Adding an observability subscriber" -msgstr "" +msgstr "添加可观察性订阅者" #: ../../source/mp/architecture.rst:372 #, python-brace-format @@ -561,7 +561,7 @@ msgid "" "``lmcache/v1/mp_observability/event_bus.py``): implement " "``get_subscriptions()`` to return an ``{EventType: callback}`` mapping; " "optionally override ``shutdown()`` for cleanup." -msgstr "" +msgstr "创建一个继承自 ``EventSubscriber`` 的订阅者类（定义在 ``lmcache/v1/mp_observability/event_bus.py`` 中）：实现 ``get_subscriptions()`` 返回一个 ``{EventType: callback}`` 映射；可选地重写 ``shutdown()`` 进行清理。" #: ../../source/mp/architecture.rst:376 msgid "" @@ -569,7 +569,7 @@ msgid "" "(``subscribers/metrics/``, ``subscribers/logging/``, or " "``subscribers/tracing/``) and export it from that package's " "``__init__.py``." -msgstr "" +msgstr "将类放置在适当的关注组（``subscribers/metrics/``、``subscribers/logging/``或``subscribers/tracing/``）下，并从该包的``__init__.py``中导出。" #: ../../source/mp/architecture.rst:380 msgid "" @@ -578,15 +578,15 @@ msgid "" "``bus.register_subscriber(...)`` inside the branch matching its concern " "(metrics / logging / tracing), gated on the corresponding CLI flag if " "needed." -msgstr "" +msgstr "在 ``init_observability()`` 中注册订阅者 (``lmcache/v1/mp_observability/config.py``)，通过 ``bus.register_subscriber(...)`` 在与其关注点 (metrics / logging / tracing) 匹配的分支中进行注册，如有需要，受相应 CLI 标志的限制。" #: ../../source/mp/architecture.rst:387 msgid "Adding a new request type" -msgstr "" +msgstr "添加新的请求类型" #: ../../source/mp/architecture.rst:389 msgid "Add a new member to ``RequestType`` in ``protocols/base.py``." -msgstr "" +msgstr "在 ``protocols/base.py`` 中向 ``RequestType`` 添加一个新成员。" #: ../../source/mp/architecture.rst:390 msgid "" @@ -594,199 +594,199 @@ msgid "" "file (``engine``, ``controller``, ``observability``, ``debug``, " "``blend``, or ``blend_v2``) and add the request name to that module's " "``REQUEST_NAMES``." -msgstr "" +msgstr "在适当的 ``protocols/*.py`` 文件中创建一个 ``ProtocolDefinition``（``engine``、``controller``、``observability``、``debug``、``blend`` 或 ``blend_v2``），并将请求名称添加到该模块的 ``REQUEST_NAMES`` 中。" #: ../../source/mp/architecture.rst:393 msgid "Implement the handler method on ``MPCacheEngine`` (or ``BlendEngineV2``)." -msgstr "" +msgstr "在 ``MPCacheEngine`` （或 ``BlendEngineV2``）上实现处理程序方法。" #: ../../source/mp/architecture.rst:394 msgid "" "Register the handler in ``run_cache_server()`` via " "``add_handler_helper()``." -msgstr "" +msgstr "在 ``run_cache_server()`` 中通过 ``add_handler_helper()`` 注册处理程序。" #: ../../source/mp/architecture.rst:397 msgid "Key Source Files" -msgstr "" +msgstr "关键源文件" #: ../../source/mp/architecture.rst:403 msgid "File" -msgstr "" +msgstr "文件" #: ../../source/mp/architecture.rst:404 msgid "Purpose" -msgstr "" +msgstr "目的" #: ../../source/mp/architecture.rst:405 msgid "``lmcache/v1/multiprocess/server.py``" -msgstr "" +msgstr "``lmcache/v1/multiprocess/server.py``" #: ../../source/mp/architecture.rst:406 msgid "MPCacheEngine + ZMQ server entry point" -msgstr "" +msgstr "MPCacheEngine + ZMQ 服务器入口点" #: ../../source/mp/architecture.rst:407 msgid "``lmcache/v1/multiprocess/config.py``" -msgstr "" +msgstr "``lmcache/v1/multiprocess/config.py``" #: ../../source/mp/architecture.rst:408 msgid "MPServerConfig, HTTPFrontendConfig" -msgstr "" +msgstr "MPServerConfig, HTTPFrontendConfig" #: ../../source/mp/architecture.rst:409 msgid "``lmcache/v1/multiprocess/blend_server_v2.py``" -msgstr "" +msgstr "``lmcache/v1/multiprocess/blend_server_v2.py``" #: ../../source/mp/architecture.rst:410 msgid "BlendEngineV2 (extends MPCacheEngine)" -msgstr "" +msgstr "BlendEngineV2 (extends MPCacheEngine)" #: ../../source/mp/architecture.rst:411 msgid "``lmcache/v1/multiprocess/http_server.py``" -msgstr "" +msgstr "``lmcache/v1/multiprocess/http_server.py``" #: ../../source/mp/architecture.rst:412 msgid "FastAPI wrapper with health check and many other useful APIs" -msgstr "" +msgstr "带健康检查和许多其他有用 API 的 FastAPI 包装器" #: ../../source/mp/architecture.rst:413 msgid "``lmcache/v1/multiprocess/http_api_registry.py``" -msgstr "" +msgstr "``lmcache/v1/multiprocess/http_api_registry.py``" #: ../../source/mp/architecture.rst:414 msgid "``HTTPAPIRegistry`` that auto-discovers routers in ``http_apis/``" -msgstr "" +msgstr "``HTTPAPIRegistry`` 自动发现 ``http_apis/`` 中的路由器" #: ../../source/mp/architecture.rst:415 msgid "``lmcache/v1/multiprocess/http_apis/``" -msgstr "" +msgstr "``lmcache/v1/multiprocess/http_apis/``" #: ../../source/mp/architecture.rst:416 msgid "" "Extensible HTTP endpoints (``/``, ``/healthcheck``, ``/clear-cache``, " "``/status``)" -msgstr "" +msgstr "可扩展的 HTTP 端点 (``/``, ``/healthcheck``, ``/clear-cache``, ``/status``)" #: ../../source/mp/architecture.rst:418 msgid "``lmcache/v1/multiprocess/mp_runtime_plugin_launcher.py``" -msgstr "" +msgstr "``lmcache/v1/multiprocess/mp_runtime_plugin_launcher.py``" #: ../../source/mp/architecture.rst:419 msgid "" "``MPRuntimePluginLauncher`` that spawns runtime plugins with the full " "server config serialized into environment variables" -msgstr "" +msgstr "``MPRuntimePluginLauncher`` 通过将完整的服务器配置序列化为环境变量来生成运行时插件" #: ../../source/mp/architecture.rst:421 msgid "``lmcache/v1/multiprocess/protocols/base.py``" -msgstr "" +msgstr "``lmcache/v1/multiprocess/protocols/base.py``" #: ../../source/mp/architecture.rst:422 msgid "RequestType, HandlerType, ProtocolDefinition" -msgstr "" +msgstr "请求类型、处理程序类型、协议定义" #: ../../source/mp/architecture.rst:424 msgid "StorageManager (top-level manager)" -msgstr "" +msgstr "存储管理器（顶层管理器）" #: ../../source/mp/architecture.rst:425 msgid "``lmcache/v1/distributed/config.py``" -msgstr "" +msgstr "``lmcache/v1/distributed/config.py``" #: ../../source/mp/architecture.rst:426 msgid "StorageManagerConfig hierarchy" -msgstr "" +msgstr "StorageManagerConfig 层次结构" #: ../../source/mp/architecture.rst:428 msgid "L1Manager (object state machine)" -msgstr "" +msgstr "L1Manager（对象状态机）" #: ../../source/mp/architecture.rst:429 msgid "``lmcache/v1/distributed/l2_adapters/config.py``" -msgstr "" +msgstr "``lmcache/v1/distributed/l2_adapters/config.py``" #: ../../source/mp/architecture.rst:430 msgid "L2 adapter config registry" -msgstr "" +msgstr "L2 适配器配置注册表" #: ../../source/mp/architecture.rst:431 msgid "``lmcache/v1/distributed/l2_adapters/base.py``" -msgstr "" +msgstr "``lmcache/v1/distributed/l2_adapters/base.py``" #: ../../source/mp/architecture.rst:432 msgid "L2AdapterInterface" -msgstr "" +msgstr "L2AdapterInterface" #: ../../source/mp/architecture.rst:433 msgid "``lmcache/v1/distributed/storage_controllers/store_controller.py``" -msgstr "" +msgstr "``lmcache/v1/distributed/storage_controllers/store_controller.py``" #: ../../source/mp/architecture.rst:434 msgid "StoreController (event-driven L1->L2)" -msgstr "" +msgstr "StoreController（事件驱动 L1->L2）" #: ../../source/mp/architecture.rst:435 msgid "``lmcache/v1/distributed/storage_controllers/eviction_controller.py``" -msgstr "" +msgstr "``lmcache/v1/distributed/storage_controllers/eviction_controller.py``" #: ../../source/mp/architecture.rst:436 msgid "EvictionController (watermark-triggered)" -msgstr "" +msgstr "逐出控制器（基于水印触发）" #: ../../source/mp/architecture.rst:437 msgid "``lmcache/v1/distributed/storage_controllers/prefetch_controller.py``" -msgstr "" +msgstr "``lmcache/v1/distributed/storage_controllers/prefetch_controller.py``" #: ../../source/mp/architecture.rst:438 msgid "PrefetchController (L2->L1 on miss)" -msgstr "" +msgstr "预取控制器 (未命中时从 L2->L1)" #: ../../source/mp/architecture.rst:439 msgid "``lmcache/v1/mp_observability/config.py``" -msgstr "" +msgstr "``lmcache/v1/mp_observability/config.py``" #: ../../source/mp/architecture.rst:440 msgid "ObservabilityConfig + ``init_observability()`` entry point" -msgstr "" +msgstr "可观察性配置 + ``init_observability()`` 入口点" #: ../../source/mp/architecture.rst:441 msgid "``lmcache/v1/mp_observability/event_bus.py``" -msgstr "" +msgstr "``lmcache/v1/mp_observability/event_bus.py``" #: ../../source/mp/architecture.rst:442 msgid "EventBus singleton and ``EventSubscriber`` base class" -msgstr "" +msgstr "事件总线单例和 ``EventSubscriber`` 基类" #: ../../source/mp/architecture.rst:443 msgid "``lmcache/v1/mp_observability/event.py``" -msgstr "" +msgstr "``lmcache/v1/mp_observability/event.py``" #: ../../source/mp/architecture.rst:444 msgid "``Event`` / ``EventType`` definitions" -msgstr "" +msgstr "``Event`` / ``EventType`` 定义" #: ../../source/mp/architecture.rst:445 msgid "``lmcache/v1/mp_observability/otel_init.py``" -msgstr "" +msgstr "``lmcache/v1/mp_observability/otel_init.py``" #: ../../source/mp/architecture.rst:446 msgid "OTel metrics / tracing provider setup" -msgstr "" +msgstr "OTel 指标 / 跟踪提供程序设置" #: ../../source/mp/architecture.rst:447 msgid "``lmcache/v1/mp_observability/subscribers/``" -msgstr "" +msgstr "``lmcache/v1/mp_observability/subscribers/``" #: ../../source/mp/architecture.rst:448 msgid "Metrics, logging, and tracing subscribers" -msgstr "" +msgstr "指标、日志和追踪订阅者" #: ../../source/mp/architecture.rst:449 msgid "``lmcache/v1/mp_observability/trace/``" -msgstr "" +msgstr "``lmcache/v1/mp_observability/trace/``" #: ../../source/mp/architecture.rst:450 msgid "Trace recording (``--trace-level storage``) capture stack" -msgstr "" +msgstr "跟踪记录 (``--trace-level storage``) 捕获堆栈" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/configuration.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/configuration.po index 7332d59f1d9..6e3ffc01328 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/configuration.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/configuration.po @@ -21,36 +21,36 @@ msgstr "" #: ../../source/mp/configuration.rst:2 msgid "Configuration Reference" -msgstr "" +msgstr "配置参考" #: ../../source/mp/configuration.rst:4 msgid "" "This page documents every CLI argument accepted by the LMCache " "multiprocess server. Arguments are grouped by the config module that " "defines them." -msgstr "" +msgstr "本页面记录了 LMCache 多进程服务器接受的每个 CLI 参数。参数按定义它们的配置模块进行分组。" #: ../../source/mp/configuration.rst:12 msgid "MP Server" -msgstr "" +msgstr "MP 服务器" #: ../../source/mp/configuration.rst:14 ../../source/mp/configuration.rst:104 msgid "Source: ``lmcache/v1/multiprocess/config.py``" -msgstr "" +msgstr "源: ``lmcache/v1/multiprocess/config.py``" #: ../../source/mp/configuration.rst:20 ../../source/mp/configuration.rst:82 #: ../../source/mp/configuration.rst:112 ../../source/mp/configuration.rst:131 #: ../../source/mp/configuration.rst:158 ../../source/mp/configuration.rst:177 #: ../../source/mp/configuration.rst:209 ../../source/mp/configuration.rst:368 msgid "Argument" -msgstr "" +msgstr "参数" #: ../../source/mp/configuration.rst:21 ../../source/mp/configuration.rst:83 #: ../../source/mp/configuration.rst:113 ../../source/mp/configuration.rst:132 #: ../../source/mp/configuration.rst:159 ../../source/mp/configuration.rst:178 #: ../../source/mp/configuration.rst:210 ../../source/mp/configuration.rst:369 msgid "Default" -msgstr "" +msgstr "默认" #: ../../source/mp/configuration.rst:22 ../../source/mp/configuration.rst:84 #: ../../source/mp/configuration.rst:114 ../../source/mp/configuration.rst:133 @@ -58,119 +58,119 @@ msgstr "" #: ../../source/mp/configuration.rst:211 ../../source/mp/configuration.rst:370 #: ../../source/mp/configuration.rst:419 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/mp/configuration.rst:23 msgid "``--host``" -msgstr "" +msgstr "``--host``" #: ../../source/mp/configuration.rst:24 msgid "``localhost``" -msgstr "" +msgstr "``localhost``" #: ../../source/mp/configuration.rst:25 msgid "Host address to bind the ZMQ server." -msgstr "" +msgstr "绑定 ZMQ 服务器的主机地址。" #: ../../source/mp/configuration.rst:26 msgid "``--port``" -msgstr "" +msgstr "``--port``" #: ../../source/mp/configuration.rst:27 msgid "``5555``" -msgstr "" +msgstr "``5555``" #: ../../source/mp/configuration.rst:28 msgid "Port to bind the ZMQ server." -msgstr "" +msgstr "绑定 ZMQ 服务器的端口。" #: ../../source/mp/configuration.rst:29 msgid "``--chunk-size``" -msgstr "" +msgstr "``--chunk-size``" #: ../../source/mp/configuration.rst:30 msgid "``256``" -msgstr "" +msgstr "``256``" #: ../../source/mp/configuration.rst:31 msgid "Chunk size for KV cache operations (in tokens)." -msgstr "" +msgstr "KV Cache 操作的块大小（以 token 为单位）。" #: ../../source/mp/configuration.rst:32 msgid "``--max-workers``" -msgstr "" +msgstr "``--max-workers``" #: ../../source/mp/configuration.rst:33 msgid "``1``" -msgstr "" +msgstr "``1``" #: ../../source/mp/configuration.rst:34 msgid "" "Base number of worker threads. Sets the default for both the GPU " "(affinity) pool and the CPU (normal) pool. Can be overridden per-pool " "with ``--max-gpu-workers`` and ``--max-cpu-workers``." -msgstr "" +msgstr "工作线程的基本数量。为 GPU（亲和性）池和 CPU（正常）池设置默认值。可以通过 ``--max-gpu-workers`` 和 ``--max-cpu-workers`` 针对每个池进行覆盖。" #: ../../source/mp/configuration.rst:37 msgid "``--max-gpu-workers``" -msgstr "" +msgstr "``--max-gpu-workers``" #: ../../source/mp/configuration.rst:38 ../../source/mp/configuration.rst:43 msgid "(inherits ``--max-workers``)" -msgstr "" +msgstr "(继承 ``--max-workers``)" #: ../../source/mp/configuration.rst:39 msgid "" "Worker threads for the GPU affinity pool (STORE/RETRIEVE). Requests from " "the same vLLM instance are always dispatched to the same thread, " "eliminating GPU transfer lock contention." -msgstr "" +msgstr "用于 GPU 亲和力池（存储/检索）的工作线程。来自同一 vLLM 实例的请求始终分配到同一线程，从而消除 GPU 传输锁争用。" #: ../../source/mp/configuration.rst:42 msgid "``--max-cpu-workers``" -msgstr "" +msgstr "``--max-cpu-workers``" #: ../../source/mp/configuration.rst:44 msgid "Worker threads for the normal CPU pool (LOOKUP, etc.)." -msgstr "" +msgstr "正常 CPU 池（查找等）的工作线程。" #: ../../source/mp/configuration.rst:45 msgid "``--hash-algorithm``" -msgstr "" +msgstr "``--hash-algorithm``" #: ../../source/mp/configuration.rst:46 msgid "``blake3``" -msgstr "" +msgstr "``blake3``" #: ../../source/mp/configuration.rst:47 msgid "" "Hash algorithm for token-based operations. Choices: ``builtin``, " "``sha256_cbor``, ``blake3``." -msgstr "" +msgstr "基于令牌的操作的哈希算法。可选项：``builtin``、``sha256_cbor``、``blake3``。" #: ../../source/mp/configuration.rst:49 msgid "``--engine-type``" -msgstr "" +msgstr "``--engine-type``" #: ../../source/mp/configuration.rst:50 ../../source/mp/configuration.rst:213 #: ../../source/mp/configuration.rst:221 msgid "``default``" -msgstr "" +msgstr "``default``" #: ../../source/mp/configuration.rst:51 msgid "" "Cache engine backend type. ``default`` uses MPCacheEngine; ``blend`` uses" " BlendEngineV2 for cross-request KV reuse. Choices: ``default``, " "``blend``." -msgstr "" +msgstr "缓存引擎后端类型。``default`` 使用 MPCacheEngine；``blend`` 使用 BlendEngineV2 进行跨请求的 KV 重用。可选项：``default``，``blend``。" #: ../../source/mp/configuration.rst:55 msgid "``--runtime-plugin-locations``" -msgstr "" +msgstr "``--runtime-plugin-locations``" #: ../../source/mp/configuration.rst:56 msgid "``[]``" -msgstr "" +msgstr "``[]``" #: ../../source/mp/configuration.rst:57 msgid "" @@ -178,16 +178,16 @@ msgid "" "alongside the server. Plugins are spawned by ``MPRuntimePluginLauncher`` " "and receive the full server config via the " "``LMCACHE_RUNTIME_PLUGIN_CONFIG`` environment variable." -msgstr "" +msgstr "零个或多个路径，用于运行时插件脚本或目录，以便与服务器一起启动。插件由 ``MPRuntimePluginLauncher`` 生成，并通过 ``LMCACHE_RUNTIME_PLUGIN_CONFIG`` 环境变量接收完整的服务器配置。" #: ../../source/mp/configuration.rst:61 msgid "``--runtime-plugin-config``" -msgstr "" +msgstr "``--runtime-plugin-config``" #: ../../source/mp/configuration.rst:62 #, python-brace-format msgid "``\"{}\"``" -msgstr "" +msgstr "``\"{}\"``" #: ../../source/mp/configuration.rst:63 #, python-brace-format @@ -196,15 +196,15 @@ msgid "" "``LMCACHE_RUNTIME_PLUGIN_EXTRA_CONFIG``. Example: " "``'{\"plugin.frontend.heartbeat_url\": " "\"http://localhost:5000/heartbeat\"}'``." -msgstr "" +msgstr "通过 ``LMCACHE_RUNTIME_PLUGIN_EXTRA_CONFIG`` 转发到运行时插件的额外键值配置的 JSON 字符串。示例：``'{\\\"plugin.frontend.heartbeat_url\\\": \\\"http://localhost:5000/heartbeat\\\"}'``。" #: ../../source/mp/configuration.rst:68 msgid "Lookup Hash Logging" -msgstr "" +msgstr "查找哈希日志记录" #: ../../source/mp/configuration.rst:70 msgid "Source: ``lmcache/v1/mp_observability/subscribers/logging/lookup_hash.py``" -msgstr "" +msgstr "源: ``lmcache/v1/mp_observability/subscribers/logging/lookup_hash.py``" #: ../../source/mp/configuration.rst:72 msgid "" @@ -213,188 +213,188 @@ msgid "" "``LookupHashLoggingSubscriber`` writes these to rotating JSONL files for " "offline analysis. Disabled by default. These arguments are part of the " "Observability group." -msgstr "" +msgstr "启用时，服务器将在 EventBus 上发布在 ``lookup()`` 期间计算的块哈希作为 ``MP_LOOKUP`` 事件。 ``LookupHashLoggingSubscriber`` 将这些写入旋转的 JSONL 文件以供离线分析。 默认情况下禁用。 这些参数是可观察性组的一部分。" #: ../../source/mp/configuration.rst:85 msgid "``--lookup-hash-log-dir``" -msgstr "" +msgstr "``--lookup-hash-log-dir``" #: ../../source/mp/configuration.rst:86 msgid "``\"\"`` (disabled)" -msgstr "" +msgstr "``\"\"`` (禁用)" #: ../../source/mp/configuration.rst:87 msgid "" "Directory to write lookup hash JSONL files. An empty string disables " "logging." -msgstr "" +msgstr "写入查找哈希 JSONL 文件的目录。空字符串将禁用日志记录。" #: ../../source/mp/configuration.rst:89 msgid "``--lookup-hash-log-rotation-interval``" -msgstr "" +msgstr "``--lookup-hash-log-rotation-interval``" #: ../../source/mp/configuration.rst:90 msgid "``21600`` (6 h)" -msgstr "" +msgstr "``21600`` (6 小时)" #: ../../source/mp/configuration.rst:91 msgid "Time interval in seconds before rotating to a new log file." -msgstr "" +msgstr "在切换到新日志文件之前的时间间隔（以秒为单位）。" #: ../../source/mp/configuration.rst:92 msgid "``--lookup-hash-log-rotation-max-size``" -msgstr "" +msgstr "``--lookup-hash-log-rotation-max-size``" #: ../../source/mp/configuration.rst:93 msgid "``104857600`` (100 MB)" -msgstr "" +msgstr "``104857600`` (100 MB)" #: ../../source/mp/configuration.rst:94 msgid "" "Max file size in bytes before rotating even if the time interval has not " "elapsed." -msgstr "" +msgstr "在时间间隔尚未到达之前，旋转前的最大文件大小（以字节为单位）。" #: ../../source/mp/configuration.rst:96 msgid "``--lookup-hash-log-max-files``" -msgstr "" +msgstr "``--lookup-hash-log-max-files``" #: ../../source/mp/configuration.rst:97 msgid "``100``" -msgstr "" +msgstr "``100``" #: ../../source/mp/configuration.rst:98 msgid "" "Max number of log files to keep. Oldest files are deleted when this " "limit is exceeded." -msgstr "" +msgstr "保留的最大日志文件数量。当超过此限制时，最旧的文件将被删除。" #: ../../source/mp/configuration.rst:102 msgid "HTTP Frontend" -msgstr "" +msgstr "HTTP 前端" #: ../../source/mp/configuration.rst:106 msgid "The HTTP frontend is included when running ``lmcache server``." -msgstr "" +msgstr "HTTP 前端在运行 ``lmcache server`` 时包含在内。" #: ../../source/mp/configuration.rst:115 msgid "``--http-host``" -msgstr "" +msgstr "``--http-host``" #: ../../source/mp/configuration.rst:116 msgid "``0.0.0.0``" -msgstr "" +msgstr "``0.0.0.0``" #: ../../source/mp/configuration.rst:117 msgid "Host to bind the HTTP (FastAPI/uvicorn) server." -msgstr "" +msgstr "绑定 HTTP (FastAPI/uvicorn) 服务器的主机。" #: ../../source/mp/configuration.rst:118 msgid "``--http-port``" -msgstr "" +msgstr "``--http-port``" #: ../../source/mp/configuration.rst:119 msgid "``8080``" -msgstr "" +msgstr "``8080``" #: ../../source/mp/configuration.rst:120 msgid "Port to bind the HTTP server." -msgstr "" +msgstr "绑定 HTTP 服务器的端口。" #: ../../source/mp/configuration.rst:123 msgid "L1 Memory Manager" -msgstr "" +msgstr "L1 内存管理器" #: ../../source/mp/configuration.rst:125 ../../source/mp/configuration.rst:152 #: ../../source/mp/configuration.rst:171 ../../source/mp/configuration.rst:203 msgid "Source: ``lmcache/v1/distributed/config.py``" -msgstr "" +msgstr "来源: ``lmcache/v1/distributed/config.py``" #: ../../source/mp/configuration.rst:134 msgid "``--l1-size-gb``" -msgstr "" +msgstr "``--l1-size-gb``" #: ../../source/mp/configuration.rst:135 ../../source/mp/configuration.rst:181 msgid "*required*" -msgstr "" +msgstr "*必需*" #: ../../source/mp/configuration.rst:136 msgid "Size of L1 memory in GB." -msgstr "" +msgstr "L1 内存的大小（以 GB 为单位）。" #: ../../source/mp/configuration.rst:137 msgid "``--l1-use-lazy`` / ``--no-l1-use-lazy``" -msgstr "" +msgstr "``--l1-use-lazy`` / ``--no-l1-use-lazy``" #: ../../source/mp/configuration.rst:138 msgid "``True``" -msgstr "" +msgstr "``True``" #: ../../source/mp/configuration.rst:139 msgid "" "Enable or disable lazy allocation for L1 memory. Pass ``--l1-use-lazy`` " "to enable (default) or ``--no-l1-use-lazy`` to explicitly disable." -msgstr "" +msgstr "启用或禁用 L1 内存的延迟分配。传递 ``--l1-use-lazy`` 以启用（默认）或 ``--no-l1-use-lazy`` 以显式禁用。" #: ../../source/mp/configuration.rst:142 msgid "``--l1-init-size-gb``" -msgstr "" +msgstr "``--l1-init-size-gb``" #: ../../source/mp/configuration.rst:143 msgid "``20``" -msgstr "" +msgstr "``20``" #: ../../source/mp/configuration.rst:144 msgid "Initial allocation size (GB) when using lazy allocation." -msgstr "" +msgstr "使用延迟分配时的初始分配大小（GB）。" #: ../../source/mp/configuration.rst:145 msgid "``--l1-align-bytes``" -msgstr "" +msgstr "``--l1-align-bytes``" #: ../../source/mp/configuration.rst:146 msgid "``4096``" -msgstr "" +msgstr "``4096``" #: ../../source/mp/configuration.rst:147 msgid "Alignment size in bytes (default 4 KB)." -msgstr "" +msgstr "对齐大小（以字节为单位，默认 4 KB）。" #: ../../source/mp/configuration.rst:150 msgid "L1 Manager TTLs" -msgstr "" +msgstr "L1 管理器 TTLs" #: ../../source/mp/configuration.rst:161 msgid "``--l1-write-ttl-seconds``" -msgstr "" +msgstr "``--l1-write-ttl-seconds``" #: ../../source/mp/configuration.rst:162 msgid "``600``" -msgstr "" +msgstr "``600``" #: ../../source/mp/configuration.rst:163 msgid "Time-to-live for each object's write lock (seconds)." -msgstr "" +msgstr "每个对象的写锁的生存时间（秒）。" #: ../../source/mp/configuration.rst:164 msgid "``--l1-read-ttl-seconds``" -msgstr "" +msgstr "``--l1-read-ttl-seconds``" #: ../../source/mp/configuration.rst:165 msgid "``300``" -msgstr "" +msgstr "``300``" #: ../../source/mp/configuration.rst:166 msgid "Time-to-live for each object's read lock (seconds)." -msgstr "" +msgstr "每个对象的读取锁的生存时间（秒）。" #: ../../source/mp/configuration.rst:169 msgid "Eviction Policy" -msgstr "" +msgstr "逐出策略" #: ../../source/mp/configuration.rst:180 msgid "``--eviction-policy``" -msgstr "" +msgstr "``--eviction-policy``" #: ../../source/mp/configuration.rst:182 msgid "" @@ -406,39 +406,39 @@ msgid "" "http-quota-api`); a ``cache_salt`` with no registered quota has an " "effective limit of ``0`` bytes, so its data is evicted at the next " "eviction cycle (allowlist semantics)." -msgstr "" +msgstr "逐出策略。可选项：``LRU``、``IsolatedLRU``、``noop``。在仅缓冲模式下使用``noop``，此时 L1 作为纯写缓冲区（数据在 L2 存储后从 L1 中删除）。``IsolatedLRU`` 为每个 ``cache_salt`` 维护一个 LRU 列表，并要求在运行时通过 ``/quota`` HTTP 端点配置每个 ``cache_salt`` 的配额（参见 :ref:`mp-http-quota-api`）；没有注册配额的 ``cache_salt`` 的有效限制为 ``0`` 字节，因此其数据将在下一个逐出周期被逐出（白名单语义）。" #: ../../source/mp/configuration.rst:193 msgid "``--eviction-trigger-watermark``" -msgstr "" +msgstr "``--eviction-trigger-watermark``" #: ../../source/mp/configuration.rst:194 msgid "``0.8``" -msgstr "" +msgstr "``0.8``" #: ../../source/mp/configuration.rst:195 msgid "Memory usage ratio (0.0--1.0) that triggers eviction." -msgstr "" +msgstr "触发逐出的内存使用比例 (0.0--1.0)。" #: ../../source/mp/configuration.rst:196 msgid "``--eviction-ratio``" -msgstr "" +msgstr "``--eviction-ratio``" #: ../../source/mp/configuration.rst:197 msgid "``0.2``" -msgstr "" +msgstr "``0.2``" #: ../../source/mp/configuration.rst:198 msgid "Fraction of allocated memory to evict when triggered (0.0--1.0)." -msgstr "" +msgstr "触发时逐出的已分配内存比例 (0.0--1.0)。" #: ../../source/mp/configuration.rst:201 msgid "L2 Policies" -msgstr "" +msgstr "L2 策略" #: ../../source/mp/configuration.rst:212 msgid "``--l2-store-policy``" -msgstr "" +msgstr "``--l2-store-policy``" #: ../../source/mp/configuration.rst:214 msgid "" @@ -447,11 +447,11 @@ msgid "" "all keys to all adapters and keeps L1. The ``skip_l1`` policy stores all " "keys to all adapters and then deletes them from L1 (buffer-only mode). " "Choices: ``default``, ``skip_l1``." -msgstr "" +msgstr "L2 存储策略。确定每个适配器接收哪些键，以及在 L2 存储后是否从 L1 中删除键。``default`` 策略将所有键存储到所有适配器并保留 L1。``skip_l1`` 策略将所有键存储到所有适配器，然后从 L1 中删除它们（仅缓冲区模式）。可选项：``default``，``skip_l1``。" #: ../../source/mp/configuration.rst:220 msgid "``--l2-prefetch-policy``" -msgstr "" +msgstr "``--l2-prefetch-policy``" #: ../../source/mp/configuration.rst:222 msgid "" @@ -460,30 +460,30 @@ msgid "" " (lowest index). Prefetched keys are temporary (deleted after the reader " "finishes). The ``retain`` policy uses the same load plan but keeps " "prefetched keys permanently in L1. Choices: ``default``, ``retain``." -msgstr "" +msgstr "L2 预取策略。确定在多个适配器拥有每个键时，哪个适配器加载该键。``default`` 策略选择第一个适配器（最低索引）。预取的键是临时的（在读取器完成后删除）。``retain`` 策略使用相同的加载计划，但将预取的键永久保留在 L1 中。选择：``default``，``retain``。" #: ../../source/mp/configuration.rst:229 msgid "``--l2-prefetch-max-in-flight``" -msgstr "" +msgstr "``--l2-prefetch-max-in-flight``" #: ../../source/mp/configuration.rst:230 msgid "``8``" -msgstr "" +msgstr "``8``" #: ../../source/mp/configuration.rst:231 msgid "" "Maximum number of concurrent prefetch (L2 load) requests. Limits how many" " in-flight loads the PrefetchController may issue at once, preventing " "excessive L1 memory pressure." -msgstr "" +msgstr "最大并发预取（L2 加载）请求的数量。限制 PrefetchController 同时发出的在途加载数量，以防止过度的 L1 内存压力。" #: ../../source/mp/configuration.rst:236 msgid "L2 Adapters" -msgstr "" +msgstr "L2 适配器" #: ../../source/mp/configuration.rst:238 msgid "Source: ``lmcache/v1/distributed/l2_adapters/config.py``" -msgstr "" +msgstr "源: ``lmcache/v1/distributed/l2_adapters/config.py``" #: ../../source/mp/configuration.rst:240 msgid "" @@ -491,250 +491,250 @@ msgid "" "arguments. Each JSON object must include a ``\"type\"`` field that " "selects the adapter type. The order of ``--l2-adapter`` arguments " "determines the adapter order (cascade)." -msgstr "" +msgstr "L2 适配器通过可重复的 ``--l2-adapter `` 参数进行配置。每个 JSON 对象必须包含一个 ``\"type\"`` 字段，用于选择适配器类型。``--l2-adapter`` 参数的顺序决定了适配器的顺序（级联）。" #: ../../source/mp/configuration.rst:244 msgid "" "Registered adapter types: ``nixl_store``, ``nixl_store_dynamic``, ``fs``," " ``fs_native``, ``mock``, ``mooncake_store``, ``s3``, ``resp``, " "``plugin``, ``native_plugin``, ``raw_block``, ``dax``." -msgstr "" +msgstr "注册的适配器类型：``nixl_store``、``nixl_store_dynamic``、``fs``、``fs_native``、``mock``、``mooncake_store``、``s3``、``resp``、``plugin``、``native_plugin``、``raw_block``、``dax``。" #: ../../source/mp/configuration.rst:249 msgid "``nixl_store`` -- NIXL-based persistent storage" -msgstr "" +msgstr "``nixl_store`` -- 基于 NIXL 的持久存储" #: ../../source/mp/configuration.rst:251 ../../source/mp/configuration.rst:287 #: ../../source/mp/configuration.rst:307 ../../source/mp/configuration.rst:325 msgid "Fields:" -msgstr "" +msgstr "字段：" #: ../../source/mp/configuration.rst:253 msgid "" "``backend`` *(required)*: One of ``POSIX``, ``GDS``, ``GDS_MT``, " "``HF3FS``, ``OBJ``, ``AZURE_BLOB``." -msgstr "" +msgstr "``backend`` *(必需)*: 选项包括 ``POSIX``, ``GDS``, ``GDS_MT``, ``HF3FS``, ``OBJ``, ``AZURE_BLOB``。" #: ../../source/mp/configuration.rst:254 msgid "" "``backend_params`` *(required for file-based backends)*: Dict of string " "key-value pairs. File-based backends (``GDS``, ``GDS_MT``, ``POSIX``, " "``HF3FS``) require ``file_path`` and ``use_direct_io``." -msgstr "" +msgstr "``backend_params`` *(文件基础后端必需)*：字符串键值对的字典。文件基础后端（``GDS``, ``GDS_MT``, ``POSIX``, ``HF3FS``）需要 ``file_path`` 和 ``use_direct_io``。" #: ../../source/mp/configuration.rst:257 msgid "" "``pool_size`` *(required)*: Number of storage descriptors to pre-allocate" " (> 0)." -msgstr "" +msgstr "``pool_size`` *(必需)*: 预分配的存储描述符数量 (> 0)。" #: ../../source/mp/configuration.rst:259 ../../source/mp/configuration.rst:294 msgid "Examples:" -msgstr "" +msgstr "示例：" #: ../../source/mp/configuration.rst:283 msgid "``fs`` -- File-system backed storage" -msgstr "" +msgstr "``fs`` -- 文件系统支持的存储" #: ../../source/mp/configuration.rst:285 msgid "A pure file-system L2 adapter using async I/O." -msgstr "" +msgstr "一个使用异步 I/O 的纯文件系统 L2 适配器。" #: ../../source/mp/configuration.rst:289 msgid "``base_path`` *(required)*: Directory for storing KV cache files." -msgstr "" +msgstr "``base_path`` *(必需)*: 存储 KV Cache 文件的目录。" #: ../../source/mp/configuration.rst:290 msgid "``relative_tmp_dir`` *(optional)*: Relative sub-dir for temp files." -msgstr "" +msgstr "``relative_tmp_dir`` *(可选)*: 临时文件的相对子目录。" #: ../../source/mp/configuration.rst:291 msgid "" "``read_ahead_size`` *(optional)*: Trigger read-ahead by reading this many" " bytes first." -msgstr "" +msgstr "``read_ahead_size`` *(可选)*: 通过首先读取这么多字节来触发预读。" #: ../../source/mp/configuration.rst:292 msgid "" "``use_odirect`` *(optional)*: Bypass page cache via ``O_DIRECT`` (default" " ``false``)." -msgstr "" +msgstr "``use_odirect`` *(可选)*：通过 ``O_DIRECT`` 跳过页面缓存（默认 ``false``）。" #: ../../source/mp/configuration.rst:305 msgid "``mock`` -- Mock adapter for testing" -msgstr "" +msgstr "``mock`` -- 测试用的模拟适配器" #: ../../source/mp/configuration.rst:309 msgid "``max_size_gb`` *(required)*: Maximum size of the adapter in GB (> 0)." -msgstr "" +msgstr "``max_size_gb`` *(必需)*：适配器的最大大小（以 GB 为单位，> 0）。" #: ../../source/mp/configuration.rst:310 msgid "``mock_bandwidth_gb`` *(required)*: Simulated bandwidth in GB/sec (> 0)." -msgstr "" +msgstr "``mock_bandwidth_gb`` *(必需)*: 模拟带宽，单位为 GB/秒 (> 0)。" #: ../../source/mp/configuration.rst:312 ../../source/mp/configuration.rst:340 msgid "Example:" -msgstr "" +msgstr "示例：" #: ../../source/mp/configuration.rst:319 msgid "``s3`` -- S3-compatible object store" -msgstr "" +msgstr "``s3`` -- 兼容 S3 的对象存储" #: ../../source/mp/configuration.rst:321 msgid "" "S3-backed L2 adapter using the AWS CRT (Common Runtime) for high-" "throughput transfers to AWS S3 or any S3-compatible endpoint. See " ":doc:`l2_storage` for details." -msgstr "" +msgstr "使用 AWS CRT（通用运行时）的 S3 后端 L2 适配器，以实现高吞吐量传输到 AWS S3 或任何 S3 兼容的端点。有关详细信息，请参见 :doc:`l2_storage`。" #: ../../source/mp/configuration.rst:327 msgid "" "``s3_endpoint`` *(required)*: Bucket URL, either ``\"s3://\"`` or" " the bare host form." -msgstr "" +msgstr "``s3_endpoint`` *(必需)*: 存储桶 URL，可以是 ``\"s3://\"`` 或裸主机形式。" #: ../../source/mp/configuration.rst:329 msgid "``s3_region`` *(required)*: AWS region string." -msgstr "" +msgstr "``s3_region`` *(必需)*: AWS 区域字符串。" #: ../../source/mp/configuration.rst:330 msgid "``s3_num_io_threads`` *(optional, default ``64``)*: CRT I/O threads." -msgstr "" +msgstr "``s3_num_io_threads`` *(可选，默认 ``64``)*: CRT I/O 线程。" #: ../../source/mp/configuration.rst:331 msgid "" "``s3_prefer_http2`` *(optional, default ``true``)*: Negotiate HTTP/2 via " "ALPN." -msgstr "" +msgstr "``s3_prefer_http2`` *(可选，默认 ``true``)*: 通过 ALPN 协商 HTTP/2。" #: ../../source/mp/configuration.rst:332 msgid "" "``s3_enable_s3express`` *(optional, default ``false``)*: Enable S3 " "Express signing." -msgstr "" +msgstr "``s3_enable_s3express`` *(可选，默认 ``false``)*: 启用 S3 Express 签名。" #: ../../source/mp/configuration.rst:333 msgid "" "``disable_tls`` *(optional, default ``false``)*: Bypass TLS (for non-AWS " "HTTP endpoints)." -msgstr "" +msgstr "``disable_tls`` *(可选，默认 ``false``)*: 跳过 TLS（用于非 AWS HTTP 端点）。" #: ../../source/mp/configuration.rst:335 msgid "" "``aws_access_key_id`` / ``aws_secret_access_key`` *(optional)*: Static " "credentials; omit to use the default credential provider chain." -msgstr "" +msgstr "``aws_access_key_id`` / ``aws_secret_access_key`` *(可选)*: 静态凭证；省略以使用默认凭证提供程序链。" #: ../../source/mp/configuration.rst:337 msgid "" "``max_capacity_gb`` *(optional, default ``0.0``)*: Aggregate capacity " "used by ``get_usage()``. A value of ``0`` disables aggregate eviction." -msgstr "" +msgstr "``max_capacity_gb`` *(可选，默认 ``0.0``)*: ``get_usage()`` 使用的总容量。值为 ``0`` 将禁用总的逐出。" #: ../../source/mp/configuration.rst:347 msgid "Multiple adapters (cascade)" -msgstr "" +msgstr "多个适配器（级联）" #: ../../source/mp/configuration.rst:349 msgid "" "Pass ``--l2-adapter`` multiple times. Adapters are used in the order " "given:" -msgstr "" +msgstr "多次传递 ``--l2-adapter``。适配器按给定顺序使用：" #: ../../source/mp/configuration.rst:357 msgid "Observability" -msgstr "" +msgstr "可观察性" #: ../../source/mp/configuration.rst:359 msgid "Source: ``lmcache/v1/mp_observability/config.py``" -msgstr "" +msgstr "源: ``lmcache/v1/mp_observability/config.py``" #: ../../source/mp/configuration.rst:361 msgid "" "See :doc:`observability` for full details on the three modes (metrics, " "logging, tracing)." -msgstr "" +msgstr "有关三种模式（指标、日志、跟踪）的完整详细信息，请参见 :doc:`observability`。" #: ../../source/mp/configuration.rst:371 msgid "``--disable-observability``" -msgstr "" +msgstr "``--disable-observability``" #: ../../source/mp/configuration.rst:372 ../../source/mp/configuration.rst:375 #: ../../source/mp/configuration.rst:378 ../../source/mp/configuration.rst:381 msgid "off" -msgstr "" +msgstr "关闭" #: ../../source/mp/configuration.rst:373 msgid "Master switch: disable the EventBus entirely." -msgstr "" +msgstr "主开关：完全禁用 EventBus。" #: ../../source/mp/configuration.rst:374 msgid "``--disable-metrics``" -msgstr "" +msgstr "``--disable-metrics``" #: ../../source/mp/configuration.rst:376 msgid "Skip metrics subscribers (no Prometheus endpoint)." -msgstr "" +msgstr "跳过指标订阅者（没有 Prometheus 端点）。" #: ../../source/mp/configuration.rst:377 msgid "``--disable-logging``" -msgstr "" +msgstr "``--disable-logging``" #: ../../source/mp/configuration.rst:379 msgid "Skip logging subscribers." -msgstr "" +msgstr "跳过日志订阅者。" #: ../../source/mp/configuration.rst:380 msgid "``--enable-tracing``" -msgstr "" +msgstr "``--enable-tracing``" #: ../../source/mp/configuration.rst:382 msgid "Register tracing subscribers. Requires ``--otlp-endpoint``." -msgstr "" +msgstr "注册追踪订阅者。需要 ``--otlp-endpoint``。" #: ../../source/mp/configuration.rst:383 msgid "``--event-bus-queue-size``" -msgstr "" +msgstr "``--event-bus-queue-size``" #: ../../source/mp/configuration.rst:384 msgid "``10000``" -msgstr "" +msgstr "``10000``" #: ../../source/mp/configuration.rst:385 msgid "Max events in the EventBus queue before tail-drop." -msgstr "" +msgstr "事件总线队列中最大事件数，超过后将进行尾部丢弃。" #: ../../source/mp/configuration.rst:386 msgid "``--otlp-endpoint``" -msgstr "" +msgstr "``--otlp-endpoint``" #: ../../source/mp/configuration.rst:387 msgid "*(none)*" -msgstr "" +msgstr "*(无)*" #: ../../source/mp/configuration.rst:388 msgid "OTLP gRPC endpoint for exporting metrics and traces." -msgstr "" +msgstr "用于导出指标和跟踪的 OTLP gRPC 端点。" #: ../../source/mp/configuration.rst:389 msgid "``--prometheus-port``" -msgstr "" +msgstr "``--prometheus-port``" #: ../../source/mp/configuration.rst:390 msgid "``9090``" -msgstr "" +msgstr "``9090``" #: ../../source/mp/configuration.rst:391 msgid "Port for the Prometheus ``/metrics`` endpoint." -msgstr "" +msgstr "Prometheus ``/metrics`` 端点的端口。" #: ../../source/mp/configuration.rst:392 msgid "``--service-instance-id``" -msgstr "" +msgstr "``--service-instance-id``" #: ../../source/mp/configuration.rst:393 msgid "*(unset, default UUID v4)*" -msgstr "" +msgstr "*(未设置，默认 UUID v4)*" #: ../../source/mp/configuration.rst:394 msgid "" @@ -742,47 +742,47 @@ msgid "" "attribute ``service.instance.id`` on every metric and span. When the flag" " is not passed, defaults to a random UUID v4. Pass ``--service-instance-" "id=\"\"`` to force an empty value." -msgstr "" +msgstr "此 MP 服务器实例的标识符，作为 OTel 资源属性 ``service.instance.id`` 附加在每个指标和跨度上。当未传递该标志时，默认为随机 UUID v4。传递 ``--service-instance-id=\\\"\\\"`` 以强制设置为空值。" #: ../../source/mp/configuration.rst:400 msgid "vLLM Client Configuration" -msgstr "" +msgstr "vLLM 客户端配置" #: ../../source/mp/configuration.rst:402 msgid "" "On the vLLM side, specify the LMCache server host and port via the " "``kv_connector_extra_config`` parameter:" -msgstr "" +msgstr "在 vLLM 端，通过 ``kv_connector_extra_config`` 参数指定 LMCache 服务器的主机和端口：" #: ../../source/mp/configuration.rst:412 msgid "Environment Variables" -msgstr "" +msgstr "环境变量" #: ../../source/mp/configuration.rst:418 msgid "Variable" -msgstr "" +msgstr "变量" #: ../../source/mp/configuration.rst:420 msgid "``LMCACHE_LOG_LEVEL``" -msgstr "" +msgstr "``LMCACHE_LOG_LEVEL``" #: ../../source/mp/configuration.rst:421 msgid "" "Log level for LMCache (``DEBUG``, ``INFO``, ``WARNING``, ``ERROR``). Set " "to ``DEBUG`` to see L2 store activity, prefetch results, etc." -msgstr "" +msgstr "LMCache 的日志级别（``DEBUG``、``INFO``、``WARNING``、``ERROR``）。设置为 ``DEBUG`` 以查看 L2 存储活动、预取结果等。" #: ../../source/mp/configuration.rst:423 msgid "``PYTHONHASHSEED``" -msgstr "" +msgstr "``PYTHONHASHSEED``" #: ../../source/mp/configuration.rst:424 msgid "" "Set to a fixed value for reproducible hashing across processes (relevant " "when using ``--hash-algorithm builtin``)." -msgstr "" +msgstr "设置为固定值以实现跨进程的可重复哈希（在使用 ``--hash-algorithm builtin`` 时相关）。" #: ../../source/mp/configuration.rst:428 msgid "Full Example" -msgstr "" +msgstr "完整示例" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/deployment.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/deployment.po index fc31ef2e146..c7c17399c68 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/deployment.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/deployment.po @@ -21,171 +21,171 @@ msgstr "" #: ../../source/mp/deployment.rst:2 msgid "Deployment Guide" -msgstr "" +msgstr "部署指南" #: ../../source/mp/deployment.rst:4 msgid "" "This page covers deploying LMCache multiprocess mode in Docker and " "Kubernetes environments, along with production best practices." -msgstr "" +msgstr "本页介绍了在 Docker 和 Kubernetes 环境中部署 LMCache 多进程模式的最佳实践。" #: ../../source/mp/deployment.rst:12 msgid "Docker" -msgstr "" +msgstr "Docker" #: ../../source/mp/deployment.rst:14 msgid "**LMCache container:**" -msgstr "" +msgstr "**LMCache 容器：**" #: ../../source/mp/deployment.rst:25 msgid "**vLLM container:**" -msgstr "" +msgstr "**vLLM 容器：**" #: ../../source/mp/deployment.rst:37 msgid "Required Docker flags:" -msgstr "" +msgstr "所需的 Docker 标志：" #: ../../source/mp/deployment.rst:39 msgid "" "``--network host`` -- Allows the vLLM container to reach LMCache on " "localhost." -msgstr "" +msgstr "``--network host`` -- 允许 vLLM 容器访问本地主机上的 LMCache。" #: ../../source/mp/deployment.rst:40 msgid "" "``--ipc host`` -- Required for CUDA IPC shared memory transfers between " "containers." -msgstr "" +msgstr "``--ipc host`` -- 需要用于容器之间的 CUDA IPC 共享内存传输。" #: ../../source/mp/deployment.rst:42 msgid "" "``--runtime nvidia --gpus all`` -- GPU access via the NVIDIA container " "runtime." -msgstr "" +msgstr "``--runtime nvidia --gpus all`` -- 通过 NVIDIA 容器运行时访问 GPU。" #: ../../source/mp/deployment.rst:45 msgid "**HTTP server variant:**" -msgstr "" +msgstr "**HTTP 服务器变体：**" #: ../../source/mp/deployment.rst:47 msgid "" "For health-check and cache management API support (useful with container " "orchestrators), use the HTTP server entry point:" -msgstr "" +msgstr "对于健康检查和缓存管理 API 支持（在容器编排器中非常有用），请使用 HTTP 服务器入口点：" #: ../../source/mp/deployment.rst:60 msgid "Kubernetes" -msgstr "" +msgstr "Kubernetes" #: ../../source/mp/deployment.rst:62 msgid "" "LMCache is designed for a **DaemonSet + Deployment** pattern: one LMCache" " server per node (DaemonSet) shared by multiple vLLM pods (Deployment)." -msgstr "" +msgstr "LMCache 设计为 **DaemonSet + Deployment** 模式：每个节点一个 LMCache 服务器（DaemonSet），由多个 vLLM pod（Deployment）共享。" #: ../../source/mp/deployment.rst:65 msgid "Example YAML files are provided in ``examples/multi_process/``." -msgstr "" +msgstr "在 ``examples/multi_process/`` 中提供了示例 YAML 文件。" #: ../../source/mp/deployment.rst:68 msgid "Prerequisites" -msgstr "" +msgstr "前提条件" #: ../../source/mp/deployment.rst:70 msgid "Kubernetes cluster with GPU support (NVIDIA GPU Operator installed)" -msgstr "" +msgstr "支持 GPU 的 Kubernetes 集群（安装了 NVIDIA GPU Operator）" #: ../../source/mp/deployment.rst:71 msgid "At least 4 GPUs per node" -msgstr "" +msgstr "每个节点至少需要 4 个 GPU" #: ../../source/mp/deployment.rst:72 msgid "``kubectl`` configured to access your cluster" -msgstr "" +msgstr "``kubectl`` 配置为访问您的集群" #: ../../source/mp/deployment.rst:75 msgid "Step-by-Step" -msgstr "" +msgstr "逐步指南" #: ../../source/mp/deployment.rst:77 msgid "**Step 1: Create namespace**" -msgstr "" +msgstr "**步骤 1：创建命名空间**" #: ../../source/mp/deployment.rst:83 msgid "**Step 2: Deploy LMCache DaemonSet**" -msgstr "" +msgstr "**步骤 2：部署 LMCache DaemonSet**" #: ../../source/mp/deployment.rst:89 msgid "**Step 3: Deploy vLLM**" -msgstr "" +msgstr "**步骤 3：部署 vLLM**" #: ../../source/mp/deployment.rst:96 msgid "" "The default model is ``Qwen/Qwen3-14B``. For gated models (e.g., Llama)," " create a Secret with your Hugging Face token:" -msgstr "" +msgstr "默认模型是 ``Qwen/Qwen3-14B``。对于受限模型（例如 Llama），请使用您的 Hugging Face 令牌创建一个 Secret：" #: ../../source/mp/deployment.rst:105 msgid "Then add the ``HF_TOKEN`` environment variable to the vLLM container spec." -msgstr "" +msgstr "然后将 ``HF_TOKEN`` 环境变量添加到 vLLM 容器规格中。" #: ../../source/mp/deployment.rst:107 msgid "**Step 4: Monitor deployment**" -msgstr "" +msgstr "**步骤 4：监控部署**" #: ../../source/mp/deployment.rst:126 msgid "**Step 5: Send test requests**" -msgstr "" +msgstr "**步骤 5：发送测试请求**" #: ../../source/mp/deployment.rst:141 msgid "Architecture Notes" -msgstr "" +msgstr "架构说明" #: ../../source/mp/deployment.rst:143 msgid "" "**DaemonSet uses ``hostNetwork: true``** so vLLM pods discover the " "LMCache server via ``status.hostIP``." -msgstr "" +msgstr "**DaemonSet 使用 ``hostNetwork: true``** 以便 vLLM pod 通过 ``status.hostIP`` 发现 LMCache 服务器。" #: ../../source/mp/deployment.rst:145 msgid "" "**Both containers mount ``/dev/shm``** from the host to enable CUDA IPC " "memory sharing." -msgstr "" +msgstr "**两个容器从主机挂载 ``/dev/shm``** 以启用 CUDA IPC 内存共享。" #: ../../source/mp/deployment.rst:147 msgid "" "**GPUs are NOT requested in the DaemonSet** -- this allows GPUs to remain" " exclusively allocated to vLLM pods. The NVIDIA container runtime " "automatically provides GPU access for IPC-based memory transfers." -msgstr "" +msgstr "**不在 DaemonSet 中请求 GPU** -- 这允许 GPU 保持专门分配给 vLLM pod。NVIDIA 容器运行时自动提供 GPU 访问以进行基于 IPC 的内存传输。" #: ../../source/mp/deployment.rst:150 msgid "" "**Multiple vLLM pods** on the same node automatically connect to the same" " LMCache DaemonSet instance." -msgstr "" +msgstr "**多个 vLLM pod** 在同一节点上会自动连接到同一个 LMCache DaemonSet 实例。" #: ../../source/mp/deployment.rst:154 msgid "" "LMCache pods on nodes without GPUs will crash with CUDA initialization " "errors. This is expected -- LMCache only needs to run on GPU nodes where" " vLLM pods are scheduled." -msgstr "" +msgstr "没有 GPU 的节点上的 LMCache pod 会因 CUDA 初始化错误而崩溃。这是预期的——LMCache 只需要在调度有 vLLM pod 的 GPU 节点上运行。" #: ../../source/mp/deployment.rst:159 msgid "Health Checking (HTTP Server)" -msgstr "" +msgstr "健康检查 (HTTP 服务器)" #: ../../source/mp/deployment.rst:161 msgid "" "For Kubernetes liveness/readiness probes, deploy the HTTP server variant " "instead. Use the ``/healthcheck`` endpoint:" -msgstr "" +msgstr "对于 Kubernetes 活跃性/就绪性探针，请部署 HTTP 服务器变体。使用 ``/healthcheck`` 端点：" #: ../../source/mp/deployment.rst:180 msgid "Monitoring Integration" -msgstr "" +msgstr "监控集成" #: ../../source/mp/deployment.rst:182 msgid "" @@ -193,15 +193,15 @@ msgid "" "``ServiceMonitor`` or Prometheus scrape annotation to collect metrics " "from the LMCache DaemonSet pods. See :doc:`observability` for metric " "details." -msgstr "" +msgstr "默认情况下，Prometheus 指标在 9090 端口启用。添加 ``ServiceMonitor`` 或 Prometheus 抓取注释以从 LMCache DaemonSet Pod 收集指标。有关指标详细信息，请参见 :doc:`observability`。" #: ../../source/mp/deployment.rst:187 msgid "Cleanup" -msgstr "" +msgstr "清理" #: ../../source/mp/deployment.rst:196 msgid "Production Best Practices" -msgstr "" +msgstr "生产最佳实践" #: ../../source/mp/deployment.rst:198 msgid "" @@ -212,43 +212,43 @@ msgid "" "instances sharing the cache server so each instance gets its own " "dedicated thread. Use ``--max-cpu-workers`` to override the CPU pool for" " lookup and other non-GPU operations." -msgstr "" +msgstr "**工作进程数量 (``--max-workers``, ``--max-gpu-workers``, ``--max-cpu-workers``):** ``--max-workers`` 设置 GPU 亲和力池和 CPU 正常池的大小（默认值为 1）。使用 ``--max-gpu-workers`` 独立覆盖 GPU 池——将其设置为至少与共享缓存服务器的 vLLM 实例数量相同，以便每个实例获得自己的专用线程。使用 ``--max-cpu-workers`` 来覆盖查找和其他非 GPU 操作的 CPU 池。" #: ../../source/mp/deployment.rst:205 msgid "" "**L1 memory sizing (``--l1-size-gb``):** Allocate as much CPU memory as " "available after accounting for the OS and vLLM. A larger L1 cache means " "fewer L2 round-trips." -msgstr "" +msgstr "**L1 内存大小 (``--l1-size-gb``):** 在考虑操作系统和 vLLM 后，分配尽可能多的 CPU 内存。更大的 L1 缓存意味着更少的 L2 往返。" #: ../../source/mp/deployment.rst:209 msgid "**Eviction tuning:**" -msgstr "" +msgstr "**逐出调优:**" #: ../../source/mp/deployment.rst:211 #, python-format msgid "" "``--eviction-trigger-watermark 0.8`` (default) triggers eviction when L1 " "is 80% full." -msgstr "" +msgstr "``--eviction-trigger-watermark 0.8``（默认值）在 L1 达到 80% 满时触发逐出。" #: ../../source/mp/deployment.rst:213 #, python-format msgid "" "``--eviction-ratio 0.2`` (default) frees 20% of allocated memory per " "eviction cycle." -msgstr "" +msgstr "``--eviction-ratio 0.2``（默认值）在每个逐出周期释放 20% 的分配内存。" #: ../../source/mp/deployment.rst:215 msgid "" "Lower the watermark or increase the ratio if you observe frequent " "evictions under steady load." -msgstr "" +msgstr "如果在稳定负载下观察到频繁的逐出，请降低水位线或增加比率。" #: ../../source/mp/deployment.rst:218 msgid "" "**Logging:** Use ``LMCACHE_LOG_LEVEL=DEBUG`` during initial setup to " "verify L2 store/load activity. Switch to ``INFO`` (default) for " "production to reduce log volume." -msgstr "" +msgstr "**日志记录:** 在初始设置期间使用 ``LMCACHE_LOG_LEVEL=DEBUG`` 来验证 L2 存储/加载活动。 在生产中切换到 ``INFO``（默认）以减少日志量。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/http_api.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/http_api.po index c2428abe861..810df583d9b 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/http_api.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/http_api.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/mp/http_api.rst:2 msgid "HTTP API" -msgstr "" +msgstr "HTTP API" #: ../../source/mp/http_api.rst:4 msgid "" @@ -30,7 +30,7 @@ msgid "" "socket used by vLLM. This HTTP API is intended for operators, " "orchestrators (e.g. Kubernetes), and debugging tools — it is **not** on " "the inference data path." -msgstr "" +msgstr "当 MP 服务器通过 ``lmcache server`` 启动时（推荐的入口点），会暴露一个基于 FastAPI 的 HTTP 前端，以及 vLLM 使用的 ZMQ 套接字。此 HTTP API 旨在供操作员、编排者（例如 Kubernetes）和调试工具使用——它**不**在推理数据路径上。" #: ../../source/mp/http_api.rst:10 msgid "" @@ -38,7 +38,7 @@ msgid "" "``lmcache/v1/multiprocess/http_apis/``: any module named ``*_api.py`` " "that exposes a module-level ``router`` (a :class:`fastapi.APIRouter`) is " "discovered at startup." -msgstr "" +msgstr "新的端点会自动从 ``lmcache/v1/multiprocess/http_apis/`` 注册：任何名为 ``*_api.py`` 的模块，只要暴露一个模块级的 ``router``（一个 :class:`fastapi.APIRouter`），在启动时都会被发现。" #: ../../source/mp/http_api.rst:15 msgid "" @@ -50,61 +50,61 @@ msgid "" "them to the auto-discovery pipeline. Adding a new compatible module under" " ``internal_api_server/common`` therefore requires no wiring changes on " "the MP side." -msgstr "" +msgstr "在 ``lmcache/v1/internal_api_server/common/`` 下定义的部分路由也在此 HTTP 服务器上暴露。模块 ``lmcache/v1/multiprocess/http_apis/common_api.py`` 聚合了这些路由（跳过在 ``_MP_INCOMPATIBLE_MODULES`` 中列出的模块，例如 ``run_script_api``），并将它们转发到自动发现管道。因此，在 ``internal_api_server/common`` 下添加一个新的兼容模块不需要在 MP 端进行接线更改。" #: ../../source/mp/http_api.rst:29 msgid "Server Configuration" -msgstr "" +msgstr "服务器配置" #: ../../source/mp/http_api.rst:35 msgid "Argument" -msgstr "" +msgstr "参数" #: ../../source/mp/http_api.rst:36 msgid "Default" -msgstr "" +msgstr "默认" #: ../../source/mp/http_api.rst:37 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/mp/http_api.rst:38 msgid "``--http-host``" -msgstr "" +msgstr "``--http-host``" #: ../../source/mp/http_api.rst:39 msgid "``0.0.0.0``" -msgstr "" +msgstr "``0.0.0.0``" #: ../../source/mp/http_api.rst:40 msgid "Host to bind the HTTP server." -msgstr "" +msgstr "绑定 HTTP 服务器的主机。" #: ../../source/mp/http_api.rst:41 msgid "``--http-port``" -msgstr "" +msgstr "``--http-port``" #: ../../source/mp/http_api.rst:42 msgid "``8080``" -msgstr "" +msgstr "``8080``" #: ../../source/mp/http_api.rst:43 msgid "Port to bind the HTTP server." -msgstr "" +msgstr "绑定 HTTP 服务器的端口。" #: ../../source/mp/http_api.rst:45 msgid "Example:" -msgstr "" +msgstr "示例：" #: ../../source/mp/http_api.rst:53 msgid "" "All examples below assume the server is reachable at " "``http://localhost:8080``." -msgstr "" +msgstr "以下所有示例都假设服务器可以通过 ``http://localhost:8080`` 访问。" #: ../../source/mp/http_api.rst:57 msgid "Endpoints" -msgstr "" +msgstr "端点" #: ../../source/mp/http_api.rst:59 msgid "" @@ -112,19 +112,19 @@ msgid "" "(health, status, cache control) is exposed at top-level paths. Routes " "inherited from the shared ``internal_api_server`` package are kept at " "their original paths for compatibility with the vLLM-embedded API server." -msgstr "" +msgstr "下表按目的对路由进行了分组。操作接口（健康检查、状态、缓存控制）在顶层路径中暴露。来自共享 ``internal_api_server`` 包的路由保持在其原始路径上，以便与 vLLM 嵌入的 API 服务器兼容。" #: ../../source/mp/http_api.rst:69 msgid "Method" -msgstr "" +msgstr "方法" #: ../../source/mp/http_api.rst:70 msgid "Path" -msgstr "" +msgstr "路径" #: ../../source/mp/http_api.rst:71 msgid "Purpose" -msgstr "" +msgstr "目的" #: ../../source/mp/http_api.rst:72 ../../source/mp/http_api.rst:75 #: ../../source/mp/http_api.rst:78 ../../source/mp/http_api.rst:84 @@ -135,185 +135,185 @@ msgstr "" #: ../../source/mp/http_api.rst:122 ../../source/mp/http_api.rst:125 #: ../../source/mp/http_api.rst:128 ../../source/mp/http_api.rst:131 msgid "GET" -msgstr "" +msgstr "获取" #: ../../source/mp/http_api.rst:73 msgid "``/``" -msgstr "" +msgstr "``/``" #: ../../source/mp/http_api.rst:74 msgid "Basic liveness ping." -msgstr "" +msgstr "基本存活探测。" #: ../../source/mp/http_api.rst:76 msgid "``/healthcheck``" -msgstr "" +msgstr "``/healthcheck``" #: ../../source/mp/http_api.rst:77 msgid "K8s liveness/readiness probe." -msgstr "" +msgstr "K8s 存活/就绪探针。" #: ../../source/mp/http_api.rst:79 msgid "``/status``" -msgstr "" +msgstr "``/status``" #: ../../source/mp/http_api.rst:80 msgid "Detailed engine status for inspection and debugging." -msgstr "" +msgstr "详细的引擎状态以供检查和调试。" #: ../../source/mp/http_api.rst:81 ../../source/mp/http_api.rst:119 msgid "POST" -msgstr "" +msgstr "POST" #: ../../source/mp/http_api.rst:82 msgid "``/clear-cache``" -msgstr "" +msgstr "``/clear-cache``" #: ../../source/mp/http_api.rst:83 msgid "Force-clear all KV data in L1 (CPU) memory." -msgstr "" +msgstr "强制清除 L1 (CPU) 内存中的所有 KV 数据。" #: ../../source/mp/http_api.rst:85 msgid "``/quota``" -msgstr "" +msgstr "``/quota``" #: ../../source/mp/http_api.rst:86 msgid "List every registered ``cache_salt`` quota with live usage." -msgstr "" +msgstr "列出每个注册的 ``cache_salt`` 配额及其实时使用情况。" #: ../../source/mp/http_api.rst:87 msgid "PUT" -msgstr "" +msgstr "PUT" #: ../../source/mp/http_api.rst:88 ../../source/mp/http_api.rst:91 #: ../../source/mp/http_api.rst:94 #, python-brace-format msgid "``/quota/{cache_salt}``" -msgstr "" +msgstr "``/quota/{cache_salt}``" #: ../../source/mp/http_api.rst:89 msgid "Set or update the quota (in GB) for a ``cache_salt``." -msgstr "" +msgstr "设置或更新 ``cache_salt`` 的配额（以 GB 为单位）。" #: ../../source/mp/http_api.rst:92 msgid "Read the quota and live usage for a single ``cache_salt``." -msgstr "" +msgstr "读取单个 ``cache_salt`` 的配额和实时使用情况。" #: ../../source/mp/http_api.rst:93 msgid "DELETE" -msgstr "" +msgstr "删除" #: ../../source/mp/http_api.rst:95 msgid "Remove a ``cache_salt``'s quota entry (its data is evicted next cycle)." -msgstr "" +msgstr "移除 ``cache_salt`` 的配额条目（其数据将在下一个周期被逐出）。" #: ../../source/mp/http_api.rst:98 msgid "``/conf``" -msgstr "" +msgstr "``/conf``" #: ../../source/mp/http_api.rst:99 msgid "Dump merged server configurations (mp, storage_manager, observability)." -msgstr "" +msgstr "转储合并的服务器配置（mp、存储管理器、可观察性）。" #: ../../source/mp/http_api.rst:102 msgid "``/version``" -msgstr "" +msgstr "``/version``" #: ../../source/mp/http_api.rst:103 msgid "Full version descriptor (package version + commit id)." -msgstr "" +msgstr "完整版本描述符（软件包版本 + 提交 ID）。" #: ../../source/mp/http_api.rst:105 msgid "``/lmc_version``" -msgstr "" +msgstr "``/lmc_version``" #: ../../source/mp/http_api.rst:106 msgid "LMCache package version string." -msgstr "" +msgstr "LMCache 包版本字符串。" #: ../../source/mp/http_api.rst:108 msgid "``/commit_id``" -msgstr "" +msgstr "``/commit_id``" #: ../../source/mp/http_api.rst:109 msgid "Current build commit id." -msgstr "" +msgstr "当前构建提交 ID。" #: ../../source/mp/http_api.rst:111 msgid "``/env``" -msgstr "" +msgstr "``/env``" #: ../../source/mp/http_api.rst:112 msgid "Dump process environment variables (JSON, plain text)." -msgstr "" +msgstr "转储进程环境变量（JSON，纯文本）。" #: ../../source/mp/http_api.rst:114 msgid "``/loglevel``" -msgstr "" +msgstr "``/loglevel``" #: ../../source/mp/http_api.rst:115 msgid "List or inspect logger levels; also accepts ``level`` to mutate." -msgstr "" +msgstr "列出或检查日志记录器级别；也接受 ``level`` 以进行修改。" #: ../../source/mp/http_api.rst:117 msgid "``/metrics``" -msgstr "" +msgstr "``/metrics``" #: ../../source/mp/http_api.rst:118 msgid "Prometheus exposition format." -msgstr "" +msgstr "Prometheus 展示格式。" #: ../../source/mp/http_api.rst:120 msgid "``/metrics/reset``" -msgstr "" +msgstr "``/metrics/reset``" #: ../../source/mp/http_api.rst:121 msgid "Reset all observability metrics to their initial state." -msgstr "" +msgstr "重置所有可观察性指标为其初始状态。" #: ../../source/mp/http_api.rst:123 msgid "``/threads``" -msgstr "" +msgstr "``/threads``" #: ../../source/mp/http_api.rst:124 msgid "Enumerate active Python threads and their stack traces." -msgstr "" +msgstr "列出活动的 Python 线程及其堆栈跟踪。" #: ../../source/mp/http_api.rst:126 msgid "``/periodic-threads``" -msgstr "" +msgstr "``/periodic-threads``" #: ../../source/mp/http_api.rst:127 msgid "List registered periodic threads with summary counts." -msgstr "" +msgstr "列出注册的周期性线程及其摘要计数。" #: ../../source/mp/http_api.rst:129 #, python-brace-format msgid "``/periodic-threads/{thread_name}``" -msgstr "" +msgstr "``/periodic-threads/{thread_name}``" #: ../../source/mp/http_api.rst:130 msgid "Detailed status for a single periodic thread." -msgstr "" +msgstr "单个周期线程的详细状态。" #: ../../source/mp/http_api.rst:132 msgid "``/periodic-threads-health``" -msgstr "" +msgstr "``/periodic-threads-health``" #: ../../source/mp/http_api.rst:133 msgid "Quick health check for critical/high-level periodic threads." -msgstr "" +msgstr "对关键/高层周期线程的快速健康检查。" #: ../../source/mp/http_api.rst:136 msgid "``GET /``" -msgstr "" +msgstr "``GET /``" #: ../../source/mp/http_api.rst:138 msgid "" "Basic liveness check. Returns a static payload indicating the HTTP server" " is running. Use ``/healthcheck`` instead for probes that also verify the" " cache engine is initialized." -msgstr "" +msgstr "基本的存活检查。返回一个静态负载，指示 HTTP 服务器正在运行。对于还验证缓存引擎是否初始化的探测，请使用 ``/healthcheck``。" #: ../../source/mp/http_api.rst:142 ../../source/mp/http_api.rst:165 #: ../../source/mp/http_api.rst:214 ../../source/mp/http_api.rst:279 @@ -323,7 +323,7 @@ msgstr "" #: ../../source/mp/http_api.rst:560 ../../source/mp/http_api.rst:617 #: ../../source/mp/http_api.rst:657 msgid "**Response** (``200 OK``):" -msgstr "" +msgstr "**响应** (``200 OK``):" #: ../../source/mp/http_api.rst:151 ../../source/mp/http_api.rst:182 #: ../../source/mp/http_api.rst:260 ../../source/mp/http_api.rst:296 @@ -334,11 +334,11 @@ msgstr "" #: ../../source/mp/http_api.rst:591 ../../source/mp/http_api.rst:633 #: ../../source/mp/http_api.rst:644 ../../source/mp/http_api.rst:684 msgid "**Example:**" -msgstr "" +msgstr "**示例:**" #: ../../source/mp/http_api.rst:158 msgid "``GET /healthcheck``" -msgstr "" +msgstr "``GET /healthcheck``" #: ../../source/mp/http_api.rst:160 msgid "" @@ -346,19 +346,19 @@ msgid "" "probes. A ``200`` response implies the HTTP server is alive **and** the " "MP cache engine is initialized. A ``503`` response indicates the engine " "is not yet ready (still initializing, or failed to initialize)." -msgstr "" +msgstr "健康检查端点适用于 Kubernetes 的存活和就绪探针。``200`` 响应意味着 HTTP 服务器处于活动状态 **并且** MP 缓存引擎已初始化。``503`` 响应表示引擎尚未准备好（仍在初始化中，或初始化失败）。" #: ../../source/mp/http_api.rst:173 ../../source/mp/http_api.rst:287 msgid "**Response** (``503 Service Unavailable``):" -msgstr "" +msgstr "**响应** (``503 服务不可用``):" #: ../../source/mp/http_api.rst:188 msgid "**Kubernetes probe snippet:**" -msgstr "" +msgstr "**Kubernetes 探针代码片段:**" #: ../../source/mp/http_api.rst:206 msgid "``GET /status``" -msgstr "" +msgstr "``GET /status``" #: ../../source/mp/http_api.rst:208 msgid "" @@ -366,36 +366,36 @@ msgid "" "L2 adapters, registered GPU contexts, active sessions, and in-flight " "prefetch jobs. Intended for operators and debugging, not for monitoring " "(use Prometheus metrics for time-series data — see :doc:`observability`)." -msgstr "" +msgstr "返回 MP 引擎内部状态的详细快照：L1 缓存、L2 适配器、注册的 GPU 上下文、活动会话和正在进行的预取任务。旨在供操作员和调试使用，而非监控（请使用 Prometheus 指标获取时间序列数据 — 参见 :doc:`observability`）。" #: ../../source/mp/http_api.rst:251 msgid "" "**Response** (``503 Service Unavailable``) when the engine has not yet " "been initialized:" -msgstr "" +msgstr "**响应** (``503 服务不可用``) 当引擎尚未初始化时：" #: ../../source/mp/http_api.rst:267 msgid "``POST /clear-cache``" -msgstr "" +msgstr "``POST /clear-cache``" #: ../../source/mp/http_api.rst:269 msgid "Force-clears **all** KV cache data currently held in L1 (CPU) memory." -msgstr "" +msgstr "强制清除当前保存在 L1 (CPU) 内存中的 **所有** KV Cache 数据。" #: ../../source/mp/http_api.rst:273 msgid "" "This endpoint is destructive and bypasses read/write locks. In-flight " "store or prefetch operations may be corrupted. Use only when the server " "is idle, or when recovering from a known-bad cache state." -msgstr "" +msgstr "此端点是破坏性的，并绕过读/写锁。正在进行的存储或预取操作可能会被损坏。仅在服务器空闲时或从已知的坏缓存状态恢复时使用。" #: ../../source/mp/http_api.rst:277 msgid "The request body is ignored." -msgstr "" +msgstr "请求体将被忽略。" #: ../../source/mp/http_api.rst:305 msgid "``/quota`` — per-``cache_salt`` quota management" -msgstr "" +msgstr "``/quota`` — 每个``cache_salt``的配额管理" #: ../../source/mp/http_api.rst:307 msgid "" @@ -406,14 +406,14 @@ msgid "" "cycle (~1 s). A ``cache_salt`` with no registered quota has an effective " "limit of ``0`` bytes, so its data is cleared next cycle (allowlist " "semantics)." -msgstr "" +msgstr "这些端点管理由 ``IsolatedLRU`` 逐出策略（通过 ``--eviction-policy IsolatedLRU`` 选择）消耗的每个 ``cache_salt`` 存储预算。配额是 **软性** 的：设置限制并不会拒绝写入 — 任何超出预算的 ``cache_salt`` 会在下一个逐出周期（约 1 秒）被逐出。没有注册配额的 ``cache_salt`` 有一个有效限制为 ``0`` 字节，因此其数据将在下一个周期被清除（白名单语义）。" #: ../../source/mp/http_api.rst:315 msgid "" "These endpoints are no-ops on engines that did not start with " "``--eviction-policy IsolatedLRU``: the ``QuotaManager`` is still present," " but the LRU policy ignores the registered quotas." -msgstr "" +msgstr "这些端点在未使用 ``--eviction-policy IsolatedLRU`` 启动的引擎上是无操作的：``QuotaManager`` 仍然存在，但 LRU 策略会忽略注册的配额。" #: ../../source/mp/http_api.rst:319 msgid "" @@ -424,37 +424,37 @@ msgid "" "data with ``cache_salt=\"_default\"`` cannot be managed via this HTTP API" " distinctly from anonymous traffic — both map to the same path parameter;" " pick any other value (e.g. ``\"default\"``) to disambiguate." -msgstr "" +msgstr "**空盐的 URL 转义。** ``cache_salt=\\\"\\\"``（无盐 / 匿名流量）不能出现在 URL 路径参数中，因此 API 接受哨兵 ``_default`` 作为替代。``PUT /quota/_default`` 设置 ``cache_salt=\\\"\\\"`` 的配额。合法存储数据的用户使用 ``cache_salt=\\\"_default\\\"``，无法通过此 HTTP API 与匿名流量区分管理——两者映射到相同的路径参数；选择任何其他值（例如 ``\\\"default\\\"``）以消除歧义。" #: ../../source/mp/http_api.rst:328 #, python-brace-format msgid "``PUT /quota/{cache_salt}``" -msgstr "" +msgstr "``PUT /quota/{cache_salt}``" #: ../../source/mp/http_api.rst:330 msgid "Create or update a quota." -msgstr "" +msgstr "创建或更新配额。" #: ../../source/mp/http_api.rst:332 #, python-brace-format msgid "**Body:** ``{\"limit_gb\": }`` (required, finite, non-negative)." -msgstr "" +msgstr "**主体:** ``{\\\"limit_gb\\\": }`` （必需，有限，非负）。" #: ../../source/mp/http_api.rst:340 msgid "" "**Errors:** ``400`` for malformed JSON, missing ``limit_gb``, non-numeric" " ``limit_gb``, ``nan`` / ``inf``, or negative values; ``503`` if the " "engine is not initialized." -msgstr "" +msgstr "**错误:** ``400`` 表示 JSON 格式错误、缺少 ``limit_gb``、``limit_gb`` 不是数字、``nan`` / ``inf`` 或负值；``503`` 表示引擎未初始化。" #: ../../source/mp/http_api.rst:353 #, python-brace-format msgid "``GET /quota/{cache_salt}``" -msgstr "" +msgstr "``GET /quota/{cache_salt}``" #: ../../source/mp/http_api.rst:355 msgid "Read the current quota and live usage for one ``cache_salt``." -msgstr "" +msgstr "读取当前配额和一个 ``cache_salt`` 的实时使用情况。" #: ../../source/mp/http_api.rst:368 msgid "" @@ -462,19 +462,19 @@ msgid "" "``cache_salt`` (``limit_gb`` is then ``0.0`` and ``current_usage_gb`` " "reflects whatever bytes are currently cached for that salt — those bytes " "will evict next cycle under ``IsolatedLRU``)." -msgstr "" +msgstr "``exists`` 为 ``false`` 当该 ``cache_salt`` 从未注册过配额时（``limit_gb`` 此时为 ``0.0``，而 ``current_usage_gb`` 反映当前为该盐缓存的字节数——这些字节将在下一个周期根据 ``IsolatedLRU`` 被逐出）。" #: ../../source/mp/http_api.rst:374 #, python-brace-format msgid "``DELETE /quota/{cache_salt}``" -msgstr "" +msgstr "``DELETE /quota/{cache_salt}``" #: ../../source/mp/http_api.rst:376 msgid "" "Remove a ``cache_salt``'s quota entry. Any bytes still cached under this " "``cache_salt`` become over-budget on the next eviction cycle (effective " "limit drops to ``0``) and will be evicted." -msgstr "" +msgstr "删除 ``cache_salt`` 的配额条目。任何仍然缓存于此 ``cache_salt`` 下的字节将在下一个逐出周期中超出预算（有效限制降至 ``0``），并将被逐出。" #: ../../source/mp/http_api.rst:386 #, python-brace-format @@ -482,19 +482,19 @@ msgid "" "When no quota was registered for the given ``cache_salt``, the response " "is ``{\"cache_salt\": \"...\", \"status\": \"not_found\"}`` (still ``200 " "OK``)." -msgstr "" +msgstr "当给定的 ``cache_salt`` 没有注册配额时，响应为 ``{\\\"cache_salt\\\": \\\"...\\\", \\\"status\\\": \\\"not_found\\\"}``（仍然是 ``200 OK``）。" #: ../../source/mp/http_api.rst:390 msgid "``GET /quota``" -msgstr "" +msgstr "``GET /quota``" #: ../../source/mp/http_api.rst:392 msgid "List every registered quota alongside its live usage." -msgstr "" +msgstr "列出每个注册的配额及其实时使用情况。" #: ../../source/mp/http_api.rst:406 msgid "``GET /conf``" -msgstr "" +msgstr "``GET /conf``" #: ../../source/mp/http_api.rst:408 msgid "" @@ -504,254 +504,254 @@ msgid "" "serialized via ``safe_asdict``; other values go through " "``make_json_safe``. Useful for confirming what the process actually " "loaded — including environment overrides — without restarting." -msgstr "" +msgstr "返回在 ``app.state.configs`` 上注册的每个服务器端配置对象（通常是 ``mp``、``storage_manager`` 和 ``observability``），作为一个单一的缩进 JSON 文档。数据类通过 ``safe_asdict`` 序列化；其他值通过 ``make_json_safe`` 处理。对于确认进程实际加载的内容（包括环境覆盖）非常有用，而无需重启。" #: ../../source/mp/http_api.rst:433 msgid "" "**Response** (``503 Service Unavailable``) when configs are not wired " "onto ``app.state`` yet:" -msgstr "" +msgstr "**响应**（``503 服务不可用``），当配置尚未连接到 ``app.state`` 时：" #: ../../source/mp/http_api.rst:449 msgid "``GET /version``" -msgstr "" +msgstr "``GET /version``" #: ../../source/mp/http_api.rst:451 msgid "" "Returns the full version descriptor (package version combined with the " "current commit id), formatted by ``lmcache.utils.get_version()``." -msgstr "" +msgstr "返回完整的版本描述符（软件包版本与当前提交 ID 的组合），格式由 ``lmcache.utils.get_version()`` 生成。" #: ../../source/mp/http_api.rst:467 msgid "``GET /lmc_version``" -msgstr "" +msgstr "``GET /lmc_version``" #: ../../source/mp/http_api.rst:469 msgid "" "Returns the raw LMCache package version string " "(``lmcache.utils.VERSION``)." -msgstr "" +msgstr "返回原始的 LMCache 包版本字符串 (``lmcache.utils.VERSION``)。" #: ../../source/mp/http_api.rst:478 msgid "``GET /commit_id``" -msgstr "" +msgstr "``GET /commit_id``" #: ../../source/mp/http_api.rst:480 msgid "" "Returns the git commit id baked into the build " "(``lmcache.utils.COMMIT_ID``)." -msgstr "" +msgstr "返回构建中嵌入的 git 提交 id (``lmcache.utils.COMMIT_ID``)。" #: ../../source/mp/http_api.rst:489 msgid "``GET /env``" -msgstr "" +msgstr "``GET /env``" #: ../../source/mp/http_api.rst:491 msgid "" "Dumps the process environment variables as a sorted, pretty-printed JSON " "document. Response ``Content-Type`` is ``text/plain`` so it can be piped " "directly to a terminal." -msgstr "" +msgstr "将进程环境变量转储为排序的、格式良好的 JSON 文档。响应的 ``Content-Type`` 为 ``text/plain``，因此可以直接通过管道传输到终端。" #: ../../source/mp/http_api.rst:497 msgid "" "The payload may contain secrets injected via environment variables. " "Restrict network access to this endpoint in production." -msgstr "" +msgstr "有效负载可能包含通过环境变量注入的机密。在生产环境中限制对该端点的网络访问。" #: ../../source/mp/http_api.rst:507 msgid "``GET /loglevel``" -msgstr "" +msgstr "``GET /loglevel``" #: ../../source/mp/http_api.rst:509 msgid "" "Inspect or mutate Python logger levels at runtime. All responses are " "``text/plain``. The endpoint has three modes driven by query parameters:" -msgstr "" +msgstr "在运行时检查或修改 Python 日志记录器级别。所有响应都是 ``text/plain``。该端点有三种模式，由查询参数驱动：" #: ../../source/mp/http_api.rst:516 ../../source/mp/http_api.rst:583 #: ../../source/mp/http_api.rst:608 msgid "Query" -msgstr "" +msgstr "查询" #: ../../source/mp/http_api.rst:517 ../../source/mp/http_api.rst:584 #: ../../source/mp/http_api.rst:609 msgid "Behavior" -msgstr "" +msgstr "行为" #: ../../source/mp/http_api.rst:518 msgid "(no params)" -msgstr "" +msgstr "(无参数)" #: ../../source/mp/http_api.rst:519 msgid "List every logger registered with :mod:`logging` and its level." -msgstr "" +msgstr "列出所有在 :mod:`logging` 中注册的记录器及其级别。" #: ../../source/mp/http_api.rst:520 msgid "``?logger_name=``" -msgstr "" +msgstr "``?logger_name=``" #: ../../source/mp/http_api.rst:521 msgid "Return the effective level of the named logger." -msgstr "" +msgstr "返回指定记录器的有效级别。" #: ../../source/mp/http_api.rst:522 msgid "``?logger_name=&level=``" -msgstr "" +msgstr "``?logger_name=&level=``" #: ../../source/mp/http_api.rst:523 msgid "" "Set the named logger (and its handlers) to ``LEVEL`` " "(``DEBUG``/``INFO``/``WARNING``/``ERROR``/``CRITICAL``). Returns ``400`` " "on an unknown level." -msgstr "" +msgstr "将命名的日志记录器（及其处理程序）设置为 ``LEVEL``（``DEBUG``/``INFO``/``WARNING``/``ERROR``/``CRITICAL``）。在未知级别时返回 ``400``。" #: ../../source/mp/http_api.rst:527 msgid "**Examples:**" -msgstr "" +msgstr "**示例：**" #: ../../source/mp/http_api.rst:541 msgid "``GET /metrics``" -msgstr "" +msgstr "``GET /metrics``" #: ../../source/mp/http_api.rst:543 msgid "" "Prometheus exposition format for every metric registered on the default " "``prometheus_client`` registry. Scrape this directly from Prometheus. See" " :doc:`observability` for the list of exported metrics." -msgstr "" +msgstr "默认 ``prometheus_client`` 注册表中注册的每个指标的 Prometheus 展示格式。直接从 Prometheus 抓取此内容。有关导出指标的列表，请参见 :doc:`observability`。" #: ../../source/mp/http_api.rst:554 msgid "``POST /metrics/reset``" -msgstr "" +msgstr "``POST /metrics/reset``" #: ../../source/mp/http_api.rst:556 msgid "" "Resets all LMCache observability metrics to their initial state " "(``reset_observability_metrics``). Intended for test harnesses and " "benchmarks — not for production." -msgstr "" +msgstr "重置所有 LMCache 可观察性指标到其初始状态（``reset_observability_metrics``）。旨在用于测试工具和基准测试——不适用于生产环境。" #: ../../source/mp/http_api.rst:573 msgid "``GET /threads``" -msgstr "" +msgstr "``GET /threads``" #: ../../source/mp/http_api.rst:575 msgid "" "Enumerate active Python threads in the server process along with their " "stack traces, plus a total-count summary. Useful for live debugging of " "hangs or runaway workers." -msgstr "" +msgstr "列出服务器进程中活动的 Python 线程及其堆栈跟踪，并提供总计摘要。这对于实时调试挂起或失控的工作线程非常有用。" #: ../../source/mp/http_api.rst:585 msgid "``?name=``" -msgstr "" +msgstr "``?name=``" #: ../../source/mp/http_api.rst:586 msgid "Keep only threads whose name contains ```` (case-insensitive)." -msgstr "" +msgstr "仅保留名称包含 ```` 的线程（不区分大小写）。" #: ../../source/mp/http_api.rst:588 msgid "``?thread_id=``" -msgstr "" +msgstr "``?thread_id=``" #: ../../source/mp/http_api.rst:589 msgid "Keep only the thread with the matching ``ident``." -msgstr "" +msgstr "只保留与匹配的 ``ident`` 相关的线程。" #: ../../source/mp/http_api.rst:598 msgid "``GET /periodic-threads``" -msgstr "" +msgstr "``GET /periodic-threads``" #: ../../source/mp/http_api.rst:600 msgid "" "Returns a JSON snapshot of the " ":class:`~lmcache.v1.periodic_thread.PeriodicThreadRegistry`: counts by " "level plus per-thread status (last run timestamp, latest summary, etc.)." -msgstr "" +msgstr "返回 :class:`~lmcache.v1.periodic_thread.PeriodicThreadRegistry` 的 JSON 快照：按级别统计以及每个线程的状态（上次运行时间戳、最新摘要等）。" #: ../../source/mp/http_api.rst:610 msgid "``?level=critical|high|medium|low``" -msgstr "" +msgstr "``?level=critical|high|medium|low``" #: ../../source/mp/http_api.rst:611 msgid "Only include threads at the given level. ``400`` on unknown." -msgstr "" +msgstr "仅包含给定级别的线程。对未知情况返回 ``400``。" #: ../../source/mp/http_api.rst:612 msgid "``?running_only=true``" -msgstr "" +msgstr "``?running_only=true``" #: ../../source/mp/http_api.rst:613 msgid "Only include threads currently running." -msgstr "" +msgstr "仅包含当前正在运行的线程。" #: ../../source/mp/http_api.rst:614 msgid "``?active_only=true``" -msgstr "" +msgstr "``?active_only=true``" #: ../../source/mp/http_api.rst:615 msgid "Only include threads considered active (recent tick)." -msgstr "" +msgstr "仅包括被视为活动的线程（最近的滴答）。" #: ../../source/mp/http_api.rst:640 #, python-brace-format msgid "``GET /periodic-threads/{thread_name}``" -msgstr "" +msgstr "``GET /periodic-threads/{thread_name}``" #: ../../source/mp/http_api.rst:642 msgid "Detailed status for a single periodic thread (``404`` if not found)." -msgstr "" +msgstr "单个周期性线程的详细状态（如果未找到则返回 ``404``）。" #: ../../source/mp/http_api.rst:651 msgid "``GET /periodic-threads-health``" -msgstr "" +msgstr "``GET /periodic-threads-health``" #: ../../source/mp/http_api.rst:653 msgid "" "Fast health check covering only ``critical`` and ``high`` level periodic " "threads. A thread is flagged unhealthy when it is marked running but has " "not ticked within its expected interval." -msgstr "" +msgstr "快速健康检查仅覆盖 ``critical`` 和 ``high`` 级别的周期性线程。当线程被标记为正在运行但在预期间隔内未进行滴答时，它会被标记为不健康。" #: ../../source/mp/http_api.rst:667 msgid "When something is lagging:" -msgstr "" +msgstr "当某些东西滞后时：" #: ../../source/mp/http_api.rst:691 msgid "Adding New Endpoints" -msgstr "" +msgstr "添加新端点" #: ../../source/mp/http_api.rst:693 msgid "" "Endpoints are auto-discovered from " "``lmcache/v1/multiprocess/http_apis/``. To add a new endpoint:" -msgstr "" +msgstr "端点会从 ``lmcache/v1/multiprocess/http_apis/`` 自动发现。要添加一个新端点：" #: ../../source/mp/http_api.rst:696 msgid "Create a new module in that directory named ``_api.py``." -msgstr "" +msgstr "在该目录中创建一个名为 ``_api.py`` 的新模块。" #: ../../source/mp/http_api.rst:697 msgid "Define a module-level ``router = APIRouter()``." -msgstr "" +msgstr "定义一个模块级的 ``router = APIRouter()``。" #: ../../source/mp/http_api.rst:698 msgid "Register handlers on ``router`` using FastAPI decorators." -msgstr "" +msgstr "使用 FastAPI 装饰器在 ``router`` 上注册处理程序。" #: ../../source/mp/http_api.rst:699 msgid "" "Access the engine via ``request.app.state.engine`` and guard for the " "``None`` case (engine not yet initialized)." -msgstr "" +msgstr "通过 ``request.app.state.engine`` 访问引擎，并检查 ``None`` 情况（引擎尚未初始化）。" #: ../../source/mp/http_api.rst:702 msgid "" "The :class:`~lmcache.v1.multiprocess.http_api_registry.HTTPAPIRegistry` " "will pick the module up automatically at startup — no central " "registration list to edit." -msgstr "" +msgstr ":class:`~lmcache.v1.multiprocess.http_api_registry.HTTPAPIRegistry` 将在启动时自动加载模块 — 无需编辑中央注册列表。" #: ../../source/mp/http_api.rst:706 msgid "" @@ -761,12 +761,12 @@ msgid "" "module name is listed in ``_MP_INCOMPATIBLE_MODULES`` there (used for " "modules that require vLLM-specific ``app.state`` attributes, e.g. " "``run_script_api``)." -msgstr "" +msgstr "如果路由足够通用，可以与嵌入 vLLM 的 API 服务器共享，请将其添加到 ``lmcache/v1/internal_api_server/common/`` 下。它将在 MP 端通过 ``common_api.py`` 被识别，除非其模块名称在 ``_MP_INCOMPATIBLE_MODULES`` 中列出（用于需要 vLLM 特定 ``app.state`` 属性的模块，例如 ``run_script_api``）。" #: ../../source/mp/http_api.rst:713 msgid "" "When adding a new endpoint, please also add a matching section to this " "page documenting the endpoint's purpose, request/response schema, and an " "example ``curl`` invocation." -msgstr "" +msgstr "添加新端点时，请在此页面上添加一个匹配的部分，记录端点的目的、请求/响应架构以及一个示例 ``curl`` 调用。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/index.po index c958ec75daa..31459db2e05 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/index.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/index.po @@ -21,11 +21,11 @@ msgstr "" #: ../../source/mp/index.rst:54 msgid "Contents" -msgstr "" +msgstr "目录" #: ../../source/mp/index.rst:2 msgid "Multiprocess Mode" -msgstr "" +msgstr "多进程模式" #: ../../source/mp/index.rst:4 msgid "" @@ -33,18 +33,18 @@ msgid "" "that vLLM instances connect to over ZMQ. One LMCache server per node can" " serve multiple vLLM pods, providing process isolation, shared caching, " "and independent resource scaling." -msgstr "" +msgstr "LMCache 多进程 (MP) 模式将 LMCache 作为一个 **独立服务** 运行，vLLM 实例通过 ZMQ 连接到该服务。每个节点的 LMCache 服务器可以为多个 vLLM pod 提供服务，提供进程隔离、共享缓存和独立资源扩展。" #: ../../source/mp/index.rst:10 msgid "Key Benefits" -msgstr "" +msgstr "关键好处" #: ../../source/mp/index.rst:12 msgid "" "**Process isolation** -- LMCache and vLLM run in separate processes (or " "containers), so a cache-related issue does not crash the inference " "engine." -msgstr "" +msgstr "**进程隔离** -- LMCache 和 vLLM 在独立的进程（或容器）中运行，因此与缓存相关的问题不会导致推理引擎崩溃。" #: ../../source/mp/index.rst:14 msgid "" @@ -52,63 +52,63 @@ msgid "" "running LMCache in a separate process, its Python GIL and CPU work " "(hashing, memory management, L2 I/O) do not compete with vLLM's inference" " threads." -msgstr "" +msgstr "**推理路径上没有 GIL 争用或 Python 开销** -- 通过在单独的进程中运行 LMCache，它的 Python GIL 和 CPU 工作（哈希、内存管理、L2 I/O）不会与 vLLM 的推理线程竞争。" #: ../../source/mp/index.rst:17 msgid "" "**Shared caching across pods** -- Multiple vLLM instances on the same " "node share a single L1 cache, maximizing KV reuse." -msgstr "" +msgstr "**跨 Pod 共享缓存** -- 同一节点上的多个 vLLM 实例共享一个 L1 缓存，最大化 KV 重用。" #: ../../source/mp/index.rst:19 msgid "" "**Independent resource scaling** -- Allocate CPU memory for caching " "independently of GPU memory for inference." -msgstr "" +msgstr "**独立资源扩展** -- 为缓存分配 CPU 内存，与用于推理的 GPU 显存独立。" #: ../../source/mp/index.rst:21 msgid "" "**Multi-tier storage (L1 + L2)** -- In-memory L1 cache backed by " "persistent L2 storage via NIXL (GDS, POSIX, HF3FS, and more)." -msgstr "" +msgstr "**多层存储 (L1 + L2)** -- 由持久化 L2 存储通过 NIXL (GDS、POSIX、HF3FS 等) 支持的内存 L1 缓存。" #: ../../source/mp/index.rst:23 msgid "" "**Built-in observability** -- Prometheus metrics and a telemetry event " "system out of the box." -msgstr "" +msgstr "**内置可观察性** -- 开箱即用的 Prometheus 指标和遥测事件系统。" #: ../../source/mp/index.rst:27 msgid "Prerequisites" -msgstr "" +msgstr "先决条件" #: ../../source/mp/index.rst:29 msgid "**vLLM** latest version is recommended for best compatibility" -msgstr "" +msgstr "**建议使用最新版本的 vLLM 以获得最佳兼容性**" #: ../../source/mp/index.rst:30 msgid "**LMCache** latest dev branch" -msgstr "" +msgstr "**LMCache** 最新开发分支" #: ../../source/mp/index.rst:33 msgid "Server Variants" -msgstr "" +msgstr "服务器变体" #: ../../source/mp/index.rst:35 msgid "LMCache ships three server entry points:" -msgstr "" +msgstr "LMCache 提供三个服务器入口点：" #: ../../source/mp/index.rst:41 msgid "Entry Point" -msgstr "" +msgstr "入口点" #: ../../source/mp/index.rst:42 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/mp/index.rst:43 msgid "``lmcache server``" -msgstr "" +msgstr "``lmcache server``" #: ../../source/mp/index.rst:44 msgid "" @@ -116,25 +116,25 @@ msgid "" "K8s probes, ``/clear-cache``, ``/status`` — see :doc:`http_api`). Use " "``--engine-type blend`` to enable BlendEngineV2 for cross-request KV " "reuse." -msgstr "" +msgstr "**推荐。** ZMQ + FastAPI HTTP 前端（为 K8s 探针添加 ``/healthcheck``、``/clear-cache``、``/status`` — 参见 :doc:`http_api`）。使用 ``--engine-type blend`` 启用 BlendEngineV2 以实现跨请求的 KV 重用。" #: ../../source/mp/index.rst:48 msgid "``python3 -m lmcache.v1.multiprocess.server``" -msgstr "" +msgstr "``python3 -m lmcache.v1.multiprocess.server``" #: ../../source/mp/index.rst:49 msgid "" "(Legacy) ZMQ-only server using MPCacheEngine (no HTTP endpoints). Prefer " "``lmcache server``." -msgstr "" +msgstr "(遗留) 仅使用 MPCacheEngine 的 ZMQ 服务器（没有 HTTP 端点）。请使用 ``lmcache server``。" #: ../../source/mp/index.rst:51 msgid "``python3 -m lmcache.v1.multiprocess.blend_server_v2``" -msgstr "" +msgstr "``python3 -m lmcache.v1.multiprocess.blend_server_v2``" #: ../../source/mp/index.rst:52 msgid "" "(Legacy) CacheBlend-enabled server. Prefer ``lmcache server --engine-type" " blend``." -msgstr "" +msgstr "（遗留）启用 CacheBlend 的服务器。建议使用 ``lmcache server --engine-type blend``。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage.po index e920fa6fd86..785733e1274 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage.po @@ -21,17 +21,17 @@ msgstr "" #: ../../source/mp/l2_storage.rst:2 msgid "L2 Storage (Persistent Cache)" -msgstr "" +msgstr "L2 存储（持久缓存）" #: ../../source/mp/l2_storage.rst:4 msgid "LMCache multiprocess mode supports a two-tier storage architecture:" -msgstr "" +msgstr "LMCache 多进程模式支持两级存储架构：" #: ../../source/mp/l2_storage.rst:6 msgid "" "**L1 (in-memory)** -- Fast CPU memory managed by the L1 Manager. All KV " "cache chunks live here during active use." -msgstr "" +msgstr "**L1 (内存)** -- 由 L1 管理器管理的快速 CPU 内存。所有 KV Cache 块在活跃使用期间都存储在这里。" #: ../../source/mp/l2_storage.rst:8 msgid "" @@ -39,251 +39,251 @@ msgid "" "file-system/raw-block). The StoreController asynchronously pushes data " "from L1 to L2, and the PrefetchController loads data from L2 back into L1" " on cache misses." -msgstr "" +msgstr "**L2 (持久性)** -- 耐用的存储后端（基于 NIXL 或普通文件系统/原始块）。StoreController 异步将数据从 L1 推送到 L2，而 PrefetchController 在缓存未命中时将数据从 L2 加载回 L1。" #: ../../source/mp/l2_storage.rst:18 msgid "Data Flow" -msgstr "" +msgstr "数据流" #: ../../source/mp/l2_storage.rst:20 msgid "**Write path (L1 -> L2):**" -msgstr "" +msgstr "**写入路径 (L1 -> L2):**" #: ../../source/mp/l2_storage.rst:22 msgid "vLLM stores KV cache chunks into L1 via the ``STORE`` RPC." -msgstr "" +msgstr "vLLM 通过 ``STORE`` RPC 将 KV Cache 块存储到 L1。" #: ../../source/mp/l2_storage.rst:23 msgid "" "The ``StoreController`` detects new objects (via eventfd) and " "asynchronously submits store tasks to each configured L2 adapter." -msgstr "" +msgstr "``StoreController`` 检测到新对象（通过 eventfd），并异步提交存储任务到每个配置的 L2 适配器。" #: ../../source/mp/l2_storage.rst:25 msgid "The L2 adapter writes the data to its backend (e.g., local SSD via GDS)." -msgstr "" +msgstr "L2 适配器将数据写入其后端（例如，通过 GDS 的本地 SSD）。" #: ../../source/mp/l2_storage.rst:27 msgid "**Read path (L2 -> L1):**" -msgstr "" +msgstr "**读取路径 (L2 -> L1):**" #: ../../source/mp/l2_storage.rst:29 msgid "A ``LOOKUP`` RPC checks L1 for prefix hits." -msgstr "" +msgstr "一个 ``LOOKUP`` RPC 检查 L1 中的前缀命中。" #: ../../source/mp/l2_storage.rst:30 msgid "" "For keys not found in L1, the ``PrefetchController`` submits lookup " "requests to L2 adapters." -msgstr "" +msgstr "对于在 L1 中未找到的键，``PrefetchController`` 向 L2 适配器提交查找请求。" #: ../../source/mp/l2_storage.rst:32 msgid "" "If found in L2, the data is loaded back into L1 and read-locked for the " "pending ``RETRIEVE`` RPC." -msgstr "" +msgstr "如果在 L2 中找到，数据将被加载回 L1，并为待处理的 ``RETRIEVE`` RPC 进行读锁定。" #: ../../source/mp/l2_storage.rst:36 msgid "Adapter Types" -msgstr "" +msgstr "适配器类型" #: ../../source/mp/l2_storage.rst:39 msgid "``nixl_store`` -- NIXL-based persistent storage" -msgstr "" +msgstr "``nixl_store`` -- 基于 NIXL 的持久存储" #: ../../source/mp/l2_storage.rst:41 msgid "" "The primary production adapter. Uses NIXL (NVIDIA Interconnect Library) " "for high-performance storage I/O." -msgstr "" +msgstr "主要的生产适配器。使用 NIXL（NVIDIA 互连库）进行高性能存储 I/O。" #: ../../source/mp/l2_storage.rst:44 ../../source/mp/l2_storage.rst:118 #: ../../source/mp/l2_storage.rst:170 ../../source/mp/l2_storage.rst:207 #: ../../source/mp/l2_storage.rst:260 ../../source/mp/l2_storage.rst:343 #: ../../source/mp/l2_storage.rst:512 msgid "**Required fields:**" -msgstr "" +msgstr "**必填字段：**" #: ../../source/mp/l2_storage.rst:46 msgid "" "``backend``: Storage backend -- one of ``POSIX``, ``GDS``, ``GDS_MT``, " "``HF3FS``, ``OBJ``, ``AZURE_BLOB``." -msgstr "" +msgstr "``backend``: 存储后端 -- 其中之一为 ``POSIX``, ``GDS``, ``GDS_MT``, ``HF3FS``, ``OBJ``, ``AZURE_BLOB``。" #: ../../source/mp/l2_storage.rst:48 msgid "" "``pool_size``: Number of storage descriptors to pre-allocate (must be > " "0)." -msgstr "" +msgstr "``pool_size``: 预分配的存储描述符数量（必须大于 0）。" #: ../../source/mp/l2_storage.rst:50 ../../source/mp/l2_storage.rst:123 msgid "**Backend-specific parameters (``backend_params``):**" -msgstr "" +msgstr "**后端特定参数 (``backend_params``):**" #: ../../source/mp/l2_storage.rst:52 msgid "File-based backends (``GDS``, ``GDS_MT``, ``POSIX``, ``HF3FS``) require:" -msgstr "" +msgstr "基于文件的后端（``GDS``, ``GDS_MT``, ``POSIX``, ``HF3FS``）需要：" #: ../../source/mp/l2_storage.rst:54 msgid "``file_path``: Directory path for storing L2 data." -msgstr "" +msgstr "``file_path``: 存储 L2 数据的目录路径。" #: ../../source/mp/l2_storage.rst:55 msgid "" "``use_direct_io``: ``\"true\"`` or ``\"false\"`` -- whether to use direct" " I/O." -msgstr "" +msgstr "``use_direct_io``: ``\"true\"`` 或 ``\"false\"`` -- 是否使用直接 I/O。" #: ../../source/mp/l2_storage.rst:57 msgid "" "The ``OBJ`` and ``AZURE_BLOB`` backends (object stores) do not require " "``file_path``." -msgstr "" +msgstr "``OBJ`` 和 ``AZURE_BLOB`` 后端（对象存储）不需要 ``file_path``。" #: ../../source/mp/l2_storage.rst:59 msgid "**Backend descriptions:**" -msgstr "" +msgstr "**后端描述：**" #: ../../source/mp/l2_storage.rst:65 msgid "Backend" -msgstr "" +msgstr "后端" #: ../../source/mp/l2_storage.rst:66 ../../source/mp/l2_storage.rst:632 #: ../../source/mp/l2_storage.rst:686 ../../source/mp/l2_storage.rst:726 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/mp/l2_storage.rst:67 msgid "``POSIX``" -msgstr "" +msgstr "``POSIX``" #: ../../source/mp/l2_storage.rst:68 msgid "Standard POSIX file I/O. Works on any file system. No direct I/O." -msgstr "" +msgstr "标准的 POSIX 文件 I/O。适用于任何文件系统。无直接 I/O。" #: ../../source/mp/l2_storage.rst:69 msgid "``GDS``" -msgstr "" +msgstr "``GDS``" #: ../../source/mp/l2_storage.rst:70 msgid "" "NVIDIA GPU Direct Storage. Enables direct GPU-to-storage transfers " "bypassing the CPU. Requires NVMe SSDs with GDS support." -msgstr "" +msgstr "NVIDIA GPU 直接存储。支持直接的 GPU 到存储的传输，绕过 CPU。需要支持 GDS 的 NVMe SSD。" #: ../../source/mp/l2_storage.rst:72 msgid "``GDS_MT``" -msgstr "" +msgstr "``GDS_MT``" #: ../../source/mp/l2_storage.rst:73 msgid "Multi-threaded variant of GDS for higher throughput." -msgstr "" +msgstr "GDS 的多线程变体，以提高吞吐量。" #: ../../source/mp/l2_storage.rst:74 msgid "``HF3FS``" -msgstr "" +msgstr "``HF3FS``" #: ../../source/mp/l2_storage.rst:75 msgid "Shared file system backend (e.g., for distributed/networked storage)." -msgstr "" +msgstr "共享文件系统后端（例如，用于分布式/网络存储）。" #: ../../source/mp/l2_storage.rst:76 msgid "``OBJ``" -msgstr "" +msgstr "``OBJ``" #: ../../source/mp/l2_storage.rst:77 msgid "Object store backend. No local file path required." -msgstr "" +msgstr "对象存储后端。无需本地文件路径。" #: ../../source/mp/l2_storage.rst:78 msgid "``AZURE_BLOB``" -msgstr "" +msgstr "``AZURE_BLOB``" #: ../../source/mp/l2_storage.rst:79 msgid "Object store backend for Azure Blob Storage. No local file path required." -msgstr "" +msgstr "Azure Blob Storage 的对象存储后端。无需本地文件路径。" #: ../../source/mp/l2_storage.rst:81 ../../source/mp/l2_storage.rst:140 #: ../../source/mp/l2_storage.rst:183 ../../source/mp/l2_storage.rst:308 #: ../../source/mp/l2_storage.rst:383 ../../source/mp/l2_storage.rst:533 msgid "**Configuration examples:**" -msgstr "" +msgstr "**配置示例：**" #: ../../source/mp/l2_storage.rst:104 msgid "``nixl_store_dynamic`` -- NIXL-based dynamic storage with persist/recover" -msgstr "" +msgstr "``nixl_store_dynamic`` -- 基于 NIXL 的动态存储，具有持久化/恢复功能" #: ../../source/mp/l2_storage.rst:106 msgid "" "A dynamic variant of the NIXL adapter that opens and registers files per-" "operation instead of pre-allocating them at init. This enables:" -msgstr "" +msgstr "NIXL 适配器的动态变体，根据操作打开和注册文件，而不是在初始化时预分配。这使得：" #: ../../source/mp/l2_storage.rst:109 msgid "**Persist/recover** -- cached KV metadata survives restarts." -msgstr "" +msgstr "**持久化/恢复** -- 缓存的 KV 元数据在重启后依然存在。" #: ../../source/mp/l2_storage.rst:110 msgid "" "**No fd limits** -- files are opened and closed per transfer, so the " "cache can grow beyond OS open-file-descriptor limits." -msgstr "" +msgstr "**无文件描述符限制** -- 文件在每次传输时打开和关闭，因此缓存可以超出操作系统的打开文件描述符限制。" #: ../../source/mp/l2_storage.rst:115 msgid "" "Only file-based backends are supported (``POSIX``, ``GDS``, ``GDS_MT``, " "``HF3FS``). The ``OBJ`` and ``AZURE_BLOB`` backends are not supported " "yet." -msgstr "" +msgstr "仅支持基于文件的后端（``POSIX``, ``GDS``, ``GDS_MT``, ``HF3FS``）。``OBJ`` 和 ``AZURE_BLOB`` 后端尚不支持。" #: ../../source/mp/l2_storage.rst:120 msgid "" "``backend``: Storage backend -- one of ``POSIX``, ``GDS``, ``GDS_MT``, " "``HF3FS``." -msgstr "" +msgstr "``backend``: 存储后端 -- 其中之一为 ``POSIX``, ``GDS``, ``GDS_MT``, ``HF3FS``。" #: ../../source/mp/l2_storage.rst:125 msgid "``file_path``: Directory path for storing L2 data files." -msgstr "" +msgstr "``file_path``: 存储 L2 数据文件的目录路径。" #: ../../source/mp/l2_storage.rst:126 msgid "``use_direct_io``: ``\"true\"`` or ``\"false\"``." -msgstr "" +msgstr "``use_direct_io``: ``\"true\"`` 或 ``\"false\"``。" #: ../../source/mp/l2_storage.rst:127 msgid "" "``max_capacity_gb``: Maximum storage capacity in GB. The adapter rejects " "stores when this limit is reached. Required for the eviction controller " "to compute usage." -msgstr "" +msgstr "``max_capacity_gb``: 最大存储容量（以 GB 为单位）。当达到此限制时，适配器会拒绝存储。此项对于逐出控制器计算使用情况是必需的。" #: ../../source/mp/l2_storage.rst:131 msgid "**Optional fields (for persist):**" -msgstr "" +msgstr "**可选字段（用于持久化）：**" #: ../../source/mp/l2_storage.rst:133 msgid "" "``persist_enabled`` (bool, default ``true``): If ``true``, data files are" " kept on disk at shutdown. If ``false``, all data files are deleted on " "shutdown." -msgstr "" +msgstr "``persist_enabled`` (bool, 默认 ``true``): 如果 ``true``，数据文件将在关闭时保留在磁盘上。如果 ``false``，所有数据文件将在关闭时被删除。" #: ../../source/mp/l2_storage.rst:137 msgid "" "Lookup always checks secondary storage (disk) on miss and lazily " "populates the in-memory index when a file is found." -msgstr "" +msgstr "查找总是在未命中时检查二级存储（磁盘），并在找到文件时懒惰地填充内存中的索引。" #: ../../source/mp/l2_storage.rst:153 msgid "**Persist / secondary lookup behaviour:**" -msgstr "" +msgstr "**持久化 / 次级查找行为:**" #: ../../source/mp/l2_storage.rst:155 msgid "" "On **shutdown**, the adapter keeps data files on disk by default " "(``persist_enabled`` defaults to ``true``). If explicitly set to " "``false``, all data files are deleted to avoid orphaned storage." -msgstr "" +msgstr "在 **关闭** 时，适配器默认将数据文件保留在磁盘上（``persist_enabled`` 默认为 ``true``）。如果明确设置为 ``false``，则所有数据文件将被删除，以避免孤立存储。" #: ../../source/mp/l2_storage.rst:158 msgid "" @@ -291,11 +291,11 @@ msgid "" "through to a secondary lookup on disk: if the deterministic file exists, " "it is treated as a hit and the in-memory index is populated lazily from " "the file size." -msgstr "" +msgstr "在 **启动** 时，内存中的索引是空的。每次查找未命中都会转到磁盘上的二次查找：如果确定性文件存在，则将其视为命中，并根据文件大小懒惰地填充内存中的索引。" #: ../../source/mp/l2_storage.rst:164 msgid "``fs`` -- File-system backed storage" -msgstr "" +msgstr "``fs`` -- 文件系统支持的存储" #: ../../source/mp/l2_storage.rst:166 msgid "" @@ -303,39 +303,39 @@ msgid "" "cache object is stored as a raw ``.data`` file whose name encodes the " "full ``ObjectKey``. Does **not** require NIXL -- works on any POSIX file" " system." -msgstr "" +msgstr "一个纯文件系统的 L2 适配器，使用异步 I/O (``aiofiles``)。每个 KV Cache 对象作为一个原始的 ``.data`` 文件存储，其名称编码了完整的 ``ObjectKey``。**不**需要 NIXL -- 可以在任何 POSIX 文件系统上工作。" #: ../../source/mp/l2_storage.rst:172 ../../source/mp/l2_storage.rst:262 msgid "``base_path``: Directory for storing KV cache files." -msgstr "" +msgstr "``base_path``: 存储 KV Cache 文件的目录。" #: ../../source/mp/l2_storage.rst:174 ../../source/mp/l2_storage.rst:215 #: ../../source/mp/l2_storage.rst:264 ../../source/mp/l2_storage.rst:348 #: ../../source/mp/l2_storage.rst:518 msgid "**Optional fields:**" -msgstr "" +msgstr "**可选字段：**" #: ../../source/mp/l2_storage.rst:176 msgid "" "``relative_tmp_dir``: Relative sub-directory for temporary files during " "writes (atomic rename on completion)." -msgstr "" +msgstr "``relative_tmp_dir``: 写入期间临时文件的相对子目录（完成时进行原子重命名）。" #: ../../source/mp/l2_storage.rst:178 msgid "" "``read_ahead_size``: Trigger file-system read-ahead by reading this many " "bytes first (positive integer, optional)." -msgstr "" +msgstr "``read_ahead_size``: 通过首先读取这么多字节来触发文件系统的预读（正整数，可选）。" #: ../../source/mp/l2_storage.rst:180 msgid "" "``use_odirect``: ``true`` or ``false`` (default ``false``) -- bypass the " "page cache via ``O_DIRECT``." -msgstr "" +msgstr "``use_odirect``: ``true`` 或 ``false``（默认 ``false``）-- 通过 ``O_DIRECT`` 绕过页面缓存。" #: ../../source/mp/l2_storage.rst:197 msgid "``dax`` -- Device-DAX fixed-slot storage" -msgstr "" +msgstr "``dax`` -- 设备-DAX 固定槽存储" #: ../../source/mp/l2_storage.rst:199 msgid "" @@ -343,7 +343,7 @@ msgid "" "``/dev/dax1.0``, and stores KV cache objects in fixed-size slots. This " "adapter is intended for byte-addressable memory devices such as " "persistent memory or CXL memory." -msgstr "" +msgstr "一个 L2 适配器，它映射单个 Device-DAX 路径，例如 ``/dev/dax1.0``，并在固定大小的槽中存储 KV Cache 对象。该适配器旨在用于字节可寻址内存设备，例如持久性内存或 CXL 内存。" #: ../../source/mp/l2_storage.rst:203 msgid "" @@ -351,78 +351,78 @@ msgid "" "index in server memory and rebuilds an empty index on restart. Old bytes" " may remain on the DAX device, but they are unreachable after the LMCache" " server restarts." -msgstr "" +msgstr "在此版本中，MP ``dax`` 适配器是易失性的。它将键索引保存在服务器内存中，并在重启时重建一个空索引。旧字节可能仍然保留在 DAX 设备上，但在 LMCache 服务器重启后无法访问。" #: ../../source/mp/l2_storage.rst:209 msgid "``device_path``: Path to the mmap-able DAX device or test file." -msgstr "" +msgstr "``device_path``: 可映射的 DAX 设备或测试文件的路径。" #: ../../source/mp/l2_storage.rst:210 msgid "``max_dax_size_gb``: Number of GiB to map from ``device_path``." -msgstr "" +msgstr "``max_dax_size_gb``: 从 ``device_path`` 映射的 GiB 数量。" #: ../../source/mp/l2_storage.rst:211 msgid "" "``slot_bytes``: Fixed slot size in bytes. This must be large enough for " "one full LMCache chunk because MP memory descriptors do not expose the " "non-MP full-chunk size." -msgstr "" +msgstr "``slot_bytes``: 固定槽大小（以字节为单位）。这必须足够大，以容纳一个完整的 LMCache 块，因为 MP 内存描述符不暴露非 MP 完整块的大小。" #: ../../source/mp/l2_storage.rst:217 msgid "``num_store_workers`` (int, default ``1``): Store worker threads." -msgstr "" +msgstr "``num_store_workers`` (int, default ``1``): 存储工作线程。" #: ../../source/mp/l2_storage.rst:218 msgid "``num_lookup_workers`` (int, default ``1``): Lookup worker threads." -msgstr "" +msgstr "``num_lookup_workers`` (int, default ``1``): 查找工作线程。" #: ../../source/mp/l2_storage.rst:219 msgid "" "``num_load_workers`` (int, default ``min(4, os.cpu_count())``): Load " "worker threads." -msgstr "" +msgstr "``num_load_workers`` (int, default ``min(4, os.cpu_count())``): 加载工作线程。" #: ../../source/mp/l2_storage.rst:221 msgid "" "``persist_enabled`` (bool): Accepted by common L2 config parsing but has " "no effect for ``dax`` because restart recovery is not implemented." -msgstr "" +msgstr "``persist_enabled`` (bool): 被常见的 L2 配置解析接受，但对 ``dax`` 没有影响，因为未实现重启恢复。" #: ../../source/mp/l2_storage.rst:224 ../../source/mp/l2_storage.rst:461 msgid "**Configuration example:**" -msgstr "" +msgstr "**配置示例:**" #: ../../source/mp/l2_storage.rst:243 msgid "**Current limits:**" -msgstr "" +msgstr "**当前限制：**" #: ../../source/mp/l2_storage.rst:245 msgid "" "Uses one server-owned mapped DAX path. Per-TP partitions and multi-device" " striping are not implemented." -msgstr "" +msgstr "使用一个服务器拥有的映射 DAX 路径。未实现每个 TP 的分区和多设备条带化。" #: ../../source/mp/l2_storage.rst:247 msgid "" "Only single-buffer objects are supported. Multi-tensor objects are " "rejected." -msgstr "" +msgstr "仅支持单缓冲区对象。多张量对象会被拒绝。" #: ../../source/mp/l2_storage.rst:248 msgid "" "Capacity is slot-based, not payload-byte-based. L2 eviction and usage " "metrics count occupied slots." -msgstr "" +msgstr "容量是基于插槽的，而不是基于有效载荷字节的。L2 逐出和使用指标计算占用的插槽。" #: ../../source/mp/l2_storage.rst:250 msgid "" "Lookups acquire DAX-side external locks. ``submit_unlock`` releases those" " locks after load/retrieve completes, making entries evictable again." -msgstr "" +msgstr "查找会获取 DAX 侧的外部锁。``submit_unlock`` 在加载/检索完成后释放这些锁，使条目可以再次被逐出。" #: ../../source/mp/l2_storage.rst:253 msgid "``fs_native`` -- Native C++ file-system connector" -msgstr "" +msgstr "``fs_native`` -- 原生 C++ 文件系统连接器" #: ../../source/mp/l2_storage.rst:255 msgid "" @@ -430,43 +430,43 @@ msgid "" "wrapped with ``NativeConnectorL2Adapter``. I/O is dispatched through a " "C++ worker-thread pool with eventfd-driven completions, giving a true I/O" " queue depth on a single Python thread." -msgstr "" +msgstr "一个由原生 C++ ``LMCacheFSClient`` 支持的文件系统 L2 适配器，封装在 ``NativeConnectorL2Adapter`` 中。I/O 通过一个 C++ 工作线程池进行调度，使用 eventfd 驱动的完成机制，在单个 Python 线程上提供真实的 I/O 队列深度。" #: ../../source/mp/l2_storage.rst:266 msgid "" "``num_workers`` (int, default ``4``, > 0): Number of C++ worker threads " "inside the connector. This is the real I/O queue depth -- raise to push " "throughput on filesystems whose aggregate BW exceeds per-stream BW." -msgstr "" +msgstr "``num_workers`` (int, default ``4``, > 0): 连接器内部的 C++ 工作线程数量。这是真正的 I/O 队列深度——增加此值以提高在总带宽超过每个流带宽的文件系统上的吞吐量。" #: ../../source/mp/l2_storage.rst:270 msgid "" "``relative_tmp_dir`` (str, default ``\"\"``): Relative sub-directory for " "temporary files during writes (atomic rename on completion)." -msgstr "" +msgstr "``relative_tmp_dir`` (str, default ``\"\"``): 写入期间临时文件的相对子目录（完成时原子重命名）。" #: ../../source/mp/l2_storage.rst:272 msgid "" "``use_odirect`` (bool, default ``false``): Bypass the page cache via " "``O_DIRECT``. Required to measure real disk bandwidth. See alignment " "caveat below." -msgstr "" +msgstr "``use_odirect`` (bool, default ``false``): 通过 ``O_DIRECT`` 绕过页面缓存。 需要测量真实的磁盘带宽。 请参见下面的对齐注意事项。" #: ../../source/mp/l2_storage.rst:275 msgid "" "``read_ahead_size`` (int, optional): Trigger filesystem readahead by " "issuing a warm-up read of this many bytes at open time." -msgstr "" +msgstr "``read_ahead_size`` (int, optional): 在打开时通过发出此字节数的预热读取来触发文件系统的预读取。" #: ../../source/mp/l2_storage.rst:277 msgid "" "``max_capacity_gb`` (float, default ``0``): Maximum L2 capacity in GB for" " client-side usage tracking. Default ``0`` disables tracking." -msgstr "" +msgstr "``max_capacity_gb`` (float, default ``0``): 客户端使用跟踪的最大 L2 容量（以 GB 为单位）。默认值 ``0`` 禁用跟踪。" #: ../../source/mp/l2_storage.rst:282 msgid "``O_DIRECT`` has two independent alignment requirements:" -msgstr "" +msgstr "``O_DIRECT`` 有两个独立的对齐要求：" #: ../../source/mp/l2_storage.rst:284 #, python-format @@ -481,7 +481,7 @@ msgid "" "that the resulting per-chunk byte size is a multiple of the FS block " "size. GPFS and similar parallel filesystems often use large blocks (e.g." " several MiB)." -msgstr "" +msgstr "**长度对齐。** 传输长度必须是文件系统块大小的倍数。连接器在构造时查询磁盘块大小，并在每次操作时检查 ``len % disk_block_size``。如果长度 **不是** 倍数，连接器会默默回退到缓冲打开（不使用 ``O_DIRECT``）进行该操作——正确性得以保留，但您无法获得真正的直接 I/O。为了确保实际使用 ``O_DIRECT``，选择 ``--chunk-size`` 使得每块的字节大小是文件系统块大小的倍数。GPFS 和类似的并行文件系统通常使用较大的块（例如几个 MiB）。" #: ../../source/mp/l2_storage.rst:296 msgid "" @@ -493,24 +493,24 @@ msgid "" "underlying ``read``/``write`` syscall returns ``EINVAL`` (this is **not**" " caught by the length-fallback path above and will surface as a runtime " "error)." -msgstr "" +msgstr "**内存缓冲区对齐。** I/O 缓冲区指针本身也必须对齐（通常在本地磁盘上对齐到 4096 字节，或在并行文件系统上对齐到 FS 块大小）。这由 ``--l1-align-bytes`` 控制（默认值为 ``4096``）——在使用块较大的文件系统时，将其提高以匹配 FS 块大小。如果缓冲区未对齐，底层的 ``read``/``write`` 系统调用将返回 ``EINVAL``（这不会被上面的长度回退路径捕获，并将作为运行时错误出现）。" #: ../../source/mp/l2_storage.rst:305 msgid "" "If unsure, start with ``use_odirect: false`` and confirm correctness " "before enabling ``O_DIRECT``." -msgstr "" +msgstr "如果不确定，请先使用 ``use_odirect: false`` 并确认正确性，然后再启用 ``O_DIRECT``。" #: ../../source/mp/l2_storage.rst:321 msgid "" "**Buffer-only mode example.** L1 acts as a pure write buffer that " "absorbs the peak burst of in-flight chunks while the C++ worker pool " "drains them to disk; nothing is retained in L1 once a store completes:" -msgstr "" +msgstr "**仅缓冲区模式示例。** L1 充当一个纯写缓冲区，吸收在飞行块的峰值突发，同时 C++ 工作池将它们写入磁盘；一旦存储完成，L1 中不会保留任何内容：" #: ../../source/mp/l2_storage.rst:336 msgid "``raw_block`` -- Raw block device backed persistent storage" -msgstr "" +msgstr "``raw_block`` -- 原始块设备支持的持久存储" #: ../../source/mp/l2_storage.rst:338 msgid "" @@ -518,126 +518,126 @@ msgid "" " block device or pre-sized file using the Rust raw-device I/O bindings. " "It reuses the existing raw-block metadata checkpoint model and writes " "directly into the caller-provided load buffers during prefetch." -msgstr "" +msgstr "一个内置的 L2 适配器，它使用 Rust 原始设备 I/O 绑定将 KV 对象存储在原始块设备或预先大小化的文件的固定大小槽中。它重用现有的原始块元数据检查点模型，并在预取期间直接写入调用者提供的加载缓冲区。" #: ../../source/mp/l2_storage.rst:345 msgid "``device_path``: Raw device path or pre-sized file path." -msgstr "" +msgstr "``device_path``: 原始设备路径或预先大小的文件路径。" #: ../../source/mp/l2_storage.rst:346 msgid "" "``slot_bytes``: Fixed slot size in bytes. Must be aligned to " "``block_align``." -msgstr "" +msgstr "``slot_bytes``: 固定的槽大小（以字节为单位）。必须与 ``block_align`` 对齐。" #: ../../source/mp/l2_storage.rst:350 msgid "" "``capacity_bytes``: Optional cap on the usable device bytes. Default " "``0`` means use the full device/file size." -msgstr "" +msgstr "``capacity_bytes``: 可选的可用设备字节上限。默认值 ``0`` 表示使用整个设备/文件大小。" #: ../../source/mp/l2_storage.rst:352 msgid "``use_odirect``: ``true`` or ``false`` (default ``true``)." -msgstr "" +msgstr "``use_odirect``: ``true`` 或 ``false``（默认 ``true``）。" #: ../../source/mp/l2_storage.rst:353 msgid "``block_align``: Device alignment in bytes (default ``4096``)." -msgstr "" +msgstr "``block_align``: 设备对齐字节数（默认 ``4096``）。" #: ../../source/mp/l2_storage.rst:354 msgid "``header_bytes``: Per-slot header reservation (default ``4096``)." -msgstr "" +msgstr "``header_bytes``: 每个槽位的头部保留字节数（默认 ``4096``）。" #: ../../source/mp/l2_storage.rst:355 msgid "" "``meta_total_bytes``: Reserved metadata checkpoint region (default " "``256MiB``)." -msgstr "" +msgstr "``meta_total_bytes``: 保留的元数据检查点区域（默认 ``256MiB``）。" #: ../../source/mp/l2_storage.rst:356 msgid "" "``meta_magic`` / ``meta_version``: Metadata checkpoint identity/version " "knobs." -msgstr "" +msgstr "``meta_magic`` / ``meta_version``: 元数据检查点标识/版本控制。" #: ../../source/mp/l2_storage.rst:357 msgid "" "``meta_checkpoint_interval_sec`` / ``meta_idle_quiet_ms`` / " "``meta_enable_periodic`` / ``meta_verify_on_load``: Checkpoint and " "recovery controls carried over from the legacy raw-block backend." -msgstr "" +msgstr "``meta_checkpoint_interval_sec`` / ``meta_idle_quiet_ms`` / ``meta_enable_periodic`` / ``meta_verify_on_load``: 从遗留的原始块后端继承的检查点和恢复控制。" #: ../../source/mp/l2_storage.rst:360 msgid "" "``load_checkpoint_on_init``: Load an existing on-device metadata " "checkpoint during startup (default ``true``). Set to ``false`` to start " "with an empty in-memory index instead." -msgstr "" +msgstr "``load_checkpoint_on_init``: 在启动时加载现有的设备元数据检查点（默认值为 ``true``）。设置为 ``false`` 以使用空的内存索引开始。" #: ../../source/mp/l2_storage.rst:363 msgid "``enable_zero_copy``: Try aligned direct-buffer I/O when possible." -msgstr "" +msgstr "``enable_zero_copy``: 尝试在可能的情况下使用对齐的直接缓冲区 I/O。" #: ../../source/mp/l2_storage.rst:364 msgid "" "``io_engine``: Rust raw-block I/O engine. Valid values are ``\"posix\"`` " "(default synchronous ``pread``/``pwrite`` path), ``\"io_uring\"`` (direct" " Rust io_uring syscall path)." -msgstr "" +msgstr "``io_engine``: Rust 原始块 I/O 引擎。有效值为 ``\"posix\"``（默认同步 ``pread``/``pwrite`` 路径），``\"io_uring\"``（直接 Rust io_uring 系统调用路径）。" #: ../../source/mp/l2_storage.rst:367 msgid "``iouring_queue_depth``: Queue depth for ``io_engine=\"io_uring\"``." -msgstr "" +msgstr "``iouring_queue_depth``: ``io_engine=\"io_uring\"`` 的队列深度。" #: ../../source/mp/l2_storage.rst:368 msgid "" "``num_store_workers`` / ``num_lookup_workers`` / ``num_load_workers``: " "Worker-thread counts for each operation type." -msgstr "" +msgstr "``num_store_workers`` / ``num_lookup_workers`` / ``num_load_workers``: 每种操作类型的工作线程数量。" #: ../../source/mp/l2_storage.rst:371 msgid "**Notes:**" -msgstr "" +msgstr "**注意：**" #: ../../source/mp/l2_storage.rst:373 msgid "" "``raw_block`` is a server-owned MP adapter. It does **not** support per-" "TP device-path mappings in MP mode." -msgstr "" +msgstr "``raw_block`` 是一个由服务器拥有的 MP 适配器。它 **不** 支持 MP 模式下的每个 TP 设备路径映射。" #: ../../source/mp/l2_storage.rst:375 msgid "" "``raw_block`` remains ``\"type\": \"raw_block\"`` for both supported " "engines." -msgstr "" +msgstr "``raw_block`` 在两个支持的引擎中保持 ``\"type\": \"raw_block\"``。" #: ../../source/mp/l2_storage.rst:376 msgid "" "``raw_block`` owns on-device slot allocation, checkpointing, and recovery" " through ``RawBlockCore``. Slot reclamation is driven by the " "shared/global L2 eviction controller or explicit ``delete()`` calls." -msgstr "" +msgstr "``raw_block`` 拥有设备上的槽分配、检查点和通过 ``RawBlockCore`` 的恢复。槽回收由共享/全局 L2 逐出控制器或显式的 ``delete()`` 调用驱动。" #: ../../source/mp/l2_storage.rst:379 msgid "" "If ``use_odirect`` is enabled, the server's ``--l1-align-bytes`` should " "be at least ``block_align``." -msgstr "" +msgstr "如果启用了 ``use_odirect``，则服务器的 ``--l1-align-bytes`` 应至少为 ``block_align``。" #: ../../source/mp/l2_storage.rst:381 msgid "``persist_enabled`` must remain ``true`` for this adapter." -msgstr "" +msgstr "``persist_enabled`` 必须保持为 ``true`` 以便此适配器正常工作。" #: ../../source/mp/l2_storage.rst:394 msgid "``mooncake_store`` -- Mooncake Store native connector" -msgstr "" +msgstr "``mooncake_store`` -- Mooncake Store 原生连接器" #: ../../source/mp/l2_storage.rst:396 msgid "" "An L2 adapter backed by the native C++ Mooncake Store connector. Uses " "`Mooncake `_ for high-performance" " distributed KV cache storage with RDMA support." -msgstr "" +msgstr "一个由原生 C++ Mooncake Store 连接器支持的 L2 适配器。使用 `Mooncake `_ 进行高性能分布式 KV Cache 存储，并支持 RDMA。" #: ../../source/mp/l2_storage.rst:400 msgid "" @@ -647,87 +647,87 @@ msgid "" "adapter factory automatically in MP mode. If the descriptor is missing " "or invalid, adapter creation fails with ``ValueError`` instead of " "silently falling back to a non-RDMA path." -msgstr "" +msgstr "当 Mooncake 配置为 ``\"protocol\": \"rdma\"`` 时，LMCache 还必须有一个有效的连续 L1 内存区域可用。分布式存储管理器会在 MP 模式下自动将此 L1 内存描述符传递给适配器工厂。如果描述符缺失或无效，适配器创建将失败并抛出 ``ValueError``，而不是默默回退到非 RDMA 路径。" #: ../../source/mp/l2_storage.rst:406 msgid "**Prerequisites -- Building with Mooncake support:**" -msgstr "" +msgstr "**前提条件 -- 构建 Mooncake 支持：**" #: ../../source/mp/l2_storage.rst:408 msgid "" "The Mooncake extension is **not** built by default. You must explicitly " "enable it:" -msgstr "" +msgstr "Mooncake 扩展**默认情况下**并未构建。您必须显式启用它：" #: ../../source/mp/l2_storage.rst:415 msgid "The ``BUILD_MOONCAKE`` environment variable controls compilation:" -msgstr "" +msgstr "``BUILD_MOONCAKE`` 环境变量控制编译：" #: ../../source/mp/l2_storage.rst:417 msgid "``BUILD_MOONCAKE=1``: Enable the Mooncake C++ extension." -msgstr "" +msgstr "``BUILD_MOONCAKE=1``：启用 Mooncake C++ 扩展。" #: ../../source/mp/l2_storage.rst:418 msgid "" "``BUILD_MOONCAKE=0``: Force disable (highest priority), even if " "``MOONCAKE_INCLUDE_DIR`` is set." -msgstr "" +msgstr "``BUILD_MOONCAKE=0``: 强制禁用（最高优先级），即使 ``MOONCAKE_INCLUDE_DIR`` 已设置。" #: ../../source/mp/l2_storage.rst:420 msgid "" "**Not set**: Falls back to checking ``MOONCAKE_INCLUDE_DIR`` for backward" " compatibility. If ``MOONCAKE_INCLUDE_DIR`` is also unset, the extension" " is skipped." -msgstr "" +msgstr "**未设置**：回退到检查 ``MOONCAKE_INCLUDE_DIR`` 以保持向后兼容。如果 ``MOONCAKE_INCLUDE_DIR`` 也未设置，则跳过该扩展。" #: ../../source/mp/l2_storage.rst:424 msgid "" "If the Mooncake headers are not installed in the system include path " "(e.g., ``/usr/local/include``), you must point to them explicitly:" -msgstr "" +msgstr "如果系统的包含路径中没有安装 Mooncake 头文件（例如，``/usr/local/include``），您必须明确指定它们：" #: ../../source/mp/l2_storage.rst:434 msgid "**LMCache-specific fields:**" -msgstr "" +msgstr "**LMCache-specific fields:**" #: ../../source/mp/l2_storage.rst:436 msgid "" "``num_workers``: Number of C++ worker threads for the shared pool " "(default ``4``, must be > 0)." -msgstr "" +msgstr "``num_workers``: 共享池的 C++ 工作线程数量（默认 ``4``，必须大于 0）。" #: ../../source/mp/l2_storage.rst:439 msgid "" "``per_op_workers`` (``dict[str, int]``, optional): A dict mapping lane " "keys to dedicated worker thread counts. Supported keys:" -msgstr "" +msgstr "``per_op_workers`` (``dict[str, int]``, 可选): 一个将车道键映射到专用工作线程计数的字典。 支持的键:" #: ../../source/mp/l2_storage.rst:442 msgid "``\"lookup\"`` — threads for ``EXISTS`` operations." -msgstr "" +msgstr "``\"lookup\"`` — 用于 ``EXISTS`` 操作的线程。" #: ../../source/mp/l2_storage.rst:443 msgid "``\"retrieve\"`` — threads for ``GET`` / load operations." -msgstr "" +msgstr "``\"retrieve\"`` — 处理 ``GET`` / 加载操作的线程。" #: ../../source/mp/l2_storage.rst:444 msgid "``\"store\"`` — threads for ``SET`` / put operations." -msgstr "" +msgstr "``\"store\"`` — 处理 ``SET`` / 放置操作的线程。" #: ../../source/mp/l2_storage.rst:445 msgid "``\"delete\"`` — threads for ``DELETE`` operations." -msgstr "" +msgstr "``\"delete\"`` — 处理 ``DELETE`` 操作的线程。" #: ../../source/mp/l2_storage.rst:447 msgid "" "Operations whose lane key is **not** present in the dict use the shared " "``num_workers`` pool. There is no requirement to set all keys — you can " "configure only the lanes that need dedicated pools." -msgstr "" +msgstr "在字典中不存在车道键的操作使用共享的 ``num_workers`` 池。没有必要设置所有键 — 您可以仅配置需要专用池的车道。" #: ../../source/mp/l2_storage.rst:451 msgid "**Mooncake fields:**" -msgstr "" +msgstr "**Mooncake 字段：**" #: ../../source/mp/l2_storage.rst:453 msgid "" @@ -738,120 +738,120 @@ msgid "" "setup keys (e.g., ``local_hostname``, ``metadata_server``, " "``master_server_addr``, ``protocol``, ``rdma_devices``, " "``global_segment_size``)." -msgstr "" +msgstr "JSON 配置中的所有其他键（除了 ``type``, ``num_workers``, ``per_op_workers``, 和 ``eviction``）都将 **原样** 转发到 Mooncake 的 ``setup_internal(ConfigDict)``。有关可用的设置键（例如 ``local_hostname``, ``metadata_server``, ``master_server_addr``, ``protocol``, ``rdma_devices``, ``global_segment_size``），请参阅 `Mooncake 文档 `_ 。" #: ../../source/mp/l2_storage.rst:491 msgid "" "For full Mooncake setup instructions (master service, metadata server, " "etc.), see `Mooncake `_ ." -msgstr "" +msgstr "有关完整的 Mooncake 设置说明（主服务、元数据服务器等），请参见 `Mooncake `_ 。" #: ../../source/mp/l2_storage.rst:494 msgid "**RDMA notes:**" -msgstr "" +msgstr "**RDMA 注意事项：**" #: ../../source/mp/l2_storage.rst:496 msgid "``protocol: \"rdma\"`` requires a valid LMCache L1 memory descriptor." -msgstr "" +msgstr "``protocol: \\\"rdma\\\"`` 需要一个有效的 LMCache L1 内存描述符。" #: ../../source/mp/l2_storage.rst:497 msgid "" "When using ``protocol: \"rdma\"``, it is recommended to disable lazy L1 " "allocation with ``--no-l1-use-lazy`` so the L1 buffer is fully allocated " "before Mooncake registers it." -msgstr "" +msgstr "在使用 ``protocol: \\\"rdma\\\"`` 时，建议通过 ``--no-l1-use-lazy`` 禁用延迟 L1 分配，以便在 Mooncake 注册之前完全分配 L1 缓冲区。" #: ../../source/mp/l2_storage.rst:500 msgid "``protocol: \"tcp\"`` does not require L1 preregistration." -msgstr "" +msgstr "``protocol: \\\"tcp\\\"`` 不需要 L1 预注册。" #: ../../source/mp/l2_storage.rst:501 msgid "" "If Mooncake RDMA initialization fails at adapter creation time, verify " "that LMCache L1 memory is enabled and that the descriptor has a non-zero " "pointer and size." -msgstr "" +msgstr "如果 Mooncake RDMA 初始化在适配器创建时失败，请验证 LMCache L1 内存是否已启用，并确保描述符具有非零指针和大小。" #: ../../source/mp/l2_storage.rst:506 msgid "``s3`` -- S3-compatible object store" -msgstr "" +msgstr "``s3`` -- S3 兼容对象存储" #: ../../source/mp/l2_storage.rst:508 msgid "" "An L2 adapter that stores KV cache objects as S3 objects using the AWS " "Common Runtime (CRT). Works with AWS S3, S3 Express One Zone, and any " "S3-compatible endpoint (MinIO, Ceph RGW, etc.)." -msgstr "" +msgstr "一个 L2 适配器，使用 AWS 通用运行时 (CRT) 将 KV Cache 对象存储为 S3 对象。支持 AWS S3、S3 Express One Zone 以及任何 S3 兼容的端点（如 MinIO、Ceph RGW 等）。" #: ../../source/mp/l2_storage.rst:514 msgid "" "``s3_endpoint``: Bucket URL -- either ``\"s3://\"`` or the bare " "host form (used for non-AWS endpoints)." -msgstr "" +msgstr "``s3_endpoint``: 存储桶 URL -- 可以是 ``\"s3://\"`` 或裸主机形式（用于非 AWS 端点）。" #: ../../source/mp/l2_storage.rst:516 msgid "``s3_region``: AWS region string (e.g. ``\"us-west-2\"``)." -msgstr "" +msgstr "``s3_region``: AWS 区域字符串（例如 ``\"us-west-2\"``）。" #: ../../source/mp/l2_storage.rst:520 msgid "``s3_num_io_threads`` (int, default ``64``): Number of CRT I/O threads." -msgstr "" +msgstr "``s3_num_io_threads`` (int, default ``64``): CRT I/O 线程的数量。" #: ../../source/mp/l2_storage.rst:521 msgid "``s3_prefer_http2`` (bool, default ``true``): Negotiate HTTP/2 via ALPN." -msgstr "" +msgstr "``s3_prefer_http2`` (bool, default ``true``): 通过 ALPN 协商 HTTP/2。" #: ../../source/mp/l2_storage.rst:522 msgid "" "``s3_enable_s3express`` (bool, default ``false``): Enable S3 Express " "signing for S3 Express One Zone buckets." -msgstr "" +msgstr "``s3_enable_s3express`` (bool, default ``false``): 为 S3 Express 单区桶启用 S3 Express 签名。" #: ../../source/mp/l2_storage.rst:524 msgid "" "``disable_tls`` (bool, default ``false``): Bypass TLS when pointing at a " "plain-HTTP endpoint (e.g. a local MinIO)." -msgstr "" +msgstr "``disable_tls`` (bool, default ``false``): 在指向普通 HTTP 端点时绕过 TLS（例如，本地 MinIO）。" #: ../../source/mp/l2_storage.rst:526 msgid "" "``aws_access_key_id`` / ``aws_secret_access_key`` (string): Static " "credentials; omit both to use the AWS default credential provider chain " "(environment, EC2 instance profile, etc.)." -msgstr "" +msgstr "``aws_access_key_id`` / ``aws_secret_access_key`` (字符串): 静态凭证；省略两者以使用 AWS 默认凭证提供程序链（环境、EC2 实例配置文件等）。" #: ../../source/mp/l2_storage.rst:529 msgid "" "``max_capacity_gb`` (float, default ``0.0``): Aggregate capacity used by " "``get_usage()``. A value of ``0`` disables aggregate eviction " "(``usage_fraction == -1.0``)." -msgstr "" +msgstr "``max_capacity_gb`` (浮点数，默认 ``0.0``)：``get_usage()`` 使用的总容量。值为 ``0`` 将禁用总的逐出（``usage_fraction == -1.0``）。" #: ../../source/mp/l2_storage.rst:547 msgid "``mock`` -- Mock adapter for testing" -msgstr "" +msgstr "``mock`` -- 测试用的模拟适配器" #: ../../source/mp/l2_storage.rst:549 msgid "" "Simulates L2 storage with configurable size and bandwidth. Useful for " "testing the L2 pipeline without real storage hardware." -msgstr "" +msgstr "模拟具有可配置大小和带宽的 L2 存储。对于在没有真实存储硬件的情况下测试 L2 管道非常有用。" #: ../../source/mp/l2_storage.rst:552 msgid "**Fields:**" -msgstr "" +msgstr "**字段：**" #: ../../source/mp/l2_storage.rst:554 msgid "``max_size_gb``: Maximum size in GB (> 0)." -msgstr "" +msgstr "``max_size_gb``: 最大大小（以 GB 为单位，> 0）。" #: ../../source/mp/l2_storage.rst:555 msgid "``mock_bandwidth_gb``: Simulated bandwidth in GB/sec (> 0)." -msgstr "" +msgstr "``mock_bandwidth_gb``: 模拟带宽，单位为 GB/秒 (> 0)。" #: ../../source/mp/l2_storage.rst:562 msgid "Multiple Adapters (Cascade)" -msgstr "" +msgstr "多个适配器（级联）" #: ../../source/mp/l2_storage.rst:564 msgid "" @@ -859,11 +859,11 @@ msgid "" "argument. Adapters are used in the order they are specified. The " "``StoreController`` pushes data to all configured adapters, and the " "``PrefetchController`` queries adapters in order during lookups." -msgstr "" +msgstr "您可以通过重复 ``--l2-adapter`` 参数来配置多个 L2 适配器。适配器按指定的顺序使用。``StoreController`` 将数据推送到所有配置的适配器，而 ``PrefetchController`` 在查找期间按顺序查询适配器。" #: ../../source/mp/l2_storage.rst:576 msgid "Store and Prefetch Policies" -msgstr "" +msgstr "存储和预取策略" #: ../../source/mp/l2_storage.rst:578 msgid "" @@ -872,76 +872,76 @@ msgid "" " L2 store. The **prefetch policy** controls how keys flow from L2 back " "to L1: when multiple adapters have the same key, the policy decides which" " adapter loads it." -msgstr "" +msgstr "**存储策略** 控制键从 L1 流向 L2 的方式：哪些适配器接收每个键，以及在成功存储到 L2 后是否从 L1 删除键。 **预取策略** 控制键从 L2 流回 L1 的方式：当多个适配器具有相同的键时，该策略决定哪个适配器加载它。" #: ../../source/mp/l2_storage.rst:584 msgid "Select policies via CLI:" -msgstr "" +msgstr "通过 CLI 选择策略：" #: ../../source/mp/l2_storage.rst:591 msgid "**Built-in policies:**" -msgstr "" +msgstr "**内置策略：**" #: ../../source/mp/l2_storage.rst:597 ../../source/mp/l2_storage.rst:630 #: ../../source/mp/l2_storage.rst:684 msgid "Flag" -msgstr "" +msgstr "标志" #: ../../source/mp/l2_storage.rst:598 msgid "Name" -msgstr "" +msgstr "名称" #: ../../source/mp/l2_storage.rst:599 msgid "Behaviour" -msgstr "" +msgstr "行为" #: ../../source/mp/l2_storage.rst:600 ../../source/mp/l2_storage.rst:603 msgid "``--l2-store-policy``" -msgstr "" +msgstr "``--l2-store-policy``" #: ../../source/mp/l2_storage.rst:601 ../../source/mp/l2_storage.rst:609 msgid "``default``" -msgstr "" +msgstr "``default``" #: ../../source/mp/l2_storage.rst:602 msgid "Store all keys to all adapters. Never delete from L1." -msgstr "" +msgstr "将所有键存储到所有适配器。永远不要从 L1 中删除。" #: ../../source/mp/l2_storage.rst:604 msgid "``skip_l1``" -msgstr "" +msgstr "``skip_l1``" #: ../../source/mp/l2_storage.rst:605 msgid "" "Buffer-only mode. Store all keys to all adapters, then **delete them " "from L1** immediately. Pair with ``--eviction-policy noop`` to avoid " "useless LRU overhead." -msgstr "" +msgstr "仅缓冲区模式。将所有键存储到所有适配器，然后**立即从 L1 中删除它们**。与 ``--eviction-policy noop`` 配对以避免无用的 LRU 开销。" #: ../../source/mp/l2_storage.rst:608 ../../source/mp/l2_storage.rst:612 msgid "``--l2-prefetch-policy``" -msgstr "" +msgstr "``--l2-prefetch-policy``" #: ../../source/mp/l2_storage.rst:610 msgid "" "For each key, pick the first (lowest-indexed) adapter that has it. " "Prefetched keys are **temporary** (deleted after the reader finishes)." -msgstr "" +msgstr "对于每个键，选择第一个（索引最低的）具有该键的适配器。预取的键是**临时的**（在读取器完成后删除）。" #: ../../source/mp/l2_storage.rst:613 msgid "``retain``" -msgstr "" +msgstr "``retain``" #: ../../source/mp/l2_storage.rst:614 msgid "" "Same load plan as ``default``, but prefetched keys are **retained** " "permanently in L1. Useful when prefetched data is likely reused by " "subsequent requests (e.g. shared system-prompt chunks)." -msgstr "" +msgstr "与 ``default`` 相同的加载计划，但预取的键在 L1 中**永久保留**。当预取的数据可能被后续请求重用时（例如共享的系统提示块），这非常有用。" #: ../../source/mp/l2_storage.rst:619 msgid "Prefetch Concurrency" -msgstr "" +msgstr "预取并发性" #: ../../source/mp/l2_storage.rst:621 msgid "" @@ -949,28 +949,28 @@ msgid "" "prefetch requests that the ``PrefetchController`` can have in flight at " "any time. A higher value increases L2-to-L1 throughput but also " "increases L1 memory pressure from in-flight data." -msgstr "" +msgstr "``--l2-prefetch-max-in-flight`` 标志限制了 ``PrefetchController`` 在任何时候可以同时进行的预取请求数量。更高的值会增加 L2 到 L1 的吞吐量，但也会增加来自在飞数据的 L1 内存压力。" #: ../../source/mp/l2_storage.rst:631 ../../source/mp/l2_storage.rst:685 #: ../../source/mp/l2_storage.rst:725 msgid "Default" -msgstr "" +msgstr "默认" #: ../../source/mp/l2_storage.rst:633 msgid "``--l2-prefetch-max-in-flight``" -msgstr "" +msgstr "``--l2-prefetch-max-in-flight``" #: ../../source/mp/l2_storage.rst:634 msgid "``8``" -msgstr "" +msgstr "``8``" #: ../../source/mp/l2_storage.rst:635 msgid "Maximum number of concurrent prefetch requests." -msgstr "" +msgstr "最大并发预取请求数。" #: ../../source/mp/l2_storage.rst:638 msgid "Buffer-Only Mode" -msgstr "" +msgstr "仅缓冲区模式" #: ../../source/mp/l2_storage.rst:640 msgid "" @@ -979,7 +979,7 @@ msgid "" "This combination deletes keys from L1 as soon as they are stored to L2 " "and disables the LRU eviction tracker entirely, reducing memory and CPU " "overhead." -msgstr "" +msgstr "当 L1 仅作为写缓冲区使用（所有数据存储在 L2 中）时，使用 ``--l2-store-policy skip_l1`` 和 ``--eviction-policy noop``。此组合会在将键存储到 L2 后立即从 L1 中删除这些键，并完全禁用 LRU 逐出跟踪器，从而减少内存和 CPU 开销。" #: ../../source/mp/l2_storage.rst:652 msgid "" @@ -987,11 +987,11 @@ msgid "" "in ``storage_controllers/`` and calling ``register_store_policy()`` or " "``register_prefetch_policy()`` at import time. See the design doc " "``l2_adapters/design_docs/overall.md`` for details." -msgstr "" +msgstr "策略是可扩展的 -- 可以通过在 ``storage_controllers/`` 中创建文件并在导入时调用 ``register_store_policy()`` 或 ``register_prefetch_policy()`` 来添加新策略。有关详细信息，请参见设计文档 ``l2_adapters/design_docs/overall.md``。" #: ../../source/mp/l2_storage.rst:658 msgid "Serde (compression / quantization)" -msgstr "" +msgstr "序列化（压缩 / 量化）" #: ../../source/mp/l2_storage.rst:660 msgid "" @@ -999,76 +999,76 @@ msgid "" "that transforms data on the way in and out of L2 — e.g. fp8 quantization " "for disk backends, or encryption for remote adapters. See :doc:`serde` " "for details and configuration." -msgstr "" +msgstr "每个适配器可以选择性地运行一个 **serde**（序列化器/反序列化器），在数据进出 L2 时进行转换——例如，针对磁盘后端的 fp8 量化，或针对远程适配器的加密。有关详细信息和配置，请参见 :doc:`serde`。" #: ../../source/mp/l2_storage.rst:666 msgid "Eviction" -msgstr "" +msgstr "逐出" #: ../../source/mp/l2_storage.rst:668 msgid "" "LMCache supports eviction at both storage tiers so that each tier can " "operate within a fixed capacity budget." -msgstr "" +msgstr "LMCache 支持在两个存储层次上进行逐出，以便每个层次可以在固定的容量预算内运行。" #: ../../source/mp/l2_storage.rst:672 msgid "L1 Eviction" -msgstr "" +msgstr "L1 逐出" #: ../../source/mp/l2_storage.rst:674 msgid "" "L1 eviction runs a single background thread that monitors overall L1 " "memory usage. When usage exceeds ``trigger_watermark``, the eviction " "policy evicts a fraction of the least-recently-used keys." -msgstr "" +msgstr "L1 逐出运行一个后台线程，监控整体 L1 内存使用情况。当使用量超过 ``trigger_watermark`` 时，逐出策略会逐出一部分最近最少使用的键。" #: ../../source/mp/l2_storage.rst:678 msgid "**CLI flags:**" -msgstr "" +msgstr "**命令行标志：**" #: ../../source/mp/l2_storage.rst:687 msgid "``--eviction-policy``" -msgstr "" +msgstr "``--eviction-policy``" #: ../../source/mp/l2_storage.rst:688 ../../source/mp/l2_storage.rst:728 msgid "*(required)*" -msgstr "" +msgstr "*(必需)*" #: ../../source/mp/l2_storage.rst:689 msgid "Policy name: ``LRU`` or ``noop``." -msgstr "" +msgstr "策略名称：``LRU`` 或 ``noop``。" #: ../../source/mp/l2_storage.rst:690 msgid "``--eviction-trigger-watermark``" -msgstr "" +msgstr "``--eviction-trigger-watermark``" #: ../../source/mp/l2_storage.rst:691 ../../source/mp/l2_storage.rst:731 msgid "``0.8``" -msgstr "" +msgstr "``0.8``" #: ../../source/mp/l2_storage.rst:692 msgid "L1 usage fraction [0, 1] above which eviction is triggered." -msgstr "" +msgstr "触发逐出的 L1 使用比例 [0, 1]。" #: ../../source/mp/l2_storage.rst:693 msgid "``--eviction-ratio``" -msgstr "" +msgstr "``--eviction-ratio``" #: ../../source/mp/l2_storage.rst:694 ../../source/mp/l2_storage.rst:734 msgid "``0.2``" -msgstr "" +msgstr "``0.2``" #: ../../source/mp/l2_storage.rst:695 msgid "Fraction of currently allocated L1 memory to evict per cycle." -msgstr "" +msgstr "每个周期逐出当前分配的 L1 内存的比例。" #: ../../source/mp/l2_storage.rst:697 msgid "**Example:**" -msgstr "" +msgstr "**示例：**" #: ../../source/mp/l2_storage.rst:706 msgid "L2 Eviction" -msgstr "" +msgstr "L2 逐出" #: ../../source/mp/l2_storage.rst:708 msgid "" @@ -1076,7 +1076,7 @@ msgid "" "independently declare an eviction policy by adding an ``\"eviction\"`` " "sub-object to its ``--l2-adapter`` JSON spec. Adapters without an " "``\"eviction\"`` key have no eviction controller." -msgstr "" +msgstr "L2 逐出是 **每个适配器** 和 **自愿选择** 的。每个适配器可以通过在其 ``--l2-adapter`` JSON 规范中添加 ``\"eviction\"`` 子对象来独立声明逐出策略。没有 ``\"eviction\"`` 键的适配器没有逐出控制器。" #: ../../source/mp/l2_storage.rst:713 msgid "" @@ -1084,100 +1084,100 @@ msgid "" " monitors that adapter's ``get_usage()`` value. Once usage exceeds " "``trigger_watermark``, the policy evicts keys until usage drops by " "``eviction_ratio``." -msgstr "" +msgstr "当为适配器启用 L2 逐出时，一个专用的后台线程会监视该适配器的 ``get_usage()`` 值。一旦使用量超过 ``trigger_watermark``，该策略将逐出键，直到使用量降低到 ``eviction_ratio``。" #: ../../source/mp/l2_storage.rst:718 msgid "**``\"eviction\"`` sub-object fields:**" -msgstr "" +msgstr "**``\"eviction\"`` 子对象字段:**" #: ../../source/mp/l2_storage.rst:724 msgid "Field" -msgstr "" +msgstr "字段" #: ../../source/mp/l2_storage.rst:727 msgid "``eviction_policy``" -msgstr "" +msgstr "``eviction_policy``" #: ../../source/mp/l2_storage.rst:729 msgid "Policy name: ``\"LRU\"`` or ``\"noop\"``." -msgstr "" +msgstr "策略名称：``\"LRU\"`` 或 ``\"noop\"``。" #: ../../source/mp/l2_storage.rst:730 msgid "``trigger_watermark``" -msgstr "" +msgstr "``trigger_watermark``" #: ../../source/mp/l2_storage.rst:732 msgid "Adapter usage fraction [0, 1] above which eviction is triggered." -msgstr "" +msgstr "触发逐出的适配器使用比例 [0, 1]。" #: ../../source/mp/l2_storage.rst:733 msgid "``eviction_ratio``" -msgstr "" +msgstr "``eviction_ratio``" #: ../../source/mp/l2_storage.rst:735 msgid "Fraction of used capacity to evict per cycle." -msgstr "" +msgstr "每个周期逐出的已用容量的比例。" #: ../../source/mp/l2_storage.rst:737 msgid "**Example — nixl_store with LRU eviction:**" -msgstr "" +msgstr "**示例 — 使用 LRU 逐出的 nixl_store:**" #: ../../source/mp/l2_storage.rst:753 msgid "**Adapter support:**" -msgstr "" +msgstr "**适配器支持：**" #: ../../source/mp/l2_storage.rst:759 msgid "Adapter" -msgstr "" +msgstr "适配器" #: ../../source/mp/l2_storage.rst:760 msgid "L2 Eviction Support" -msgstr "" +msgstr "L2 逐出支持" #: ../../source/mp/l2_storage.rst:761 msgid "``nixl_store``" -msgstr "" +msgstr "``nixl_store``" #: ../../source/mp/l2_storage.rst:762 msgid "" "Full support. ``delete`` frees pool slots; pinned keys (in-flight loads) " "are skipped and retried on the next cycle." -msgstr "" +msgstr "完全支持。``delete`` 释放池槽；固定键（正在进行的加载）会被跳过，并在下一个周期重试。" #: ../../source/mp/l2_storage.rst:764 msgid "``nixl_store_dynamic``" -msgstr "" +msgstr "``nixl_store_dynamic``" #: ../../source/mp/l2_storage.rst:765 msgid "" "Full support. ``delete`` removes data files from disk; pinned keys are " "skipped. ``get_usage`` is byte-based (``_total_bytes / " "max_capacity_bytes``)." -msgstr "" +msgstr "完全支持。``delete`` 从磁盘中删除数据文件；被固定的键会被跳过。``get_usage`` 是基于字节的 (``_total_bytes / max_capacity_bytes``)。" #: ../../source/mp/l2_storage.rst:768 msgid "``mock``" -msgstr "" +msgstr "``mock``" #: ../../source/mp/l2_storage.rst:769 msgid "" "Full support. Useful for testing eviction behaviour without real storage " "hardware." -msgstr "" +msgstr "完全支持。对于测试逐出行为而无需真实存储硬件非常有用。" #: ../../source/mp/l2_storage.rst:771 msgid "``raw_block``" -msgstr "" +msgstr "``raw_block``" #: ../../source/mp/l2_storage.rst:772 msgid "" "Full shared/global eviction support. ``delete`` recycles raw-block slots;" " locked entries are skipped and retried on the next cycle." -msgstr "" +msgstr "完全支持共享/全局逐出。``delete`` 回收原始块槽；被锁定的条目会被跳过，并在下一个周期重试。" #: ../../source/mp/l2_storage.rst:774 msgid "``s3``" -msgstr "" +msgstr "``s3``" #: ../../source/mp/l2_storage.rst:775 msgid "" @@ -1186,87 +1186,87 @@ msgid "" "``max_capacity_gb`` is ``0`` (disabled); set a non-zero " "``max_capacity_gb`` to enable the watermark-triggered eviction " "controller." -msgstr "" +msgstr "``delete`` 从存储桶中删除对象并释放聚合字节计数。当 ``max_capacity_gb`` 为 ``0``（禁用）时，``get_usage`` 报告 ``usage_fraction == -1.0``；设置非零的 ``max_capacity_gb`` 以启用基于水印的逐出控制器。" #: ../../source/mp/l2_storage.rst:780 msgid "``dax``" -msgstr "" +msgstr "``dax``" #: ../../source/mp/l2_storage.rst:781 msgid "" "Full support. ``delete`` removes unlocked keys from the in-memory index " "immediately and recycles fixed slots once active read borrows drain. " "Usage is slot-based." -msgstr "" +msgstr "完全支持。``delete`` 会立即从内存索引中移除未锁定的键，并在活动读取借用耗尽后回收固定槽。使用基于槽的方式。" #: ../../source/mp/l2_storage.rst:784 msgid "``mooncake_store``" -msgstr "" +msgstr "``mooncake_store``" #: ../../source/mp/l2_storage.rst:785 msgid "No eviction support (native connector adapter)." -msgstr "" +msgstr "不支持逐出（原生连接器适配器）。" #: ../../source/mp/l2_storage.rst:786 msgid "``fs``" -msgstr "" +msgstr "``fs``" #: ../../source/mp/l2_storage.rst:787 msgid "No eviction support (``delete`` and ``get_usage`` are no-ops)." -msgstr "" +msgstr "不支持逐出（``delete`` 和 ``get_usage`` 是无操作）。" #: ../../source/mp/l2_storage.rst:788 msgid "native connectors" -msgstr "" +msgstr "原生连接器" #: ../../source/mp/l2_storage.rst:789 msgid "No eviction support." -msgstr "" +msgstr "不支持逐出。" #: ../../source/mp/l2_storage.rst:793 msgid "" "Each L2 adapter instance gets its own independent eviction controller and" " policy. Two adapters of the same type can have different watermarks or " "policies." -msgstr "" +msgstr "每个 L2 适配器实例都有自己独立的逐出控制器和策略。两个相同类型的适配器可以有不同的水位线或策略。" #: ../../source/mp/l2_storage.rst:798 msgid "Combined L1 + L2 Eviction Example" -msgstr "" +msgstr "L1 + L2 逐出示例" #: ../../source/mp/l2_storage.rst:818 msgid "In this setup:" -msgstr "" +msgstr "在此设置中：" #: ../../source/mp/l2_storage.rst:820 #, python-format msgid "" "L1 evicts from memory when it is 80 % full, reclaiming 20 % of allocated " "memory per cycle." -msgstr "" +msgstr "当 L1 的内存使用达到 80% 时，它会逐出内存，每个周期回收 20% 的分配内存。" #: ../../source/mp/l2_storage.rst:822 #, python-format msgid "" "L2 (NIXL/GDS) evicts from the storage pool when 90 % of pool slots are " "occupied, reclaiming 10 % per cycle." -msgstr "" +msgstr "L2 (NIXL/GDS) 在存储池占用 90% 的槽位时进行逐出，每个周期回收 10%。" #: ../../source/mp/l2_storage.rst:824 msgid "" "Both tiers use independent LRU policies, so each evicts its own least-" "recently-used keys." -msgstr "" +msgstr "两个层次使用独立的 LRU 策略，因此每个层次逐出其自身最近最少使用的键。" #: ../../source/mp/l2_storage.rst:828 msgid "Verifying L2 Storage" -msgstr "" +msgstr "验证 L2 存储" #: ../../source/mp/l2_storage.rst:830 msgid "Set ``LMCACHE_LOG_LEVEL=DEBUG`` to see L2 activity in the server logs:" -msgstr "" +msgstr "将 ``LMCACHE_LOG_LEVEL=DEBUG`` 设置为在服务器日志中查看 L2 活动：" #: ../../source/mp/l2_storage.rst:838 msgid "Expected log messages when L2 is active:" -msgstr "" +msgstr "当 L2 活动时预期的日志消息：" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/observability.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/observability.po index 153ff743e85..92b4421f53d 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/observability.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/observability.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/mp/observability.rst:2 msgid "Observability" -msgstr "" +msgstr "可观察性" #: ../../source/mp/observability.rst:4 msgid "" @@ -29,39 +29,39 @@ msgid "" "modes: **metrics** (Prometheus counters via OTel), **logging** (Python " "logging with optional OTel log forwarding), and **tracing** (OTel spans " "for per-request latency)." -msgstr "" +msgstr "LMCache 多进程模式提供三种互补的可观察性模式：**指标**（通过 OTel 的 Prometheus 计数器）、**日志**（带有可选 OTel 日志转发的 Python 日志）和 **追踪**（OTel 跨度用于每个请求的延迟）。" #: ../../source/mp/observability.rst:9 msgid "" "All three modes are powered by an internal **EventBus** that decouples " "producers (L1Manager, StorageManager, MPCacheEngine) from subscribers." -msgstr "" +msgstr "这三种模式都由一个内部 **EventBus** 驱动，该总线将生产者（L1Manager、StorageManager、MPCacheEngine）与订阅者解耦。" #: ../../source/mp/observability.rst:17 msgid "Quick Start" -msgstr "" +msgstr "快速开始" #: ../../source/mp/observability.rst:19 msgid "" "By default, **metrics** and **logging** are enabled; **tracing** is " "disabled. No extra flags are needed:" -msgstr "" +msgstr "默认情况下，**指标**和**日志记录**是启用的；**追踪**是禁用的。不需要额外的标志：" #: ../../source/mp/observability.rst:27 msgid "To enable tracing, supply an OTLP endpoint:" -msgstr "" +msgstr "要启用追踪，请提供 OTLP 端点：" #: ../../source/mp/observability.rst:36 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/mp/observability.rst:42 msgid "Argument" -msgstr "" +msgstr "参数" #: ../../source/mp/observability.rst:43 ../../source/mp/observability.rst:98 msgid "Default" -msgstr "" +msgstr "默认" #: ../../source/mp/observability.rst:44 ../../source/mp/observability.rst:99 #: ../../source/mp/observability.rst:152 ../../source/mp/observability.rst:181 @@ -72,93 +72,93 @@ msgstr "" #: ../../source/mp/observability.rst:529 ../../source/mp/observability.rst:560 #: ../../source/mp/observability.rst:618 ../../source/mp/observability.rst:738 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/mp/observability.rst:45 msgid "``--disable-observability``" -msgstr "" +msgstr "``--disable-observability``" #: ../../source/mp/observability.rst:46 ../../source/mp/observability.rst:50 #: ../../source/mp/observability.rst:53 ../../source/mp/observability.rst:56 msgid "off" -msgstr "" +msgstr "关闭" #: ../../source/mp/observability.rst:47 msgid "" "Master switch: disable the EventBus entirely (no metrics, logging, or " "tracing subscribers are registered)." -msgstr "" +msgstr "主开关：完全禁用 EventBus（没有注册任何指标、日志或追踪订阅者）。" #: ../../source/mp/observability.rst:49 msgid "``--disable-metrics``" -msgstr "" +msgstr "``--disable-metrics``" #: ../../source/mp/observability.rst:51 msgid "Skip metrics subscribers (Prometheus endpoint is not started)." -msgstr "" +msgstr "跳过指标订阅者（Prometheus 端点未启动）。" #: ../../source/mp/observability.rst:52 msgid "``--disable-logging``" -msgstr "" +msgstr "``--disable-logging``" #: ../../source/mp/observability.rst:54 msgid "Skip logging subscribers." -msgstr "" +msgstr "跳过日志订阅者。" #: ../../source/mp/observability.rst:55 msgid "``--enable-tracing``" -msgstr "" +msgstr "``--enable-tracing``" #: ../../source/mp/observability.rst:57 msgid "Register tracing subscribers. Requires ``--otlp-endpoint``." -msgstr "" +msgstr "注册追踪订阅者。需要 ``--otlp-endpoint``。" #: ../../source/mp/observability.rst:58 msgid "``--event-bus-queue-size``" -msgstr "" +msgstr "``--event-bus-queue-size``" #: ../../source/mp/observability.rst:59 msgid "``10000``" -msgstr "" +msgstr "``10000``" #: ../../source/mp/observability.rst:60 msgid "Maximum events in the EventBus queue before tail-drop." -msgstr "" +msgstr "事件总线队列中最大事件数，超过后将进行尾部丢弃。" #: ../../source/mp/observability.rst:61 msgid "``--otlp-endpoint``" -msgstr "" +msgstr "``--otlp-endpoint``" #: ../../source/mp/observability.rst:62 ../../source/mp/observability.rst:80 #: ../../source/mp/observability.rst:86 msgid "*(none)*" -msgstr "" +msgstr "*(无)*" #: ../../source/mp/observability.rst:63 msgid "" "OTLP gRPC endpoint (e.g. ``http://localhost:4317``). Used for exporting " "metrics (push mode) and traces." -msgstr "" +msgstr "OTLP gRPC 端点（例如 ``http://localhost:4317``）。用于导出指标（推送模式）和跟踪。" #: ../../source/mp/observability.rst:65 msgid "``--prometheus-port``" -msgstr "" +msgstr "``--prometheus-port``" #: ../../source/mp/observability.rst:66 msgid "``9090``" -msgstr "" +msgstr "``9090``" #: ../../source/mp/observability.rst:67 msgid "Port for the Prometheus ``/metrics`` HTTP endpoint." -msgstr "" +msgstr "Prometheus ``/metrics`` HTTP 端点的端口。" #: ../../source/mp/observability.rst:68 msgid "``--service-instance-id``" -msgstr "" +msgstr "``--service-instance-id``" #: ../../source/mp/observability.rst:69 msgid "*(unset, default UUID v4)*" -msgstr "" +msgstr "*(未设置，默认 UUID v4)*" #: ../../source/mp/observability.rst:70 msgid "" @@ -167,25 +167,25 @@ msgid "" " is not passed, defaults to a random UUID v4 minted at startup. Pass " "``--service-instance-id=\"\"`` to force an explicit empty value. See :ref" ":`mp-observability-resource`." -msgstr "" +msgstr "此 MP 服务器实例的标识符。作为 OTel 资源属性 ``service.instance.id`` 附加到每个指标和跨度上。当未传递该标志时，默认为启动时生成的随机 UUID v4。传递 ``--service-instance-id=\\\"\\\"`` 以强制设置显式空值。请参阅 :ref:`mp-observability-resource`。" #: ../../source/mp/observability.rst:75 msgid "``--metrics-sample-rate``" -msgstr "" +msgstr "``--metrics-sample-rate``" #: ../../source/mp/observability.rst:76 msgid "``0.01``" -msgstr "" +msgstr "``0.01``" #: ../../source/mp/observability.rst:77 msgid "" "Fraction of chunks/blocks to track for lifecycle histograms (0, 1.0]. " "Counters always count all events. Default is 1%." -msgstr "" +msgstr "跟踪生命周期直方图的块/块的比例 (0, 1.0]。计数器始终计算所有事件。默认值为 1%。" #: ../../source/mp/observability.rst:79 msgid "``--trace-level``" -msgstr "" +msgstr "``--trace-level``" #: ../../source/mp/observability.rst:81 msgid "" @@ -193,44 +193,44 @@ msgid "" "supported (records ``StorageManager`` public-API calls for offline " "replay). When unset, trace recording is off. See :ref:`trace-recording` " "for details." -msgstr "" +msgstr "在给定级别启用跟踪记录。目前仅支持 ``storage``（记录 ``StorageManager`` 公共 API 调用以便离线重放）。未设置时，跟踪记录处于关闭状态。有关详细信息，请参见 :ref:`trace-recording`。" #: ../../source/mp/observability.rst:85 msgid "``--trace-output``" -msgstr "" +msgstr "``--trace-output``" #: ../../source/mp/observability.rst:87 msgid "" "Path to write the trace file. If omitted while ``--trace-level`` is set, " "a timestamped file under ``$TMPDIR`` is minted (``lmcache-" "trace--.lct``) and its path is logged at INFO." -msgstr "" +msgstr "写入跟踪文件的路径。如果在设置了 ``--trace-level`` 的情况下省略此项，将在 ``$TMPDIR`` 下生成一个带时间戳的文件（``lmcache-trace--.lct``），并在 INFO 级别记录其路径。" #: ../../source/mp/observability.rst:91 msgid "**Environment variables:**" -msgstr "" +msgstr "**环境变量：**" #: ../../source/mp/observability.rst:97 msgid "Variable" -msgstr "" +msgstr "变量" #: ../../source/mp/observability.rst:100 msgid "``LMCACHE_LOG_LEVEL``" -msgstr "" +msgstr "``LMCACHE_LOG_LEVEL``" #: ../../source/mp/observability.rst:101 msgid "``INFO``" -msgstr "" +msgstr "``INFO``" #: ../../source/mp/observability.rst:102 msgid "" "Controls the log level for all LMCache loggers. Valid values: ``DEBUG``, " "``INFO``, ``WARNING``, ``ERROR``, ``CRITICAL``." -msgstr "" +msgstr "控制所有 LMCache 日志记录器的日志级别。有效值：``DEBUG``、``INFO``、``WARNING``、``ERROR``、``CRITICAL``。" #: ../../source/mp/observability.rst:106 msgid "Metrics" -msgstr "" +msgstr "指标" #: ../../source/mp/observability.rst:108 msgid "" @@ -238,18 +238,18 @@ msgid "" "in-process **Prometheus** ``/metrics`` HTTP endpoint (default port 9090)." " When ``--otlp-endpoint`` is set, metrics are also pushed to the OTel " "collector." -msgstr "" +msgstr "通过 OpenTelemetry 计数器收集指标，并通过进程内的 **Prometheus** ``/metrics`` HTTP 端点导出（默认端口 9090）。当设置 ``--otlp-endpoint`` 时，指标也会推送到 OTel 收集器。" #: ../../source/mp/observability.rst:113 msgid "" "All metrics use the ``lmcache_mp.`` prefix (multiprocess). On Prometheus," " dots are converted to underscores and counters get a ``_total`` suffix " "(e.g. ``lmcache_mp_l1_read_keys_total``)." -msgstr "" +msgstr "所有指标使用 ``lmcache_mp.`` 前缀（多进程）。在 Prometheus 中，点被转换为下划线，计数器会添加 ``_total`` 后缀（例如 ``lmcache_mp_l1_read_keys_total``）。" #: ../../source/mp/observability.rst:120 msgid "Global Resource Attributes" -msgstr "" +msgstr "全局资源属性" #: ../../source/mp/observability.rst:122 msgid "" @@ -257,31 +257,31 @@ msgid "" "attributes built at startup. These identify the process producing the " "telemetry and are orthogonal to per-metric attributes such as " "``cache_salt``." -msgstr "" +msgstr "每个由 MP 服务器导出的指标和跨度都携带在启动时构建的资源级属性。这些属性标识生成遥测的进程，并且与每个指标的属性（例如 ``cache_salt``）是正交的。" #: ../../source/mp/observability.rst:131 ../../source/mp/observability.rst:736 msgid "Attribute" -msgstr "" +msgstr "属性" #: ../../source/mp/observability.rst:132 msgid "CLI flag / config" -msgstr "" +msgstr "命令行标志 / 配置" #: ../../source/mp/observability.rst:133 msgid "Default when unset" -msgstr "" +msgstr "未设置时的默认值" #: ../../source/mp/observability.rst:134 msgid "``service.instance.id``" -msgstr "" +msgstr "``service.instance.id``" #: ../../source/mp/observability.rst:135 msgid "``--service-instance-id`` / ``ObservabilityConfig.service_instance_id``" -msgstr "" +msgstr "``--service-instance-id`` / ``ObservabilityConfig.service_instance_id``" #: ../../source/mp/observability.rst:136 msgid "Random UUID v4 minted at startup." -msgstr "" +msgstr "在启动时生成的随机 UUID v4。" #: ../../source/mp/observability.rst:138 msgid "" @@ -289,11 +289,11 @@ msgid "" "and propagate to every exported datapoint via OTLP. On Prometheus, SDK " "resource attributes surface on the ``target_info`` series rather than on " "each time-series — this is standard OTel behavior." -msgstr "" +msgstr "资源属性附加到 ``MeterProvider`` / ``TracerProvider`` 并通过 OTLP 传播到每个导出的数据点。在 Prometheus 中，SDK 资源属性出现在 ``target_info`` 系列上，而不是每个时间序列上——这是标准的 OTel 行为。" #: ../../source/mp/observability.rst:144 msgid "StorageManager Metrics" -msgstr "" +msgstr "存储管理器指标" #: ../../source/mp/observability.rst:150 ../../source/mp/observability.rst:179 #: ../../source/mp/observability.rst:216 ../../source/mp/observability.rst:251 @@ -303,7 +303,7 @@ msgstr "" #: ../../source/mp/observability.rst:527 ../../source/mp/observability.rst:558 #: ../../source/mp/observability.rst:616 msgid "Metric" -msgstr "" +msgstr "指标" #: ../../source/mp/observability.rst:151 ../../source/mp/observability.rst:180 #: ../../source/mp/observability.rst:217 ../../source/mp/observability.rst:252 @@ -313,11 +313,11 @@ msgstr "" #: ../../source/mp/observability.rst:528 ../../source/mp/observability.rst:559 #: ../../source/mp/observability.rst:617 msgid "Type" -msgstr "" +msgstr "类型" #: ../../source/mp/observability.rst:153 msgid "``lmcache_mp.sm_read_requests``" -msgstr "" +msgstr "``lmcache_mp.sm_read_requests``" #: ../../source/mp/observability.rst:154 ../../source/mp/observability.rst:157 #: ../../source/mp/observability.rst:160 ../../source/mp/observability.rst:163 @@ -333,93 +333,93 @@ msgstr "" #: ../../source/mp/observability.rst:340 ../../source/mp/observability.rst:346 #: ../../source/mp/observability.rst:355 msgid "Counter" -msgstr "" +msgstr "计数器" #: ../../source/mp/observability.rst:155 msgid "Number of read (prefetch) requests received by the StorageManager." -msgstr "" +msgstr "存储管理器接收到的读取（预取）请求的数量。" #: ../../source/mp/observability.rst:156 msgid "``lmcache_mp.sm_read_succeed_keys``" -msgstr "" +msgstr "``lmcache_mp.sm_read_succeed_keys``" #: ../../source/mp/observability.rst:158 msgid "Number of keys successfully read from LMCache." -msgstr "" +msgstr "从 LMCache 成功读取的键的数量。" #: ../../source/mp/observability.rst:159 msgid "``lmcache_mp.sm_read_failed_keys``" -msgstr "" +msgstr "``lmcache_mp.sm_read_failed_keys``" #: ../../source/mp/observability.rst:161 msgid "Number of keys that failed to read." -msgstr "" +msgstr "读取失败的键的数量。" #: ../../source/mp/observability.rst:162 msgid "``lmcache_mp.sm_write_requests``" -msgstr "" +msgstr "``lmcache_mp.sm_write_requests``" #: ../../source/mp/observability.rst:164 msgid "Number of write (reserve) requests." -msgstr "" +msgstr "写入（保留）请求的数量。" #: ../../source/mp/observability.rst:165 msgid "``lmcache_mp.sm_write_succeed_keys``" -msgstr "" +msgstr "``lmcache_mp.sm_write_succeed_keys``" #: ../../source/mp/observability.rst:167 msgid "Number of keys successfully reserved for write." -msgstr "" +msgstr "成功保留用于写入的键的数量。" #: ../../source/mp/observability.rst:168 msgid "``lmcache_mp.sm_write_failed_keys``" -msgstr "" +msgstr "``lmcache_mp.sm_write_failed_keys``" #: ../../source/mp/observability.rst:170 msgid "Number of keys that failed to reserve (OOM, write conflict)." -msgstr "" +msgstr "未能保留的键的数量（内存不足，写入冲突）。" #: ../../source/mp/observability.rst:173 msgid "L1 Metrics" -msgstr "" +msgstr "L1 指标" #: ../../source/mp/observability.rst:182 msgid "``lmcache_mp.l1_read_keys``" -msgstr "" +msgstr "``lmcache_mp.l1_read_keys``" #: ../../source/mp/observability.rst:184 msgid "Number of keys read from L1." -msgstr "" +msgstr "从 L1 读取的键的数量。" #: ../../source/mp/observability.rst:185 msgid "``lmcache_mp.l1_write_keys``" -msgstr "" +msgstr "``lmcache_mp.l1_write_keys``" #: ../../source/mp/observability.rst:187 msgid "Number of keys written to L1." -msgstr "" +msgstr "写入 L1 的键数量。" #: ../../source/mp/observability.rst:188 msgid "``lmcache_mp.l1_evicted_keys``" -msgstr "" +msgstr "``lmcache_mp.l1_evicted_keys``" #: ../../source/mp/observability.rst:190 msgid "Number of keys evicted by the EvictionController." -msgstr "" +msgstr "EvictionController 逐出的键的数量。" #: ../../source/mp/observability.rst:191 msgid "``lmcache_mp.l1_eviction_loop_ticks``" -msgstr "" +msgstr "``lmcache_mp.l1_eviction_loop_ticks``" #: ../../source/mp/observability.rst:193 msgid "" "L1 eviction-loop iterations (every cycle, regardless of whether the " "watermark was crossed). Driven by ``L1_EVICTION_LOOP_TICK``." -msgstr "" +msgstr "L1 逐出循环迭代（每个周期，无论水位线是否被跨越）。由 ``L1_EVICTION_LOOP_TICK`` 驱动。" #: ../../source/mp/observability.rst:195 msgid "``lmcache_mp.l1_eviction_loop_triggered``" -msgstr "" +msgstr "``lmcache_mp.l1_eviction_loop_triggered``" #: ../../source/mp/observability.rst:197 msgid "" @@ -427,11 +427,11 @@ msgid "" " policy actually ran. The two counters distinguish \"loop is alive\" from" " \"eviction fired\" — important when debugging short-lived benchmarks " "that complete faster than the 1 Hz polling cycle." -msgstr "" +msgstr "L1 逐出循环迭代，其中 ``usage >= watermark`` 且逐出策略实际运行。两个计数器区分“循环仍在运行”和“逐出已触发”——在调试完成速度快于 1 Hz 轮询周期的短期基准时，这一点很重要。" #: ../../source/mp/observability.rst:204 msgid "L1 Chunk Lifecycle Histograms" -msgstr "" +msgstr "L1 块生命周期直方图" #: ../../source/mp/observability.rst:206 msgid "" @@ -440,11 +440,11 @@ msgid "" "counters above always count all events. Sampling is deterministic (hash-" "based), so the same key always gets the same decision with zero memory " "overhead." -msgstr "" +msgstr "通过 ``L1LifecycleSubscriber`` 进行采样的（默认 1%）块级生命周期跟踪。只有采样的块会对直方图产生贡献；上述计数器始终计算所有事件。采样是确定性的（基于哈希），因此相同的键总是会得到相同的决策，且没有内存开销。" #: ../../source/mp/observability.rst:219 msgid "``lmcache_mp.l1_chunk_lifetime_seconds``" -msgstr "" +msgstr "``lmcache_mp.l1_chunk_lifetime_seconds``" #: ../../source/mp/observability.rst:220 ../../source/mp/observability.rst:223 #: ../../source/mp/observability.rst:226 ../../source/mp/observability.rst:229 @@ -453,39 +453,39 @@ msgstr "" #: ../../source/mp/observability.rst:475 ../../source/mp/observability.rst:509 #: ../../source/mp/observability.rst:512 msgid "Histogram" -msgstr "" +msgstr "直方图" #: ../../source/mp/observability.rst:221 msgid "Time from allocation to eviction per sampled chunk." -msgstr "" +msgstr "每个采样块从分配到逐出的时间。" #: ../../source/mp/observability.rst:222 msgid "``lmcache_mp.l1_chunk_idle_before_evict_seconds``" -msgstr "" +msgstr "``lmcache_mp.l1_chunk_idle_before_evict_seconds``" #: ../../source/mp/observability.rst:224 msgid "Time from last access to eviction per sampled chunk." -msgstr "" +msgstr "每个采样块从最后访问到逐出的时间。" #: ../../source/mp/observability.rst:225 msgid "``lmcache_mp.l1_chunk_reuse_gap_seconds``" -msgstr "" +msgstr "``lmcache_mp.l1_chunk_reuse_gap_seconds``" #: ../../source/mp/observability.rst:227 msgid "Time gap between consecutive touches (read or write) of the same chunk." -msgstr "" +msgstr "同一块的连续访问（读取或写入）之间的时间间隔。" #: ../../source/mp/observability.rst:228 msgid "``lmcache_mp.l1_chunk_evict_reuse_gap_seconds``" -msgstr "" +msgstr "``lmcache_mp.l1_chunk_evict_reuse_gap_seconds``" #: ../../source/mp/observability.rst:230 msgid "Time from eviction to next reuse (capped at 300 s)." -msgstr "" +msgstr "逐出到下次重用的时间（上限为 300 秒）。" #: ../../source/mp/observability.rst:233 msgid "StorageManager Real-Reuse Metrics" -msgstr "" +msgstr "存储管理器真实重用指标" #: ../../source/mp/observability.rst:235 msgid "" @@ -494,7 +494,7 @@ msgid "" "(``SM_READ_PREFETCHED_FINISHED``, ``SM_WRITE_FINISHED``). Internal read-" "lock releases by the store/prefetch controllers are excluded so the " "signal reflects user-driven access only." -msgstr "" +msgstr "由 ``SMLifecycleSubscriber`` 发出的工作负载级重用直方图，由面向调用者的 StorageManager 事件驱动（``SM_READ_PREFETCHED_FINISHED``，``SM_WRITE_FINISHED``）。 存储/预取控制器的内部读锁释放被排除，因此信号仅反映用户驱动的访问。" #: ../../source/mp/observability.rst:241 msgid "" @@ -503,26 +503,26 @@ msgid "" "chunk (regardless of sampling) so the chunks-gap reflects true storage " "volume; the histogram itself records gaps only for chunks that pass the " "(deterministic, hash-based) sampling gate." -msgstr "" +msgstr "两个直方图都标记有 ``cache_salt`` 以实现每个租户的隔离。每个块的每次读取和写入（无论是否采样）都会使每个盐值的访问计数器增加，因此块间隙反映了真实的存储量；直方图本身仅记录通过（确定性、基于哈希的）采样门的块的间隙。" #: ../../source/mp/observability.rst:254 msgid "``lmcache_mp.real_reuse_gap_seconds``" -msgstr "" +msgstr "``lmcache_mp.real_reuse_gap_seconds``" #: ../../source/mp/observability.rst:255 ../../source/mp/observability.rst:260 msgid "Histogram (tag: ``cache_salt``)" -msgstr "" +msgstr "直方图（标签：``cache_salt``）" #: ../../source/mp/observability.rst:256 msgid "" "Time gap between a chunk's last access (read or write) and its next read." " Captures storage cost — how long a stored chunk sat between accesses. " "Emitted only on read events." -msgstr "" +msgstr "块的最后一次访问（读取或写入）与下一次读取之间的时间间隔。捕获存储成本——一个存储块在访问之间静止了多久。仅在读取事件中发出。" #: ../../source/mp/observability.rst:259 msgid "``lmcache_mp.real_reuse_gap_chunks``" -msgstr "" +msgstr "``lmcache_mp.real_reuse_gap_chunks``" #: ../../source/mp/observability.rst:261 msgid "" @@ -530,119 +530,119 @@ msgid "" "chunk. Captures storage volume — how many chunk-accesses occurred while " "this chunk waited for its next read. Emitted on read events for sampled " "chunks." -msgstr "" +msgstr "每个 ``cache_salt`` 的访问计数器在同一块的两次读取之间的间隙。捕获存储量——在此块等待下一个读取时发生了多少次块访问。在采样块的读取事件中发出。" #: ../../source/mp/observability.rst:267 msgid "L2 Metrics" -msgstr "" +msgstr "L2 指标" #: ../../source/mp/observability.rst:276 msgid "``lmcache_mp.l2_store_tasks``" -msgstr "" +msgstr "``lmcache_mp.l2_store_tasks``" #: ../../source/mp/observability.rst:278 msgid "Number of L2 store tasks submitted." -msgstr "" +msgstr "提交的 L2 存储任务数量。" #: ../../source/mp/observability.rst:279 msgid "``lmcache_mp.l2_store_keys``" -msgstr "" +msgstr "``lmcache_mp.l2_store_keys``" #: ../../source/mp/observability.rst:281 msgid "Number of keys submitted for L2 store." -msgstr "" +msgstr "提交到 L2 存储的键的数量。" #: ../../source/mp/observability.rst:282 msgid "``lmcache_mp.l2_store_completed``" -msgstr "" +msgstr "``lmcache_mp.l2_store_completed``" #: ../../source/mp/observability.rst:283 ../../source/mp/observability.rst:313 msgid "Counter (attr: ``l2_name``)" -msgstr "" +msgstr "计数器 (属性: ``l2_name``)" #: ../../source/mp/observability.rst:284 msgid "Number of L2 store tasks completed, labeled by adapter type." -msgstr "" +msgstr "按适配器类型标记的已完成 L2 存储任务数量。" #: ../../source/mp/observability.rst:285 msgid "``lmcache_mp.l2_store_succeeded_keys``" -msgstr "" +msgstr "``lmcache_mp.l2_store_succeeded_keys``" #: ../../source/mp/observability.rst:287 msgid "Number of keys successfully stored to L2." -msgstr "" +msgstr "成功存储到 L2 的键的数量。" #: ../../source/mp/observability.rst:288 msgid "``lmcache_mp.l2_store_failed_keys``" -msgstr "" +msgstr "``lmcache_mp.l2_store_failed_keys``" #: ../../source/mp/observability.rst:290 msgid "Number of keys that failed to store to L2." -msgstr "" +msgstr "存储到 L2 的键的数量。" #: ../../source/mp/observability.rst:291 msgid "``lmcache_mp.l2_prefetch_lookups``" -msgstr "" +msgstr "``lmcache_mp.l2_prefetch_lookups``" #: ../../source/mp/observability.rst:293 msgid "Number of L2 prefetch lookup requests." -msgstr "" +msgstr "L2 预取查找请求的数量。" #: ../../source/mp/observability.rst:294 msgid "``lmcache_mp.l2_prefetch_lookup_keys``" -msgstr "" +msgstr "``lmcache_mp.l2_prefetch_lookup_keys``" #: ../../source/mp/observability.rst:296 msgid "Number of keys submitted for L2 prefetch lookup." -msgstr "" +msgstr "提交用于 L2 预取查找的键的数量。" #: ../../source/mp/observability.rst:297 msgid "``lmcache_mp.l2_prefetch_hit_keys``" -msgstr "" +msgstr "``lmcache_mp.l2_prefetch_hit_keys``" #: ../../source/mp/observability.rst:299 msgid "Number of prefix keys found in L2 lookup." -msgstr "" +msgstr "在 L2 查找中找到的前缀键数量。" #: ../../source/mp/observability.rst:300 msgid "``lmcache_mp.l2_prefetch_load_tasks``" -msgstr "" +msgstr "``lmcache_mp.l2_prefetch_load_tasks``" #: ../../source/mp/observability.rst:302 msgid "Number of L2 prefetch load tasks submitted." -msgstr "" +msgstr "提交的 L2 预取加载任务数量。" #: ../../source/mp/observability.rst:303 msgid "``lmcache_mp.l2_prefetch_load_keys``" -msgstr "" +msgstr "``lmcache_mp.l2_prefetch_load_keys``" #: ../../source/mp/observability.rst:305 msgid "Number of keys submitted for L2 load." -msgstr "" +msgstr "提交用于 L2 加载的键的数量。" #: ../../source/mp/observability.rst:306 msgid "``lmcache_mp.l2_prefetch_loaded_keys``" -msgstr "" +msgstr "``lmcache_mp.l2_prefetch_loaded_keys``" #: ../../source/mp/observability.rst:308 msgid "Number of keys successfully loaded from L2." -msgstr "" +msgstr "成功从 L2 加载的键的数量。" #: ../../source/mp/observability.rst:309 msgid "``lmcache_mp.l2_prefetch_failed_keys``" -msgstr "" +msgstr "``lmcache_mp.l2_prefetch_failed_keys``" #: ../../source/mp/observability.rst:311 msgid "Number of keys that failed to load from L2." -msgstr "" +msgstr "从 L2 加载失败的键的数量。" #: ../../source/mp/observability.rst:312 msgid "``lmcache_mp.l2_load_completed``" -msgstr "" +msgstr "``lmcache_mp.l2_load_completed``" #: ../../source/mp/observability.rst:314 msgid "Number of per-adapter L2 load tasks completed, labeled by adapter type." -msgstr "" +msgstr "每个适配器类型完成的 L2 加载任务数量。" #: ../../source/mp/observability.rst:316 #, python-brace-format @@ -653,11 +653,11 @@ msgid "" "``rate(lmcache_mp_l2_store_completed_total{l2_name=\"...\"}[1m])`` (and " "the equivalent for loads). No separate ``*_iops`` metric is exported; " "keeping the raw counter lets dashboard users pick their own window." -msgstr "" +msgstr "``l2_name`` 标签的计数器 (``l2_store_completed`` 和 ``l2_load_completed``) 存在的目的是为了让仪表板能够通过 ``rate(lmcache_mp_l2_store_completed_total{l2_name=\\\"...\\\"}[1m])``（以及加载的等效方式）按需计算每个后端的 IOPS。没有单独导出 ``*_iops`` 指标；保留原始计数器让仪表板用户可以选择自己的时间窗口。" #: ../../source/mp/observability.rst:323 msgid "Failure & Health Counters" -msgstr "" +msgstr "失败与健康计数器" #: ../../source/mp/observability.rst:325 msgid "" @@ -667,11 +667,11 @@ msgid "" "metrics are enabled. All three counters carry ``model_name`` (extracted " "from each ``ObjectKey``) so operators can slice per-model on the " "Prometheus ``/metrics`` endpoint." -msgstr "" +msgstr "在专用的 ``lmcache_mp.health`` OTel 计量器上发出的健康监测计数器。由 ``L1FailureMetricsSubscriber`` 和 ``L2FailureMetricsSubscriber`` 驱动，这些订阅者在启用指标时会自动注册。所有三个计数器都携带 ``model_name``（从每个 ``ObjectKey`` 中提取），以便操作员可以在 Prometheus ``/metrics`` 端点上按模型进行切片。" #: ../../source/mp/observability.rst:339 msgid "``lmcache_mp.l1_allocation_failure``" -msgstr "" +msgstr "``lmcache_mp.l1_allocation_failure``" #: ../../source/mp/observability.rst:341 #, python-brace-format @@ -680,11 +680,11 @@ msgid "" "``during`` ∈ {``l1_store``, ``l2_prefetch``} to distinguish user-" "initiated stores from prefetch-triggered allocations, plus " "``model_name``." -msgstr "" +msgstr "在 ``reserve_write`` 期间发生 L1 内存分配失败（OOM）。通过 ``during`` ∈ {``l1_store``, ``l2_prefetch``} 标记，以区分用户发起的存储与预取触发的分配，以及 ``model_name``。" #: ../../source/mp/observability.rst:345 msgid "``lmcache_mp.l1_read_failure``" -msgstr "" +msgstr "``lmcache_mp.l1_read_failure``" #: ../../source/mp/observability.rst:347 #, python-brace-format @@ -695,11 +695,11 @@ msgid "" " — in MP mode ``reserve_read`` is only called after a successful lookup, " "so any non-zero value indicates a lookup/reserve race or unexpected " "eviction and should stay near zero in healthy operation." -msgstr "" +msgstr "L1 ``reserve_read`` 失败。标记为 ``during`` ∈ {``l2_store``, ``l1_retrieve``}，``reason`` ∈ {``not_found``, ``write_locked``}，加上 ``model_name``。**后查找异常计数器**，而不是缓存未命中计数器——在 MP 模式下，``reserve_read`` 仅在成功查找后调用，因此任何非零值都表示查找/保留竞争或意外逐出，健康运行时应保持接近零。" #: ../../source/mp/observability.rst:354 msgid "``lmcache_mp.l2_prefetch_failure``" -msgstr "" +msgstr "``lmcache_mp.l2_prefetch_failure``" #: ../../source/mp/observability.rst:356 #, python-brace-format @@ -709,7 +709,7 @@ msgid "" "``l1_oom`` means L1 had no room to receive the prefetched object; " "``not_found`` means the adapter returned no data despite a positive " "lookup (e.g. concurrent delete)." -msgstr "" +msgstr "L2 报告在查找时存在但未能进入 L1 的键。标记为 ``reason`` ∈ {``l1_oom``, ``not_found``} 以及 ``model_name``。``l1_oom`` 表示 L1 没有空间接收预取的对象；``not_found`` 表示适配器在查找成功的情况下未返回数据（例如并发删除）。" #: ../../source/mp/observability.rst:362 msgid "" @@ -717,18 +717,18 @@ msgid "" " as an additive, non-breaking extension once L2 adapters distinguish " "deserialization errors from missing objects — no dashboard migration " "needed when that lands." -msgstr "" +msgstr "一旦 L2 适配器区分反序列化错误和缺失对象，将会将 ``reason=serde_failure`` 值作为附加的、非破坏性的扩展添加到 ``l2_prefetch_failure`` 中——当这项功能上线时，无需进行仪表板迁移。" #: ../../source/mp/observability.rst:367 msgid "" "For the full design rationale (including which event types drive each " "counter and why ``lmcache_instance_id`` is deferred), see " "``docs/design/v1/mp_observability/METRICS.md`` in the source tree." -msgstr "" +msgstr "有关完整的设计原理（包括哪些事件类型驱动每个计数器以及为什么推迟 ``lmcache_instance_id``），请参阅源树中的 ``docs/design/v1/mp_observability/METRICS.md``。" #: ../../source/mp/observability.rst:372 msgid "Lookup Hit-Rate Metrics" -msgstr "" +msgstr "查找命中率指标" #: ../../source/mp/observability.rst:374 msgid "" @@ -736,38 +736,38 @@ msgid "" "by a lookup that were served from either L1 or L2. L0 (GPU prefix cache) " "is intentionally excluded — it is vLLM-owned and not observable from " "LMCache." -msgstr "" +msgstr "按令牌级别计数器，其比例表示由查找请求的令牌中，从 L1 或 L2 服务的令牌的比例。L0（GPU 前缀缓存）故意被排除在外——它是 vLLM 所有的，无法从 LMCache 中观察到。" #: ../../source/mp/observability.rst:385 msgid "``lmcache_mp.lookup_requested_tokens``" -msgstr "" +msgstr "``lmcache_mp.lookup_requested_tokens``" #: ../../source/mp/observability.rst:386 ../../source/mp/observability.rst:390 msgid "Counter (attrs: ``model_name``, ``cache_salt``)" -msgstr "" +msgstr "计数器（属性：``model_name``, ``cache_salt``）" #: ../../source/mp/observability.rst:387 msgid "" "Total tokens submitted for lookup (denominator of the L1+L2 token-level " "hit rate). Only chunk-aligned tokens are counted." -msgstr "" +msgstr "提交查找的总令牌数（L1+L2 令牌级命中率的分母）。仅计算与块对齐的令牌。" #: ../../source/mp/observability.rst:389 msgid "``lmcache_mp.lookup_hit_tokens``" -msgstr "" +msgstr "``lmcache_mp.lookup_hit_tokens``" #: ../../source/mp/observability.rst:391 msgid "" "Total tokens found in L1 or L2 during lookup (numerator of the L1+L2 " "token-level hit rate). Counts the contiguous prefix hit only." -msgstr "" +msgstr "在查找过程中在 L1 或 L2 中找到的总令牌数（L1+L2 令牌级命中率的分子）。仅计算连续前缀命中。" #: ../../source/mp/observability.rst:394 msgid "" "Both counters are driven by the same event (``MP_LOOKUP_PREFETCH_END``), " "so they always advance together per completed lookup. Early-exit lookups " "contribute ``0`` to both, and abandoned lookups contribute to neither." -msgstr "" +msgstr "这两个计数器由同一事件（``MP_LOOKUP_PREFETCH_END``）驱动，因此它们在每次完成查找时总是一起增加。提前退出的查找对两者都贡献 ``0``，而放弃的查找则对两者都没有贡献。" #: ../../source/mp/observability.rst:398 msgid "" @@ -776,15 +776,15 @@ msgid "" "per-tenant hit rate. ``cache_salt`` can be high-cardinality (one entry " "per tenant or isolation domain); drop it at scrape time with " "``metric_relabel_configs`` if storage cost matters." -msgstr "" +msgstr "``model_name`` 和 ``cache_salt`` 属性在查找时从 ``IPCCacheEngineKey`` 中捕获，以便仪表板可以计算每个模型或每个租户的命中率。``cache_salt`` 可能具有高基数（每个租户或隔离域一个条目）；如果存储成本重要，请在抓取时通过 ``metric_relabel_configs`` 丢弃它。" #: ../../source/mp/observability.rst:404 msgid "**PromQL for hit rate:**" -msgstr "" +msgstr "**PromQL 查询命中率:**" #: ../../source/mp/observability.rst:417 msgid "L0 (GPU) Block Lifecycle Histograms" -msgstr "" +msgstr "L0 (GPU) 块生命周期直方图" #: ../../source/mp/observability.rst:419 msgid "" @@ -793,7 +793,7 @@ msgid "" "(when a block is assigned different tokens). Sampling uses random " "selection with a ``_skipped`` set (bounded by the finite number of " "physical GPU blocks)." -msgstr "" +msgstr "通过 ``L0LifecycleSubscriber`` 采样（默认 1%）GPU KV Cache 块生命周期跟踪。在重新分配时（当块被分配不同的令牌时）检测逐出。采样使用随机选择，并带有一个 ``_skipped`` 集（受物理 GPU 块有限数量的限制）。" #: ../../source/mp/observability.rst:425 #, python-brace-format @@ -803,35 +803,35 @@ msgid "" "Prometheus (e.g. " "``lmcache_mp_l0_block_lifetime_seconds{instance_id=\"12345\",model_name" "=\"llama-7b\"}``)." -msgstr "" +msgstr "所有 L0 直方图都带有 ``instance_id`` 和 ``model_name`` OTel 属性，从而支持在 Prometheus 中按实例和按模型进行指标切片（例如 ``lmcache_mp_l0_block_lifetime_seconds{instance_id=\\\"12345\\\",model_name=\\\"llama-7b\\\"}``）。" #: ../../source/mp/observability.rst:437 msgid "``lmcache_mp.l0_block_lifetime_seconds``" -msgstr "" +msgstr "``lmcache_mp.l0_block_lifetime_seconds``" #: ../../source/mp/observability.rst:439 msgid "Time from allocation to eviction per sampled GPU block." -msgstr "" +msgstr "每个采样的 GPU 块从分配到逐出的时间。" #: ../../source/mp/observability.rst:440 msgid "``lmcache_mp.l0_block_idle_before_evict_seconds``" -msgstr "" +msgstr "``lmcache_mp.l0_block_idle_before_evict_seconds``" #: ../../source/mp/observability.rst:442 msgid "Time from last access to eviction per sampled GPU block." -msgstr "" +msgstr "从最后访问到逐出的时间（每个采样的 GPU 块）。" #: ../../source/mp/observability.rst:443 msgid "``lmcache_mp.l0_block_reuse_gap_seconds``" -msgstr "" +msgstr "``lmcache_mp.l0_block_reuse_gap_seconds``" #: ../../source/mp/observability.rst:445 msgid "Time gaps between consecutive accesses of the same GPU block." -msgstr "" +msgstr "同一 GPU 块的连续访问之间的时间间隔。" #: ../../source/mp/observability.rst:448 msgid "L0 ↔ L1 Throughput Histograms" -msgstr "" +msgstr "L0 ↔ L1 吞吐量直方图" #: ../../source/mp/observability.rst:450 #, python-brace-format @@ -843,7 +843,7 @@ msgid "" "``MP_{STORE,RETRIEVE}_{START,END}`` events published on the GPU cupy " "stream, so they reflect true GPU-stream copy time — not Python/lock " "overhead." -msgstr "" +msgstr "每个请求的 GPU↔CPU 复制吞吐量通过 ``L0L1ThroughputSubscriber`` 进行记录。每个存储/检索请求为相应的直方图贡献一个样本：``total_bytes / (end_ts - start_ts)``，单位为 GB/s。时间戳来自在 GPU cupy 流上发布的 ``MP_{STORE,RETRIEVE}_{START,END}`` 事件，因此它们反映了真实的 GPU 流复制时间——而不是 Python/锁的开销。" #: ../../source/mp/observability.rst:458 #, python-brace-format @@ -854,27 +854,27 @@ msgid "" "Prometheus (e.g. " "``lmcache_mp_l0_l1_store_throughput_gbs{engine_id=\"0\",device=\"cuda:3\",model_name" "=\"meta-llama/Llama-3.1-8B\"}``)." -msgstr "" +msgstr "所有吞吐量直方图都带有 ``engine_id`` (vLLM 工作实例 ID)、``device`` (例如 ``\"cuda:3\"``) 和 ``model_name`` OTel 属性，从而支持在 Prometheus 中按工作者、设备和模型进行切片 (例如 ``lmcache_mp_l0_l1_store_throughput_gbs{engine_id=\"0\",device=\"cuda:3\",model_name=\"meta-llama/Llama-3.1-8B\"}``)。" #: ../../source/mp/observability.rst:471 msgid "``lmcache_mp.l0_l1_store_throughput_gbs``" -msgstr "" +msgstr "``lmcache_mp.l0_l1_store_throughput_gbs``" #: ../../source/mp/observability.rst:473 msgid "GPU→CPU (L0→L1) store throughput in GB/s per request." -msgstr "" +msgstr "每个请求的 GPU→CPU (L0→L1) 存储吞吐量（单位：GB/s）。" #: ../../source/mp/observability.rst:474 msgid "``lmcache_mp.l0_l1_load_throughput_gbs``" -msgstr "" +msgstr "``lmcache_mp.l0_l1_load_throughput_gbs``" #: ../../source/mp/observability.rst:476 msgid "CPU→GPU (L1→L0) load throughput in GB/s per request." -msgstr "" +msgstr "每个请求的 CPU→GPU (L1→L0) 加载吞吐量（GB/s）。" #: ../../source/mp/observability.rst:479 msgid "L1 ↔ L2 Throughput Histograms" -msgstr "" +msgstr "L1 ↔ L2 吞吐量直方图" #: ../../source/mp/observability.rst:481 msgid "" @@ -885,7 +885,7 @@ msgid "" " ``(request_id, adapter_index)``; the request-level " "``L2_PREFETCH_LOAD_*`` events used by the key-count counters aggregate " "across adapters and cannot be attributed to a specific ``l2_name``." -msgstr "" +msgstr "每个任务的 L1↔L2 传输吞吐量通过 ``L2ThroughputSubscriber`` 进行监控。存储路径通过 ``(adapter_index, task_id)`` 关联 ``L2_STORE_SUBMITTED`` → ``L2_STORE_COMPLETED``。加载路径通过 ``(request_id, adapter_index)`` 关联每个适配器的 ``L2_LOAD_TASK_SUBMITTED`` → ``L2_LOAD_TASK_COMPLETED`` 事件；请求级别的 ``L2_PREFETCH_LOAD_*`` 事件用于键计数器在适配器之间聚合，无法归因于特定的 ``l2_name``。" #: ../../source/mp/observability.rst:490 msgid "" @@ -894,7 +894,7 @@ msgid "" " not raw transfer rate. Use these histograms to compare adapter types and" " catch regressions; use the L0↔L1 histograms when you need pure copy-time" " throughput." -msgstr "" +msgstr "时间戳跨越 **提交 → 完成**，因此持续时间包括适配器队列、网络和磁盘 I/O — 该值为 *字节 / 端到端延迟*，而不是原始传输速率。使用这些直方图来比较适配器类型并捕捉回归；当您需要纯粹的复制时间吞吐量时，请使用 L0↔L1 直方图。" #: ../../source/mp/observability.rst:496 #, python-brace-format @@ -903,27 +903,27 @@ msgid "" " — the registered adapter type (e.g. ``\"fs\"``, ``\"nixl_store\"``, " "``\"mooncake_store\"``) — enabling per-backend slicing in Prometheus " "(e.g. ``lmcache_mp_l2_store_throughput_gbs{l2_name=\"nixl_store\"}``)." -msgstr "" +msgstr "所有 L1↔L2 吞吐量直方图都携带一个 ``l2_name`` OTel 属性——注册的适配器类型（例如 ``\\\"fs\\\"``, ``\\\"nixl_store\\\"``, ``\\\"mooncake_store\\\"``）——使得在 Prometheus 中能够按后端进行切片（例如 ``lmcache_mp_l2_store_throughput_gbs{l2_name=\\\"nixl_store\\\"}``）。" #: ../../source/mp/observability.rst:508 msgid "``lmcache_mp.l2_store_throughput_gbs``" -msgstr "" +msgstr "``lmcache_mp.l2_store_throughput_gbs``" #: ../../source/mp/observability.rst:510 msgid "L1→L2 store throughput in GB/s per task." -msgstr "" +msgstr "每个任务的 L1→L2 存储吞吐量（GB/s）。" #: ../../source/mp/observability.rst:511 msgid "``lmcache_mp.l2_load_throughput_gbs``" -msgstr "" +msgstr "``lmcache_mp.l2_load_throughput_gbs``" #: ../../source/mp/observability.rst:513 msgid "L2→L1 load throughput in GB/s per (request, adapter) pair." -msgstr "" +msgstr "每对（请求，适配器）的 L2→L1 加载吞吐量（单位：GB/s）。" #: ../../source/mp/observability.rst:516 msgid "Engine Counters" -msgstr "" +msgstr "引擎计数器" #: ../../source/mp/observability.rst:518 msgid "" @@ -931,15 +931,15 @@ msgid "" "vLLM worker via ``retrieve()``. Labeled by ``worker_id`` (the vLLM " "worker instance id) — distinct from any scheduler-scoped id that may " "appear on other metrics." -msgstr "" +msgstr "与 MP 服务器通过 ``retrieve()`` 返回给每个 vLLM 工作线程相关的工作线程范围计数器。通过 ``worker_id``（vLLM 工作线程实例 ID）标记——与可能出现在其他指标上的任何调度器范围 ID 不同。" #: ../../source/mp/observability.rst:530 msgid "``lmcache_mp.num_chunks_loaded``" -msgstr "" +msgstr "``lmcache_mp.num_chunks_loaded``" #: ../../source/mp/observability.rst:531 msgid "Counter (attrs: ``worker_id``, ``model_name``, ``cache_salt``)" -msgstr "" +msgstr "计数器（属性：``worker_id``, ``model_name``, ``cache_salt``）" #: ../../source/mp/observability.rst:532 msgid "" @@ -948,69 +948,69 @@ msgid "" "tenant / isolation domain (``cache_salt``). ``cache_salt`` may be high-" "cardinality; drop it at scrape time with ``metric_relabel_configs`` if " "storage cost matters." -msgstr "" +msgstr "加载到引擎中的 LMCache 块的总数，按所有 ``retrieve()`` 完成的总和计算。可以按工作者、模型和租户/隔离域（``cache_salt``）进行切片。``cache_salt`` 可能具有高基数；如果存储成本很重要，请在抓取时使用 ``metric_relabel_configs`` 丢弃它。" #: ../../source/mp/observability.rst:539 msgid "Observable Gauges" -msgstr "" +msgstr "可观察的仪表盘" #: ../../source/mp/observability.rst:541 msgid "" "Point-in-time state snapshots registered via ``register_gauge`` (pull-" "based OTel observable gauges)." -msgstr "" +msgstr "通过 ``register_gauge`` 注册的时间点状态快照（基于拉取的 OTel 可观察量度）。" #: ../../source/mp/observability.rst:544 msgid "" "The three in-flight metrics carry two attributes that distinguish " "adapters even when more than one is registered with the same backend type" " — same shape as ``lmcache_mp.l2_store_completed``:" -msgstr "" +msgstr "这三个正在进行的指标携带两个属性，即使注册了多个相同后端类型的适配器，也能区分它们——与 ``lmcache_mp.l2_store_completed`` 具有相同形状：" #: ../../source/mp/observability.rst:548 msgid "" "``l2_name`` — the registered adapter type (e.g. ``\"fs\"``, " "``\"nixl_store\"``, ``\"mooncake_store\"``)." -msgstr "" +msgstr "``l2_name`` — 注册的适配器类型（例如 ``\"fs\"``, ``\"nixl_store\"``, ``\"mooncake_store\"``）。" #: ../../source/mp/observability.rst:550 msgid "``adapter_index`` — position in the controller's adapter list." -msgstr "" +msgstr "``adapter_index`` — 控制器适配器列表中的位置。" #: ../../source/mp/observability.rst:552 msgid "Adapters with no in-flight work emit no datapoint for that scrape." -msgstr "" +msgstr "没有正在进行的工作的适配器不会为该抓取发出数据点。" #: ../../source/mp/observability.rst:561 msgid "``lmcache_mp.active_prefetch_jobs``" -msgstr "" +msgstr "``lmcache_mp.active_prefetch_jobs``" #: ../../source/mp/observability.rst:562 ../../source/mp/observability.rst:566 #: ../../source/mp/observability.rst:571 ../../source/mp/observability.rst:620 #: ../../source/mp/observability.rst:624 msgid "ObservableGauge" -msgstr "" +msgstr "可观察仪表" #: ../../source/mp/observability.rst:563 msgid "" "Number of prefetch jobs currently in-flight. A sustained high value may " "indicate slow L2 backends or polling delays." -msgstr "" +msgstr "当前正在进行的预取作业数量。持续的高值可能表示 L2 后端缓慢或轮询延迟。" #: ../../source/mp/observability.rst:565 msgid "``lmcache_mp.l1_memory_usage_bytes``" -msgstr "" +msgstr "``lmcache_mp.l1_memory_usage_bytes``" #: ../../source/mp/observability.rst:567 msgid "" "Bytes currently held in L1. Rising without plateauing typically " "indicates a leak; saturating at the configured ``--l1-size-gb`` indicates" " working set exceeds capacity." -msgstr "" +msgstr "当前在 L1 中占用的字节数。持续上升而没有达到平稳状态通常表示存在泄漏；在配置的 ``--l1-size-gb`` 达到饱和时表示工作集超过了容量。" #: ../../source/mp/observability.rst:570 msgid "``lmcache_mp.l1_usage_ratio``" -msgstr "" +msgstr "``lmcache_mp.l1_usage_ratio``" #: ../../source/mp/observability.rst:572 msgid "" @@ -1020,37 +1020,37 @@ msgid "" "raises during a scrape. Compare against the eviction watermark (default " "``0.8``) to read whether the eviction loop is below or above its trigger " "threshold." -msgstr "" +msgstr "L1 使用/总比率 (``0.0``–``1.0``)，在抓取时从 ``L1Manager.get_memory_usage()`` 采样。当计量目标尚未连接或 ``total_bytes`` 为零时返回 ``0.0``，因此在抓取期间回调不会触发。与逐出水位线 (默认 ``0.8``) 进行比较，以判断逐出循环是否低于或高于触发阈值。" #: ../../source/mp/observability.rst:578 msgid "``lmcache_mp.num_inflight_l2_stores``" -msgstr "" +msgstr "``lmcache_mp.num_inflight_l2_stores``" #: ../../source/mp/observability.rst:579 ../../source/mp/observability.rst:584 #: ../../source/mp/observability.rst:589 msgid "ObservableGauge (attrs: ``l2_name``, ``adapter_index``)" -msgstr "" +msgstr "可观察的仪表 (属性: ``l2_name``, ``adapter_index``)" #: ../../source/mp/observability.rst:580 msgid "" "L2 store tasks currently executing, per adapter. Sustained non-zero " "values indicate the adapter cannot keep up with the L1 → L2 write rate." -msgstr "" +msgstr "每个适配器当前正在执行的 L2 存储任务。 持续的非零值表明适配器无法跟上 L1 → L2 写入速率。" #: ../../source/mp/observability.rst:583 msgid "``lmcache_mp.num_inflight_l2_loads``" -msgstr "" +msgstr "``lmcache_mp.num_inflight_l2_loads``" #: ../../source/mp/observability.rst:585 msgid "" "L2 → L1 prefetch load tasks currently executing, per adapter. Pair with " "``num_inflight_l2_stores`` to see whether read or write traffic dominates" " a given backend." -msgstr "" +msgstr "每个适配器当前正在执行的 L2 → L1 预取加载任务。与 ``num_inflight_l2_stores`` 配对，以查看给定后端是以读取流量还是写入流量为主。" #: ../../source/mp/observability.rst:588 msgid "``lmcache_mp.inflight_load_memory_usage_bytes``" -msgstr "" +msgstr "``lmcache_mp.inflight_load_memory_usage_bytes``" #: ../../source/mp/observability.rst:590 msgid "" @@ -1059,11 +1059,11 @@ msgid "" "signal that prefetch reservations are crowding out cacheable data. Per-" "adapter byte attribution follows each request's ``load_plan`` bitmap, so " "summing across adapters never double-counts." -msgstr "" +msgstr "每个适配器保留的 L1 字节，用于正在进行的 L2 → L1 预取加载。随着正在进行的字节和 ``l1_memory_usage_bytes`` 的增加，预取保留正在挤占可缓存数据。每个适配器的字节归属遵循每个请求的 ``load_plan`` 位图，因此跨适配器求和时不会重复计算。" #: ../../source/mp/observability.rst:598 msgid "EventBus Self-Monitoring" -msgstr "" +msgstr "事件总线自我监控" #: ../../source/mp/observability.rst:600 msgid "" @@ -1072,7 +1072,7 @@ msgid "" "meter. These metrics observe bus state directly via the ``EventBus`` " "accessors and report on every OTel scrape — they are not driven by " "events, so dropping or failing subscribers cannot silence them." -msgstr "" +msgstr "EventBus 本身的健康指标，由 ``EventBusSelfMetricsSubscriber`` 在 ``lmcache.event_bus`` OTel 计量器上注册。这些指标通过 ``EventBus`` 访问器直接观察总线状态，并在每次 OTel 抓取时报告——它们不是由事件驱动的，因此丢弃或失败的订阅者无法使其静默。" #: ../../source/mp/observability.rst:606 msgid "" @@ -1081,80 +1081,80 @@ msgid "" "zero ``dropped_events_total`` or a sustained non-zero " "``drain_lag_seconds`` indicates the bus is at ``--event-bus-queue-size`` " "and tail-dropping; raise that flag or investigate slow subscribers." -msgstr "" +msgstr "使用它们来回答：EventBus 是否能够跟上发布者，是否有任何事件被丢弃，以及是否有任何订阅者回调抛出异常？非零的 ``dropped_events_total`` 或持续非零的 ``drain_lag_seconds`` 表明总线处于 ``--event-bus-queue-size`` 并且正在尾部丢弃；请提高该标志或调查慢速订阅者。" #: ../../source/mp/observability.rst:619 msgid "``lmcache_mp.event_bus.queue_depth``" -msgstr "" +msgstr "``lmcache_mp.event_bus.queue_depth``" #: ../../source/mp/observability.rst:621 msgid "Events currently queued in the EventBus (``len(_queue)`` at scrape time)." -msgstr "" +msgstr "事件总线中当前排队的事件（在抓取时的 ``len(_queue)``）。" #: ../../source/mp/observability.rst:623 msgid "``lmcache_mp.event_bus.drain_lag_seconds``" -msgstr "" +msgstr "``lmcache_mp.event_bus.drain_lag_seconds``" #: ../../source/mp/observability.rst:625 msgid "" "Seconds since the oldest queued event was published; ``0.0`` when empty." " Rising values mean the drain thread is falling behind." -msgstr "" +msgstr "自最旧的排队事件发布以来的秒数；当为空时为 ``0.0``。值上升意味着排出线程落后。" #: ../../source/mp/observability.rst:628 msgid "``lmcache_mp.event_bus.dropped_events_total``" -msgstr "" +msgstr "``lmcache_mp.event_bus.dropped_events_total``" #: ../../source/mp/observability.rst:629 msgid "ObservableCounter" -msgstr "" +msgstr "可观察计数器" #: ../../source/mp/observability.rst:630 msgid "" "Cumulative events dropped because the EventBus queue was at ``--event-" "bus-queue-size``." -msgstr "" +msgstr "由于 EventBus 队列达到 ``--event-bus-queue-size``，累计丢弃的事件。" #: ../../source/mp/observability.rst:632 msgid "``lmcache_mp.event_bus.subscriber_exceptions``" -msgstr "" +msgstr "``lmcache_mp.event_bus.subscriber_exceptions``" #: ../../source/mp/observability.rst:633 msgid "ObservableCounter (attr: ``subscriber_name``)" -msgstr "" +msgstr "可观察计数器 (属性: ``subscriber_name``)" #: ../../source/mp/observability.rst:634 msgid "" "Cumulative exceptions raised by subscriber callbacks during EventBus " "dispatch, tagged by ``subscriber_name`` (the failing callback's owning " "class for bound methods, or ``__qualname__`` for free functions)." -msgstr "" +msgstr "由 EventBus 分发期间由订阅者回调引发的累计异常，按 ``subscriber_name`` 标记（对于绑定方法为失败回调的拥有类，对于自由函数为 ``__qualname__``）。" #: ../../source/mp/observability.rst:639 msgid "" "For the full design rationale and the in-process accessors that back each" " metric see ``docs/design/v1/mp_observability/METRICS.md`` and " "``docs/design/v1/mp_observability/event-bus.md`` in the source tree." -msgstr "" +msgstr "有关完整的设计原理及支持每个指标的进程内访问器，请参见源代码树中的 ``docs/design/v1/mp_observability/METRICS.md`` 和 ``docs/design/v1/mp_observability/event-bus.md``。" #: ../../source/mp/observability.rst:644 msgid "Prometheus Scrape Configuration" -msgstr "" +msgstr "Prometheus 抓取配置" #: ../../source/mp/observability.rst:646 msgid "Add the LMCache server as a Prometheus scrape target:" -msgstr "" +msgstr "将 LMCache 服务器添加为 Prometheus 抓取目标：" #: ../../source/mp/observability.rst:656 msgid "Logging" -msgstr "" +msgstr "日志记录" #: ../../source/mp/observability.rst:658 msgid "" "Logging subscribers emit debug-level messages for store, retrieve, " "lookup, L1, and StorageManager events via Python's standard ``logging`` " "module." -msgstr "" +msgstr "日志订阅者通过 Python 的标准 ``logging`` 模块为存储、检索、查找、L1 和 StorageManager 事件发出调试级别的消息。" #: ../../source/mp/observability.rst:661 msgid "" @@ -1162,95 +1162,99 @@ msgid "" "an OTel ``LoggingHandler`` so that log records are forwarded to any " "configured OTel ``LoggerProvider``. The handler respects the " "``LMCACHE_LOG_LEVEL`` environment variable." -msgstr "" +msgstr "当安装了 OpenTelemetry 时，``init_logger`` 会自动附加一个 OTel ``LoggingHandler``，以便将日志记录转发到任何配置的 OTel ``LoggerProvider``。该处理程序遵循 ``LMCACHE_LOG_LEVEL`` 环境变量。" #: ../../source/mp/observability.rst:670 msgid "Key log messages:" -msgstr "" +msgstr "关键日志消息：" #: ../../source/mp/observability.rst:676 msgid "Level" -msgstr "" +msgstr "级别" #: ../../source/mp/observability.rst:677 msgid "Message" -msgstr "" +msgstr "消息" #: ../../source/mp/observability.rst:678 ../../source/mp/observability.rst:680 #: ../../source/mp/observability.rst:682 msgid "INFO" -msgstr "" +msgstr "信息" #: ../../source/mp/observability.rst:679 msgid "``Stored N tokens in X seconds``" -msgstr "" +msgstr "``在 X 秒内存储了 N 个令牌``" #: ../../source/mp/observability.rst:681 msgid "``Retrieved N tokens in X seconds``" msgstr "" +"```\n" +"在 X 秒内检索到 N 个令牌``" #: ../../source/mp/observability.rst:683 msgid "``Prefetch request completed (L1+L2): N/M prefix hits``" msgstr "" +"```\n" +"预取请求完成 (L1+L2): N/M 前缀命中``" #: ../../source/mp/observability.rst:684 ../../source/mp/observability.rst:686 msgid "DEBUG" -msgstr "" +msgstr "调试" #: ../../source/mp/observability.rst:685 msgid "``MP store start: session=... device=...``" -msgstr "" +msgstr "``MP store start: session=... device=...``" #: ../../source/mp/observability.rst:687 msgid "``MP retrieve end: session=... retrieved_count=...``" -msgstr "" +msgstr "``MP retrieve end: session=... retrieved_count=...``" #: ../../source/mp/observability.rst:690 msgid "Tracing" -msgstr "" +msgstr "追踪" #: ../../source/mp/observability.rst:694 msgid "" "``--enable-tracing`` **requires** ``--otlp-endpoint`` to be set. The " "server will refuse to start if tracing is enabled without an OTLP " "endpoint, since there is no local fallback for trace export." -msgstr "" +msgstr "``--enable-tracing`` **要求** 设置 ``--otlp-endpoint``。如果在没有 OTLP 端点的情况下启用追踪，服务器将拒绝启动，因为没有本地回退用于追踪导出。" #: ../../source/mp/observability.rst:698 msgid "" "When tracing is enabled (``--enable-tracing --otlp-endpoint ``), the" " tracing subscriber creates OTel spans from START/END event pairs:" -msgstr "" +msgstr "当启用追踪时（``--enable-tracing --otlp-endpoint ``），追踪订阅者会从 START/END 事件对创建 OTel spans：" #: ../../source/mp/observability.rst:701 msgid "``mp.store`` — from ``MP_STORE_START`` to ``MP_STORE_END``" -msgstr "" +msgstr "``mp.store`` — 从 ``MP_STORE_START`` 到 ``MP_STORE_END``" #: ../../source/mp/observability.rst:702 msgid "``mp.retrieve`` — from ``MP_RETRIEVE_START`` to ``MP_RETRIEVE_END``" -msgstr "" +msgstr "``mp.retrieve`` — from ``MP_RETRIEVE_START`` to ``MP_RETRIEVE_END``" #: ../../source/mp/observability.rst:703 msgid "" "``mp.lookup_prefetch`` — from ``MP_LOOKUP_PREFETCH_START`` to " "``MP_LOOKUP_PREFETCH_END``" -msgstr "" +msgstr "``mp.lookup_prefetch`` — from ``MP_LOOKUP_PREFETCH_START`` to ``MP_LOOKUP_PREFETCH_END``" #: ../../source/mp/observability.rst:705 msgid "" "Each span carries event metadata as span attributes (e.g. ``device``, " "``stored_count``, ``found_count``)." -msgstr "" +msgstr "每个跨度携带事件元数据作为跨度属性（例如 ``device``、``stored_count``、``found_count``）。" #: ../../source/mp/observability.rst:708 msgid "" "View traces in any OTel-compatible backend such as **Jaeger** or " "**Grafana Tempo**." -msgstr "" +msgstr "在任何 OTel 兼容的后端中查看追踪，例如 **Jaeger** 或 **Grafana Tempo**。" #: ../../source/mp/observability.rst:724 msgid "Per-Request Hit-Rate Attributes" -msgstr "" +msgstr "每个请求的命中率属性" #: ../../source/mp/observability.rst:726 msgid "" @@ -1259,46 +1263,46 @@ msgid "" "all child spans (``mp.store``, ``mp.retrieve``, ``mp.lookup_prefetch``) " "beneath it. When the lookup phase ends, the root span is annotated with " "three OTel attributes that summarise the request-level cache hit rate:" -msgstr "" +msgstr "每个会话都被包装在一个按请求的根跨度中——标准 MP 路径的 ``request`` 和 CacheBlend 路径的 ``cb.request``——它将所有子跨度（``mp.store``、``mp.retrieve``、``mp.lookup_prefetch``）嵌套在其下。当查找阶段结束时，根跨度会用三个 OTel 属性进行注释，这些属性总结了请求级别的缓存命中率：" #: ../../source/mp/observability.rst:737 msgid "OTel type" -msgstr "" +msgstr "OTel 类型" #: ../../source/mp/observability.rst:739 msgid "``hit_tokens``" -msgstr "" +msgstr "``hit_tokens``" #: ../../source/mp/observability.rst:740 ../../source/mp/observability.rst:743 msgid "``int``" -msgstr "" +msgstr "``int``" #: ../../source/mp/observability.rst:741 msgid "Tokens served from L1+L2 (numerator)." -msgstr "" +msgstr "从 L1+L2 服务的令牌（分子）。" #: ../../source/mp/observability.rst:742 msgid "``requested_tokens``" -msgstr "" +msgstr "``requested_tokens``" #: ../../source/mp/observability.rst:744 msgid "Chunk-aligned tokens submitted for lookup (denominator)." -msgstr "" +msgstr "用于查找的块对齐令牌（分母）。" #: ../../source/mp/observability.rst:745 msgid "``hit_rate``" -msgstr "" +msgstr "``hit_rate``" #: ../../source/mp/observability.rst:746 msgid "``float``" -msgstr "" +msgstr "``float``" #: ../../source/mp/observability.rst:747 msgid "" "``hit_tokens / requested_tokens``; ``0.0`` when the denominator is zero." " Stored as a precomputed float because trace UIs (Tempo, Jaeger) cannot " "derive it from two integer attributes at query time." -msgstr "" +msgstr "``hit_tokens / requested_tokens``; ``0.0`` 当分母为零时。存储为预计算的浮点数，因为跟踪用户界面（Tempo, Jaeger）无法在查询时从两个整数属性中推导出它。" #: ../../source/mp/observability.rst:751 msgid "" @@ -1307,22 +1311,22 @@ msgid "" "root span is still open. **Store-only requests** that never call " "``lookup_prefetch_start()`` emit no end event for the lookup phase, so " "their root span will not carry these attributes." -msgstr "" +msgstr "属性在处理 ``MP_LOOKUP_PREFETCH_END``（标准 MP 路径）或 ``CB_LOOKUP_END``（CacheBlend 路径）时被写入——当根跨度仍然打开时。**仅存储请求**在未调用 ``lookup_prefetch_start()`` 时不会为查找阶段发出结束事件，因此它们的根跨度将不会携带这些属性。" #: ../../source/mp/observability.rst:757 msgid "Example TraceQL queries (Grafana Tempo):" -msgstr "" +msgstr "示例 TraceQL 查询 (Grafana Tempo):" #: ../../source/mp/observability.rst:770 msgid "" "For the full event-to-span mapping and the registry pattern that links " "child spans back to the root see ``docs/design/observability/request-" "event-span.md`` in the source tree." -msgstr "" +msgstr "有关完整的事件到跨度映射以及将子跨度链接回根的注册模式，请参见源树中的 ``docs/design/observability/request-event-span.md``。" #: ../../source/mp/observability.rst:777 msgid "Trace Recording" -msgstr "" +msgstr "追踪记录" #: ../../source/mp/observability.rst:781 msgid "" @@ -1333,7 +1337,7 @@ msgid "" "(eventually) without a GPU. ``--enable-tracing`` exports live OTel spans " "to an OTLP endpoint for online observability. The two features are " "independent and can be used together." -msgstr "" +msgstr "跟踪记录与 ``--enable-tracing``（OTel spans）是 **不同的**。跟踪记录捕获每个对二进制文件的 ``StorageManager`` 公共 API 调用，以便可以在后续进行测试、回归查找和基准测试时 **重放** 相同的工作负载——无需 vLLM，并且最终无需 GPU。``--enable-tracing`` 将实时 OTel spans 导出到 OTLP 端点以进行在线可观察性。这两个功能是独立的，可以一起使用。" #: ../../source/mp/observability.rst:789 #, python-brace-format @@ -1342,96 +1346,96 @@ msgid "" "``StorageManager.{reserve_write, finish_write, submit_prefetch_task, " "read_prefetched_results, finish_read_prefetched}`` to a binary file for " "later replay." -msgstr "" +msgstr "当设置 ``--trace-level storage`` 时，LMCache 会将每次对 ``StorageManager.{reserve_write, finish_write, submit_prefetch_task, read_prefetched_results, finish_read_prefetched}`` 的调用记录到一个二进制文件中，以便后续重放。" #: ../../source/mp/observability.rst:794 msgid "" "Recording is **off by default** and adds near-zero overhead when off (a " "single boolean check per ``StorageManager`` call). When on, recording " "happens on the EventBus drain thread, off the request path." -msgstr "" +msgstr "记录默认是**关闭的**，关闭时几乎没有开销（每个``StorageManager``调用只需进行一次布尔检查）。开启时，记录发生在 EventBus 排出线程上，而不在请求路径上。" #: ../../source/mp/observability.rst:799 msgid "Capturing a trace" -msgstr "" +msgstr "捕获跟踪" #: ../../source/mp/observability.rst:801 msgid "With an explicit output path:" -msgstr "" +msgstr "带有显式输出路径：" #: ../../source/mp/observability.rst:809 msgid "With an implicit timestamped output path under ``$TMPDIR``:" -msgstr "" +msgstr "在 ``$TMPDIR`` 下使用隐式时间戳输出路径：" #: ../../source/mp/observability.rst:820 msgid "" "The trace file is closed cleanly on shutdown (SIGTERM is handled by the " "EventBus stop path)." -msgstr "" +msgstr "在关闭时（SIGTERM 由 EventBus 停止路径处理），跟踪文件会被干净地关闭。" #: ../../source/mp/observability.rst:824 msgid "Replay" -msgstr "" +msgstr "重放" #: ../../source/mp/observability.rst:826 msgid "" "Replaying a recorded trace, plus the full set of CLI flags for driving, " "monitoring, and exporting replay results, is covered in its own page: " ":doc:`tracing_and_debugging`." -msgstr "" +msgstr "重放记录的追踪，以及用于驱动、监控和导出重放结果的完整 CLI 标志集，详见其独立页面： :doc:`tracing_and_debugging`。" #: ../../source/mp/observability.rst:831 msgid "What is captured (and what is not)" -msgstr "" +msgstr "捕获了什么（以及未捕获的内容）" #: ../../source/mp/observability.rst:833 msgid "**Captured:**" -msgstr "" +msgstr "**捕获：**" #: ../../source/mp/observability.rst:835 msgid "The fully-qualified name of every decorated ``StorageManager`` call." -msgstr "" +msgstr "每个被装饰的 ``StorageManager`` 调用的完全限定名称。" #: ../../source/mp/observability.rst:836 msgid "" "Each call's input arguments (e.g. ``keys``, ``layout_desc``, ``mode``, " "``extra_count``, ``external_request_id``)." -msgstr "" +msgstr "每个调用的输入参数（例如 ``keys``, ``layout_desc``, ``mode``, ``extra_count``, ``external_request_id``）。" #: ../../source/mp/observability.rst:838 msgid "Wall-clock and monotonic timestamps of each call." -msgstr "" +msgstr "每个调用的墙钟时间和单调时间戳。" #: ../../source/mp/observability.rst:839 msgid "" "A header carrying a trace schema version, start times, and a SHA-256 " "digest of the active ``StorageManagerConfig`` so replay can detect " "mismatched configurations." -msgstr "" +msgstr "一个包含跟踪模式版本、开始时间和活动的 ``StorageManagerConfig`` 的 SHA-256 摘要的头部，以便重放可以检测到不匹配的配置。" #: ../../source/mp/observability.rst:843 msgid "**Not captured:**" -msgstr "" +msgstr "**未捕获：**" #: ../../source/mp/observability.rst:845 msgid "" "KV tensor bytes. Replay exercises bookkeeping and controller logic; " "payloads at replay time are zeros." -msgstr "" +msgstr "KV 张量字节。重放练习的记账和控制逻辑；重放时的有效载荷为零。" #: ../../source/mp/observability.rst:847 msgid "" "Calls inside the ``MPCacheEngine``, the message queue, or any GPU-copy " "code. These layers are **out of scope** for the storage trace level." -msgstr "" +msgstr "在 ``MPCacheEngine``、消息队列或任何 GPU 复制代码中的调用。这些层级在存储跟踪级别中是 **超出范围** 的。" #: ../../source/mp/observability.rst:852 msgid "File format" -msgstr "" +msgstr "文件格式" #: ../../source/mp/observability.rst:854 msgid "A length-prefixed `msgpack `_ stream:" -msgstr "" +msgstr "一个长度前缀的 `msgpack `_ 流：" #: ../../source/mp/observability.rst:863 msgid "" @@ -1440,7 +1444,7 @@ msgid "" "timestamps, and the StorageManagerConfig digest. Each ``Record`` carries " "a relative timestamp, a wall-clock timestamp, the fully-qualified call " "site (``qualname``), and an argument dict." -msgstr "" +msgstr "``Header`` 包含一个魔术前缀 (``LMCT``)、一个格式版本、跟踪级别 (今天是 ``storage``)、一个跟踪模式版本、开始时间戳和 StorageManagerConfig 摘要。每个 ``Record`` 包含一个相对时间戳、一个墙钟时间戳、完全限定的调用位置 (``qualname``) 和一个参数字典。" #: ../../source/mp/observability.rst:869 msgid "" @@ -1448,11 +1452,11 @@ msgid "" "``gpu``) will share this layout and use the ``level`` header field to " "discriminate. Additional captured ops add new ``qualname`` strings " "without bumping the format version." -msgstr "" +msgstr "该格式故意具有可扩展性：未来的跟踪 **级别**（``mq``，``gpu``）将共享此布局，并使用 ``level`` 头字段进行区分。额外捕获的操作会添加新的 ``qualname`` 字符串，而不会提升格式版本。" #: ../../source/mp/observability.rst:874 msgid "" "For the full design rationale see " "``docs/design/v1/mp_observability/trace.md`` in the source tree." -msgstr "" +msgstr "有关完整的设计原理，请参见源代码树中的 ``docs/design/v1/mp_observability/trace.md``。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/operator.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/operator.po index c3206ca058e..0d8b5bab52c 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/operator.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/operator.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/mp/operator.rst:2 msgid "Kubernetes Operator" -msgstr "" +msgstr "Kubernetes 操作器" #: ../../source/mp/operator.rst:4 msgid "" @@ -30,17 +30,17 @@ msgid "" "DaemonSets, Services, and ConfigMaps (as described in the manual " ":doc:`deployment` guide), you declare a single ``LMCacheEngine`` custom " "resource and the operator reconciles all underlying Kubernetes objects." -msgstr "" +msgstr "LMCache Kubernetes 操作符自动化了 LMCache 多进程服务器的部署和生命周期管理。您只需声明一个 ``LMCacheEngine`` 自定义资源，操作符将协调所有底层的 Kubernetes 对象，而无需手动编写 DaemonSets、Services 和 ConfigMaps（如手册 :doc:`deployment` 指南中所述）。" #: ../../source/mp/operator.rst:15 msgid "Why Use the Operator" -msgstr "" +msgstr "为什么使用操作器" #: ../../source/mp/operator.rst:17 msgid "" "The manual DaemonSet approach works, but it has sharp edges the operator " "eliminates:" -msgstr "" +msgstr "手动 DaemonSet 方法可以工作，但它有一些操作员消除的尖锐边缘：" #: ../../source/mp/operator.rst:20 msgid "" @@ -48,7 +48,7 @@ msgid "" "true`` and ``--host 0.0.0.0``. Forgetting ``hostIPC`` in a hand-written " "manifest causes silent CUDA IPC failures " "(``cudaErrorMapBufferObjectFailed``) that are hard to debug." -msgstr "" +msgstr "**自动注入的 Pod 设置** -- 操作器始终设置 ``hostIPC: true`` 和 ``--host 0.0.0.0``。在手动编写的清单中忘记 ``hostIPC`` 会导致难以调试的静默 CUDA IPC 失败 (``cudaErrorMapBufferObjectFailed``)。" #: ../../source/mp/operator.rst:24 msgid "" @@ -56,14 +56,14 @@ msgid "" "Service with ``internalTrafficPolicy=Local`` and a connection ConfigMap " "that vLLM pods simply mount. No ``hostNetwork``, no Downward API, no " "shell variable substitution." -msgstr "" +msgstr "**节点本地服务发现** -- 操作员创建一个 ClusterIP 服务，设置 ``internalTrafficPolicy=Local``，并创建一个连接的 ConfigMap，vLLM pod 只需挂载它。没有 ``hostNetwork``，没有向下 API，没有 shell 变量替换。" #: ../../source/mp/operator.rst:28 msgid "" "**Auto-computed resource sizing** -- Memory requests and limits are " "derived from ``l1.sizeGB``, avoiding OOM kills (under-provisioned) or " "wasted node capacity (over-provisioned)." -msgstr "" +msgstr "**自动计算资源大小** -- 内存请求和限制来自 ``l1.sizeGB``，避免了 OOM 杀死（资源不足）或浪费节点容量（资源过剩）。" #: ../../source/mp/operator.rst:31 msgid "" @@ -71,145 +71,145 @@ msgid "" "``prometheus.serviceMonitor.enabled: true`` and the operator creates a " "``ServiceMonitor`` CR that the Prometheus Operator discovers " "automatically." -msgstr "" +msgstr "**声明式 Prometheus 集成** -- 设置 ``prometheus.serviceMonitor.enabled: true``，操作符会创建一个 ``ServiceMonitor`` CR，Prometheus 操作符会自动发现它。" #: ../../source/mp/operator.rst:34 msgid "" "**CRD validation** -- OpenAPI schema validation catches misconfigurations" " (e.g., ``l1.sizeGB <= 0``, invalid port range) at ``kubectl apply`` " "time, before any pods are created." -msgstr "" +msgstr "**CRD 验证** -- OpenAPI 架构验证在 ``kubectl apply`` 时捕获配置错误（例如，``l1.sizeGB <= 0``，无效的端口范围），在创建任何 pod 之前。" #: ../../source/mp/operator.rst:39 msgid "Prerequisites" -msgstr "" +msgstr "先决条件" #: ../../source/mp/operator.rst:41 msgid "Kubernetes 1.20+" -msgstr "" +msgstr "Kubernetes 1.20+" #: ../../source/mp/operator.rst:42 msgid "``kubectl`` configured to access your cluster" -msgstr "" +msgstr "``kubectl`` 配置为访问您的集群" #: ../../source/mp/operator.rst:43 msgid "" "(Optional) `Prometheus Operator `_ for ServiceMonitor support" -msgstr "" +msgstr "(可选) `Prometheus Operator `_ 以支持 ServiceMonitor" #: ../../source/mp/operator.rst:47 msgid "Installing the Operator" -msgstr "" +msgstr "安装 Operator" #: ../../source/mp/operator.rst:49 msgid "**Option A: One-line install from release (recommended)**" -msgstr "" +msgstr "**选项 A：从发布版一行安装（推荐）**" #: ../../source/mp/operator.rst:59 msgid "**Option B: Build from source**" -msgstr "" +msgstr "**选项 B：从源代码构建**" #: ../../source/mp/operator.rst:69 msgid "Deploying an LMCacheEngine" -msgstr "" +msgstr "部署 LMCacheEngine" #: ../../source/mp/operator.rst:71 msgid "A minimal CR deploys a DaemonSet with 60 GB L1 cache on every GPU node:" -msgstr "" +msgstr "一个最小的 CR 在每个 GPU 节点上部署一个具有 60 GB L1 缓存的 DaemonSet：" #: ../../source/mp/operator.rst:87 msgid "The operator automatically:" -msgstr "" +msgstr "操作符自动：" #: ../../source/mp/operator.rst:89 msgid "Creates a DaemonSet running one LMCache server pod per matched node" -msgstr "" +msgstr "在每个匹配的节点上创建一个运行 LMCache 服务器 Pod 的 DaemonSet" #: ../../source/mp/operator.rst:90 msgid "Sets ``hostIPC: true`` and passes ``--host 0.0.0.0`` to the server" -msgstr "" +msgstr "设置 ``hostIPC: true`` 并将 ``--host 0.0.0.0`` 传递给服务器" #: ../../source/mp/operator.rst:91 msgid "Creates a node-local ClusterIP Service for vLLM discovery" -msgstr "" +msgstr "为 vLLM 发现创建一个节点本地的 ClusterIP 服务" #: ../../source/mp/operator.rst:92 msgid "" "Creates a connection ConfigMap (``my-cache-connection``) with the ``kv-" "transfer-config`` JSON that vLLM needs" -msgstr "" +msgstr "创建一个连接 ConfigMap (``my-cache-connection``)，其中包含 vLLM 所需的 ``kv-transfer-config`` JSON。" #: ../../source/mp/operator.rst:94 msgid "Auto-computes resource requests/limits from the L1 cache size" -msgstr "" +msgstr "自动计算 L1 缓存大小的资源请求/限制" #: ../../source/mp/operator.rst:95 msgid "Defaults ``nodeSelector`` to ``nvidia.com/gpu.present: \"true\"``" -msgstr "" +msgstr "默认将 ``nodeSelector`` 设置为 ``nvidia.com/gpu.present: \\\"true\\\"``" #: ../../source/mp/operator.rst:98 msgid "" "The operator defaults the container image to ``lmcache/vllm-" "openai:latest``. Override with ``spec.image.repository`` and " "``spec.image.tag`` to pin a specific version." -msgstr "" +msgstr "该操作符将容器镜像默认设置为 ``lmcache/vllm-openai:latest``。可以通过 ``spec.image.repository`` 和 ``spec.image.tag`` 来覆盖，以固定特定版本。" #: ../../source/mp/operator.rst:103 msgid "Connecting vLLM" -msgstr "" +msgstr "连接 vLLM" #: ../../source/mp/operator.rst:105 msgid "" "The operator creates a ConfigMap named ``-connection`` " "containing the ``kv-transfer-config`` JSON. Mount it in your vLLM " "Deployment:" -msgstr "" +msgstr "操作符创建一个名为 ``-connection`` 的 ConfigMap，其中包含 ``kv-transfer-config`` JSON。将其挂载到您的 vLLM 部署中：" #: ../../source/mp/operator.rst:156 msgid "Key requirements for vLLM pods:" -msgstr "" +msgstr "vLLM Pod 的关键要求：" #: ../../source/mp/operator.rst:158 msgid "" "**hostIPC: true** -- CUDA IPC (``cudaIpcOpenMemHandle``) needs a shared " "IPC namespace between vLLM and LMCache." -msgstr "" +msgstr "**hostIPC: true** -- CUDA IPC (``cudaIpcOpenMemHandle``) 需要在 vLLM 和 LMCache 之间共享 IPC 命名空间。" #: ../../source/mp/operator.rst:160 msgid "" "**PYTHONHASHSEED=0** -- Ensures deterministic token hashing so vLLM and " "LMCache produce consistent cache keys." -msgstr "" +msgstr "**PYTHONHASHSEED=0** -- 确保确定性令牌哈希，以便 vLLM 和 LMCache 生成一致的缓存键。" #: ../../source/mp/operator.rst:162 msgid "" "**ConfigMap mount** -- The ``$(cat ...)`` pattern reads the connection " "JSON inline. The ConfigMap name is always ``-connection``." -msgstr "" +msgstr "**ConfigMap挂载** -- ``$(cat ...)`` 模式在线读取连接 JSON。ConfigMap 名称始终为 ``-connection``。" #: ../../source/mp/operator.rst:164 msgid "" "**No hostNetwork needed** -- The operator's node-local Service handles " "routing via ``internalTrafficPolicy=Local``." -msgstr "" +msgstr "**不需要 hostNetwork** -- 操作员的节点本地服务通过 ``internalTrafficPolicy=Local`` 处理路由。" #: ../../source/mp/operator.rst:168 msgid "Verifying the Deployment" -msgstr "" +msgstr "验证部署" #: ../../source/mp/operator.rst:175 msgid "Expected output:" -msgstr "" +msgstr "预期输出：" #: ../../source/mp/operator.rst:194 msgid "CRD Spec Reference" -msgstr "" +msgstr "CRD 规格参考" #: ../../source/mp/operator.rst:197 msgid "Image" -msgstr "" +msgstr "图像" #: ../../source/mp/operator.rst:203 ../../source/mp/operator.rst:226 #: ../../source/mp/operator.rst:249 ../../source/mp/operator.rst:263 @@ -217,61 +217,61 @@ msgstr "" #: ../../source/mp/operator.rst:324 ../../source/mp/operator.rst:347 #: ../../source/mp/operator.rst:478 msgid "Field" -msgstr "" +msgstr "字段" #: ../../source/mp/operator.rst:204 ../../source/mp/operator.rst:227 #: ../../source/mp/operator.rst:250 ../../source/mp/operator.rst:264 #: ../../source/mp/operator.rst:284 ../../source/mp/operator.rst:310 #: ../../source/mp/operator.rst:325 ../../source/mp/operator.rst:348 msgid "Default" -msgstr "" +msgstr "默认" #: ../../source/mp/operator.rst:205 ../../source/mp/operator.rst:228 #: ../../source/mp/operator.rst:251 ../../source/mp/operator.rst:265 #: ../../source/mp/operator.rst:285 ../../source/mp/operator.rst:311 #: ../../source/mp/operator.rst:326 ../../source/mp/operator.rst:349 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/mp/operator.rst:206 msgid "``image.repository``" -msgstr "" +msgstr "``image.repository``" #: ../../source/mp/operator.rst:207 msgid "``lmcache/vllm-openai``" -msgstr "" +msgstr "``lmcache/vllm-openai``" #: ../../source/mp/operator.rst:208 msgid "Container image repository." -msgstr "" +msgstr "容器镜像仓库。" #: ../../source/mp/operator.rst:209 msgid "``image.tag``" -msgstr "" +msgstr "``image.tag``" #: ../../source/mp/operator.rst:210 msgid "``latest``" -msgstr "" +msgstr "``latest``" #: ../../source/mp/operator.rst:211 msgid "Container image tag." -msgstr "" +msgstr "容器镜像标签。" #: ../../source/mp/operator.rst:212 msgid "``image.pullPolicy``" -msgstr "" +msgstr "``image.pullPolicy``" #: ../../source/mp/operator.rst:213 msgid "``IfNotPresent``" -msgstr "" +msgstr "``如果不存在``" #: ../../source/mp/operator.rst:214 msgid "``Always``, ``Never``, or ``IfNotPresent``." -msgstr "" +msgstr "``Always``, ``Never``, 或 ``IfNotPresent``。" #: ../../source/mp/operator.rst:215 msgid "``imagePullSecrets``" -msgstr "" +msgstr "``imagePullSecrets``" #: ../../source/mp/operator.rst:216 ../../source/mp/operator.rst:299 #: ../../source/mp/operator.rst:313 ../../source/mp/operator.rst:331 @@ -281,369 +281,369 @@ msgstr "" #: ../../source/mp/operator.rst:366 ../../source/mp/operator.rst:369 #: ../../source/mp/operator.rst:372 ../../source/mp/operator.rst:375 msgid "--" -msgstr "" +msgstr "--" #: ../../source/mp/operator.rst:217 msgid "Image pull secret references." -msgstr "" +msgstr "镜像拉取密钥引用。" #: ../../source/mp/operator.rst:220 msgid "Server" -msgstr "" +msgstr "服务器" #: ../../source/mp/operator.rst:229 ../../source/mp/operator.rst:488 msgid "``server.port``" -msgstr "" +msgstr "``server.port``" #: ../../source/mp/operator.rst:230 msgid "``5555``" -msgstr "" +msgstr "``5555``" #: ../../source/mp/operator.rst:231 msgid "ZMQ listening port (1024--65535)." -msgstr "" +msgstr "ZMQ 监听端口 (1024--65535)。" #: ../../source/mp/operator.rst:232 msgid "``server.chunkSize``" -msgstr "" +msgstr "``server.chunkSize``" #: ../../source/mp/operator.rst:233 msgid "``256``" -msgstr "" +msgstr "``256``" #: ../../source/mp/operator.rst:234 msgid "Token chunk size." -msgstr "" +msgstr "令牌块大小。" #: ../../source/mp/operator.rst:235 msgid "``server.maxWorkers``" -msgstr "" +msgstr "``server.maxWorkers``" #: ../../source/mp/operator.rst:236 msgid "``1``" -msgstr "" +msgstr "``1``" #: ../../source/mp/operator.rst:237 msgid "Worker threads for ZMQ requests." -msgstr "" +msgstr "ZMQ 请求的工作线程。" #: ../../source/mp/operator.rst:238 msgid "``server.hashAlgorithm``" -msgstr "" +msgstr "``server.hashAlgorithm``" #: ../../source/mp/operator.rst:239 msgid "``blake3``" -msgstr "" +msgstr "``blake3``" #: ../../source/mp/operator.rst:240 msgid "``builtin``, ``sha256_cbor``, or ``blake3``." -msgstr "" +msgstr "``builtin``, ``sha256_cbor``, 或 ``blake3``。" #: ../../source/mp/operator.rst:243 msgid "L1 Cache" -msgstr "" +msgstr "L1 缓存" #: ../../source/mp/operator.rst:252 ../../source/mp/operator.rst:480 msgid "``l1.sizeGB``" -msgstr "" +msgstr "``l1.sizeGB``" #: ../../source/mp/operator.rst:253 msgid "*required*" -msgstr "" +msgstr "*必需*" #: ../../source/mp/operator.rst:254 msgid "L1 cache size in GB. Must be > 0." -msgstr "" +msgstr "L1 缓存大小（以 GB 为单位）。必须大于 0。" #: ../../source/mp/operator.rst:257 msgid "Eviction" -msgstr "" +msgstr "逐出" #: ../../source/mp/operator.rst:266 ../../source/mp/operator.rst:482 msgid "``eviction.policy``" -msgstr "" +msgstr "``eviction.policy``" #: ../../source/mp/operator.rst:267 msgid "``LRU``" -msgstr "" +msgstr "``LRU``" #: ../../source/mp/operator.rst:268 msgid "Only ``LRU`` is supported." -msgstr "" +msgstr "仅支持 ``LRU``。" #: ../../source/mp/operator.rst:269 ../../source/mp/operator.rst:484 msgid "``eviction.triggerWatermark``" -msgstr "" +msgstr "``eviction.triggerWatermark``" #: ../../source/mp/operator.rst:270 msgid "``0.8``" -msgstr "" +msgstr "``0.8``" #: ../../source/mp/operator.rst:271 msgid "Usage ratio (0.0--1.0] to trigger eviction." -msgstr "" +msgstr "触发逐出的使用比例 (0.0--1.0]。" #: ../../source/mp/operator.rst:272 ../../source/mp/operator.rst:486 msgid "``eviction.evictionRatio``" -msgstr "" +msgstr "``eviction.evictionRatio``" #: ../../source/mp/operator.rst:273 msgid "``0.2``" -msgstr "" +msgstr "``0.2``" #: ../../source/mp/operator.rst:274 msgid "Fraction to evict (0.0--1.0]." -msgstr "" +msgstr "逐出比例 (0.0--1.0]。" #: ../../source/mp/operator.rst:277 ../../source/mp/operator.rst:619 msgid "Prometheus" -msgstr "" +msgstr "普罗米修斯" #: ../../source/mp/operator.rst:286 msgid "``prometheus.enabled``" -msgstr "" +msgstr "``prometheus.enabled``" #: ../../source/mp/operator.rst:287 msgid "``true``" -msgstr "" +msgstr "``true``" #: ../../source/mp/operator.rst:288 msgid "Expose Prometheus metrics." -msgstr "" +msgstr "暴露 Prometheus 指标。" #: ../../source/mp/operator.rst:289 msgid "``prometheus.port``" -msgstr "" +msgstr "``prometheus.port``" #: ../../source/mp/operator.rst:290 msgid "``9090``" -msgstr "" +msgstr "``9090``" #: ../../source/mp/operator.rst:291 msgid "``/metrics`` endpoint port." -msgstr "" +msgstr "``/metrics`` 端口。" #: ../../source/mp/operator.rst:292 msgid "``prometheus.serviceMonitor.enabled``" -msgstr "" +msgstr "``prometheus.serviceMonitor.enabled``" #: ../../source/mp/operator.rst:293 msgid "``false``" -msgstr "" +msgstr "``false``" #: ../../source/mp/operator.rst:294 msgid "Create a ServiceMonitor CR." -msgstr "" +msgstr "创建一个 ServiceMonitor CR。" #: ../../source/mp/operator.rst:295 msgid "``prometheus.serviceMonitor.interval``" -msgstr "" +msgstr "``prometheus.serviceMonitor.interval``" #: ../../source/mp/operator.rst:296 msgid "``30s``" -msgstr "" +msgstr "``30s``" #: ../../source/mp/operator.rst:297 msgid "Scrape interval." -msgstr "" +msgstr "抓取间隔。" #: ../../source/mp/operator.rst:298 msgid "``prometheus.serviceMonitor.labels``" -msgstr "" +msgstr "``prometheus.serviceMonitor.labels``" #: ../../source/mp/operator.rst:300 msgid "Extra labels on the ServiceMonitor." -msgstr "" +msgstr "ServiceMonitor上的额外标签。" #: ../../source/mp/operator.rst:303 msgid "L2 Storage" -msgstr "" +msgstr "L2 存储" #: ../../source/mp/operator.rst:312 msgid "``l2Backends``" -msgstr "" +msgstr "``l2Backends``" #: ../../source/mp/operator.rst:314 msgid "List of L2 backends (``type`` + ``config``). See :doc:`l2_storage`." -msgstr "" +msgstr "L2 后端列表 (``type`` + ``config``)。请参见 :doc:`l2_storage`。" #: ../../source/mp/operator.rst:318 msgid "Scheduling" -msgstr "" +msgstr "调度" #: ../../source/mp/operator.rst:327 msgid "``nodeSelector``" -msgstr "" +msgstr "``nodeSelector``" #: ../../source/mp/operator.rst:328 msgid "GPU nodes" -msgstr "" +msgstr "GPU 节点" #: ../../source/mp/operator.rst:329 msgid "Defaults to ``nvidia.com/gpu.present: \"true\"``." -msgstr "" +msgstr "默认为 ``nvidia.com/gpu.present: \\\"true\\\"``。" #: ../../source/mp/operator.rst:330 msgid "``affinity``" -msgstr "" +msgstr "``affinity``" #: ../../source/mp/operator.rst:332 msgid "Pod affinity rules." -msgstr "" +msgstr "Pod 亲和性规则。" #: ../../source/mp/operator.rst:333 msgid "``tolerations``" -msgstr "" +msgstr "``tolerations``" #: ../../source/mp/operator.rst:335 msgid "Pod tolerations." -msgstr "" +msgstr "Pod 容忍。" #: ../../source/mp/operator.rst:336 msgid "``priorityClassName``" -msgstr "" +msgstr "``priorityClassName``" #: ../../source/mp/operator.rst:338 msgid "Priority class for pods." -msgstr "" +msgstr "用于 Pods 的优先级类。" #: ../../source/mp/operator.rst:341 msgid "Overrides & Extras" -msgstr "" +msgstr "覆盖与额外选项" #: ../../source/mp/operator.rst:350 msgid "``logLevel``" -msgstr "" +msgstr "``logLevel``" #: ../../source/mp/operator.rst:351 msgid "``INFO``" -msgstr "" +msgstr "``INFO``" #: ../../source/mp/operator.rst:352 msgid "``DEBUG``, ``INFO``, ``WARNING``, ``ERROR``." -msgstr "" +msgstr "``DEBUG``, ``INFO``, ``WARNING``, ``ERROR``。" #: ../../source/mp/operator.rst:353 msgid "``resourceOverrides``" -msgstr "" +msgstr "``resourceOverrides``" #: ../../source/mp/operator.rst:355 msgid "Override auto-computed resources." -msgstr "" +msgstr "覆盖自动计算的资源。" #: ../../source/mp/operator.rst:356 msgid "``env``" -msgstr "" +msgstr "``env``" #: ../../source/mp/operator.rst:358 msgid "Extra environment variables." -msgstr "" +msgstr "额外的环境变量。" #: ../../source/mp/operator.rst:359 msgid "``volumes``" -msgstr "" +msgstr "``volumes``" #: ../../source/mp/operator.rst:361 msgid "Extra volumes." -msgstr "" +msgstr "额外的卷。" #: ../../source/mp/operator.rst:362 msgid "``volumeMounts``" -msgstr "" +msgstr "``volumeMounts``" #: ../../source/mp/operator.rst:364 msgid "Extra volume mounts." -msgstr "" +msgstr "额外的卷挂载。" #: ../../source/mp/operator.rst:365 msgid "``podAnnotations``" -msgstr "" +msgstr "``podAnnotations``" #: ../../source/mp/operator.rst:367 msgid "Extra pod annotations." -msgstr "" +msgstr "额外的 pod 注释。" #: ../../source/mp/operator.rst:368 msgid "``podLabels``" -msgstr "" +msgstr "``podLabels``" #: ../../source/mp/operator.rst:370 msgid "Extra pod labels." -msgstr "" +msgstr "额外的 pod 标签。" #: ../../source/mp/operator.rst:371 msgid "``serviceAccountName``" -msgstr "" +msgstr "``serviceAccountName``" #: ../../source/mp/operator.rst:373 msgid "ServiceAccount for pods." -msgstr "" +msgstr "用于 pods 的 ServiceAccount。" #: ../../source/mp/operator.rst:374 msgid "``extraArgs``" -msgstr "" +msgstr "``extraArgs``" #: ../../source/mp/operator.rst:376 msgid "Extra CLI flags (appended last, can override)." -msgstr "" +msgstr "额外的 CLI 标志（最后附加，可以覆盖）。" #: ../../source/mp/operator.rst:379 msgid "Auto-Computed Resources" -msgstr "" +msgstr "自动计算资源" #: ../../source/mp/operator.rst:381 msgid "" "When ``spec.resourceOverrides`` is not set, the operator derives " "resources from ``l1.sizeGB``:" -msgstr "" +msgstr "当 ``spec.resourceOverrides`` 未设置时，操作符从 ``l1.sizeGB`` 派生资源：" #: ../../source/mp/operator.rst:384 msgid "**CPU request**: ``4`` cores" -msgstr "" +msgstr "**CPU 请求**: ``4`` 核心" #: ../../source/mp/operator.rst:385 msgid "**Memory request**: ``ceil(l1.sizeGB + 5)`` Gi" -msgstr "" +msgstr "**内存请求**: ``ceil(l1.sizeGB + 5)`` Gi" #: ../../source/mp/operator.rst:386 msgid "**Memory limit**: ``ceil(memoryRequest * 1.5)`` Gi" -msgstr "" +msgstr "**内存限制**: ``ceil(memoryRequest * 1.5)`` Gi" #: ../../source/mp/operator.rst:388 msgid "For example, ``l1.sizeGB: 60`` produces a 65 Gi request and 98 Gi limit." -msgstr "" +msgstr "例如，``l1.sizeGB: 60`` 会产生 65 Gi 的请求和 98 Gi 的限制。" #: ../../source/mp/operator.rst:391 msgid "Auto-Injected Pod Settings" -msgstr "" +msgstr "自动注入的 Pod 设置" #: ../../source/mp/operator.rst:393 msgid "" "The operator always injects these into the pod spec (they are not " "configurable via the CRD):" -msgstr "" +msgstr "操作符始终将这些注入到 Pod 规格中（它们无法通过 CRD 配置）：" #: ../../source/mp/operator.rst:396 msgid "**hostIPC: true** -- Required for CUDA IPC between LMCache and vLLM." -msgstr "" +msgstr "**hostIPC: true** -- 这是在 LMCache 和 vLLM 之间进行 CUDA IPC 所必需的。" #: ../../source/mp/operator.rst:397 msgid "" "**--host 0.0.0.0** -- Binds the server to all interfaces so the node-" "local Service can route to it." -msgstr "" +msgstr "**--host 0.0.0.0** -- 将服务器绑定到所有接口，以便节点本地服务可以路由到它。" #: ../../source/mp/operator.rst:399 msgid "" "**NVIDIA_VISIBLE_DEVICES=all** -- Ensures GPU access for IPC-based memory" " transfers." -msgstr "" +msgstr "**NVIDIA_VISIBLE_DEVICES=all** -- 确保 GPU 可用于基于 IPC 的内存传输。" #: ../../source/mp/operator.rst:401 msgid "" "**TCP socket probes** -- Startup (5s initial, 30 failures), liveness " "(10s), and readiness (5s) probes on the server port." -msgstr "" +msgstr "**TCP socket 探测** -- 启动（初始 5 秒，30 次失败）、存活（10 秒）和就绪（5 秒）探测在服务器端口上。" #: ../../source/mp/operator.rst:405 msgid "" @@ -651,328 +651,328 @@ msgid "" "``hostIPC: true``, the container sees the host's ``/dev/shm`` directly. " "Mounting an emptyDir would shadow it with a private tmpfs and break CUDA " "IPC." -msgstr "" +msgstr "操作符**不**在 ``/dev/shm`` 挂载 emptyDir。使用 ``hostIPC: true`` 时，容器直接访问主机的 ``/dev/shm``。挂载 emptyDir 会用私有 tmpfs 进行遮蔽，从而破坏 CUDA IPC。" #: ../../source/mp/operator.rst:410 msgid "Resources Created" -msgstr "" +msgstr "创建的资源" #: ../../source/mp/operator.rst:412 msgid "For an ``LMCacheEngine`` named ``my-cache``:" -msgstr "" +msgstr "对于名为 ``my-cache`` 的 ``LMCacheEngine``：" #: ../../source/mp/operator.rst:418 msgid "Resource" -msgstr "" +msgstr "资源" #: ../../source/mp/operator.rst:419 msgid "Name" -msgstr "" +msgstr "名称" #: ../../source/mp/operator.rst:420 msgid "Purpose" -msgstr "" +msgstr "目的" #: ../../source/mp/operator.rst:421 msgid "DaemonSet" -msgstr "" +msgstr "守护进程集" #: ../../source/mp/operator.rst:422 ../../source/mp/operator.rst:425 #: ../../source/mp/operator.rst:434 msgid "``my-cache``" -msgstr "" +msgstr "``my-cache``" #: ../../source/mp/operator.rst:423 msgid "Runs LMCache server pods." -msgstr "" +msgstr "运行 LMCache 服务器 Pod。" #: ../../source/mp/operator.rst:424 msgid "Service (ClusterIP)" -msgstr "" +msgstr "服务 (ClusterIP)" #: ../../source/mp/operator.rst:426 msgid "Node-local discovery (``internalTrafficPolicy=Local``)." -msgstr "" +msgstr "节点本地发现（``internalTrafficPolicy=Local``）。" #: ../../source/mp/operator.rst:427 msgid "Service (headless)" -msgstr "" +msgstr "无头服务" #: ../../source/mp/operator.rst:428 msgid "``my-cache-metrics``" -msgstr "" +msgstr "``my-cache-metrics``" #: ../../source/mp/operator.rst:429 msgid "Prometheus scrape target." -msgstr "" +msgstr "Prometheus 抓取目标。" #: ../../source/mp/operator.rst:430 msgid "ConfigMap" -msgstr "" +msgstr "ConfigMap" #: ../../source/mp/operator.rst:431 msgid "``my-cache-connection``" -msgstr "" +msgstr "``my-cache-connection``" #: ../../source/mp/operator.rst:432 msgid "``kv-transfer-config`` JSON for vLLM." -msgstr "" +msgstr "``kv-transfer-config`` vLLM 的 JSON。" #: ../../source/mp/operator.rst:433 msgid "ServiceMonitor" -msgstr "" +msgstr "服务监控器" #: ../../source/mp/operator.rst:435 msgid "Prometheus Operator integration (when enabled)." -msgstr "" +msgstr "启用时的 Prometheus Operator 集成。" #: ../../source/mp/operator.rst:437 msgid "The connection ConfigMap contains:" -msgstr "" +msgstr "连接 ConfigMap 包含：" #: ../../source/mp/operator.rst:451 msgid "Status & Conditions" -msgstr "" +msgstr "状态与条件" #: ../../source/mp/operator.rst:457 msgid "The status section includes:" -msgstr "" +msgstr "状态部分包括：" #: ../../source/mp/operator.rst:459 msgid "**phase**: ``Pending``, ``Running``, ``Degraded``, or ``Failed``." -msgstr "" +msgstr "**phase**: ``待处理``, ``运行中``, ``降级``, 或 ``失败``。" #: ../../source/mp/operator.rst:460 msgid "**readyInstances** / **desiredInstances**: Instance counts." -msgstr "" +msgstr "**readyInstances** / **desiredInstances**: 实例计数。" #: ../../source/mp/operator.rst:461 msgid "" "**endpoints**: Per-node connection info (node name, host IP, pod name, " "port, readiness)." -msgstr "" +msgstr "**endpoints**: 每个节点的连接信息（节点名称、主机 IP、Pod 名称、端口、就绪状态）。" #: ../../source/mp/operator.rst:463 msgid "**conditions**:" -msgstr "" +msgstr "**条件**：" #: ../../source/mp/operator.rst:465 msgid "``Available`` -- At least one instance is ready." -msgstr "" +msgstr "``可用`` -- 至少有一个实例已准备好。" #: ../../source/mp/operator.rst:466 msgid "``AllInstancesReady`` -- All desired instances are ready." -msgstr "" +msgstr "``AllInstancesReady`` -- 所有期望的实例都已准备好。" #: ../../source/mp/operator.rst:467 msgid "``ConfigValid`` -- Spec validation passed." -msgstr "" +msgstr "``ConfigValid`` -- 配置验证通过。" #: ../../source/mp/operator.rst:470 msgid "Validation Rules" -msgstr "" +msgstr "验证规则" #: ../../source/mp/operator.rst:472 msgid "The operator validates the CR spec at apply time:" -msgstr "" +msgstr "操作符在应用时验证 CR 规格：" #: ../../source/mp/operator.rst:479 msgid "Rule" -msgstr "" +msgstr "规则" #: ../../source/mp/operator.rst:481 msgid "Required, must be > 0." -msgstr "" +msgstr "必需，必须 > 0。" #: ../../source/mp/operator.rst:483 msgid "Must be ``LRU`` (if set)." -msgstr "" +msgstr "必须是 ``LRU``（如果设置）。" #: ../../source/mp/operator.rst:485 ../../source/mp/operator.rst:487 msgid "Must be in (0.0, 1.0]." -msgstr "" +msgstr "必须在 (0.0, 1.0] 之间。" #: ../../source/mp/operator.rst:489 msgid "Must be in [1024, 65535]." -msgstr "" +msgstr "必须在 [1024, 65535] 之间。" #: ../../source/mp/operator.rst:492 msgid "Examples" -msgstr "" +msgstr "示例" #: ../../source/mp/operator.rst:495 msgid "Target Only GPU Nodes" -msgstr "" +msgstr "仅目标 GPU 节点" #: ../../source/mp/operator.rst:497 msgid "" "Use ``nodeSelector`` to run LMCache only on GPU nodes. New GPU nodes " "automatically get an LMCache pod:" -msgstr "" +msgstr "使用 ``nodeSelector`` 仅在 GPU 节点上运行 LMCache。新的 GPU 节点会自动获得一个 LMCache pod：" #: ../../source/mp/operator.rst:513 msgid "" "The operator defaults ``nodeSelector`` to ``nvidia.com/gpu.present: " "\"true\"`` when not specified, so a minimal CR already targets GPU nodes." -msgstr "" +msgstr "操作符默认将 ``nodeSelector`` 设置为 ``nvidia.com/gpu.present: \\\"true\\\"``，当未指定时，因此最小的 CR 已经针对 GPU 节点。" #: ../../source/mp/operator.rst:517 msgid "Custom Server Port" -msgstr "" +msgstr "自定义服务器端口" #: ../../source/mp/operator.rst:519 msgid "If the default port (5555) conflicts with other services:" -msgstr "" +msgstr "如果默认端口（5555）与其他服务冲突：" #: ../../source/mp/operator.rst:533 msgid "" "The connection ConfigMap updates automatically -- vLLM pods pick up the " "new port on restart." -msgstr "" +msgstr "连接 ConfigMap 会自动更新 -- vLLM pods 在重启时会获取新端口。" #: ../../source/mp/operator.rst:537 msgid "Production with Prometheus Monitoring" -msgstr "" +msgstr "使用 Prometheus 监控的生产环境" #: ../../source/mp/operator.rst:573 msgid "See :doc:`observability` for metric names and Grafana configuration." -msgstr "" +msgstr "请参阅 :doc:`可观察性` 以获取指标名称和 Grafana 配置。" #: ../../source/mp/operator.rst:576 msgid "Override Auto-Computed Resources" -msgstr "" +msgstr "覆盖自动计算的资源" #: ../../source/mp/operator.rst:595 msgid "Operator vs Manual Deployment" -msgstr "" +msgstr "操作员与手动部署" #: ../../source/mp/operator.rst:601 msgid "Concern" -msgstr "" +msgstr "关注" #: ../../source/mp/operator.rst:602 msgid "Manual DaemonSet" -msgstr "" +msgstr "手动 DaemonSet" #: ../../source/mp/operator.rst:603 msgid "LMCacheEngine Operator" -msgstr "" +msgstr "LMCacheEngine Operator" #: ../../source/mp/operator.rst:604 msgid "hostIPC" -msgstr "" +msgstr "hostIPC" #: ../../source/mp/operator.rst:605 ../../source/mp/operator.rst:608 msgid "Must set manually" -msgstr "" +msgstr "必须手动设置" #: ../../source/mp/operator.rst:606 ../../source/mp/operator.rst:609 msgid "Auto-injected" -msgstr "" +msgstr "自动注入" #: ../../source/mp/operator.rst:607 msgid "``--host 0.0.0.0``" -msgstr "" +msgstr "``--host 0.0.0.0``" #: ../../source/mp/operator.rst:610 msgid "Service discovery" -msgstr "" +msgstr "服务发现" #: ../../source/mp/operator.rst:611 msgid "``hostNetwork`` + ``status.hostIP``" -msgstr "" +msgstr "``hostNetwork`` + ``status.hostIP``" #: ../../source/mp/operator.rst:612 msgid "Node-local ClusterIP Service + ConfigMap" -msgstr "" +msgstr "节点本地 ClusterIP 服务 + ConfigMap" #: ../../source/mp/operator.rst:613 msgid "vLLM config" -msgstr "" +msgstr "vLLM 配置" #: ../../source/mp/operator.rst:614 msgid "Copy JSON into Deployment" -msgstr "" +msgstr "将 JSON 复制到部署中" #: ../../source/mp/operator.rst:615 msgid "Mount ``-connection`` ConfigMap" -msgstr "" +msgstr "挂载 ``-connection`` ConfigMap" #: ../../source/mp/operator.rst:616 msgid "Resource sizing" -msgstr "" +msgstr "资源大小调整" #: ../../source/mp/operator.rst:617 msgid "Manual calculation" -msgstr "" +msgstr "手动计算" #: ../../source/mp/operator.rst:618 msgid "Auto-computed from ``l1.sizeGB``" -msgstr "" +msgstr "从 ``l1.sizeGB`` 自动计算" #: ../../source/mp/operator.rst:620 msgid "Manual ServiceMonitor" -msgstr "" +msgstr "手动 ServiceMonitor" #: ../../source/mp/operator.rst:621 msgid "``serviceMonitor.enabled: true``" -msgstr "" +msgstr "``serviceMonitor.enabled: true``" #: ../../source/mp/operator.rst:622 msgid "Validation" -msgstr "" +msgstr "验证" #: ../../source/mp/operator.rst:623 msgid "Runtime errors only" -msgstr "" +msgstr "仅限运行时错误" #: ../../source/mp/operator.rst:624 msgid "``kubectl apply`` rejects invalid specs" -msgstr "" +msgstr "``kubectl apply`` 拒绝无效的规格" #: ../../source/mp/operator.rst:625 msgid "New GPU nodes" -msgstr "" +msgstr "新的 GPU 节点" #: ../../source/mp/operator.rst:626 msgid "DaemonSet handles it" -msgstr "" +msgstr "DaemonSet 处理它" #: ../../source/mp/operator.rst:627 msgid "DaemonSet handles it (same)" -msgstr "" +msgstr "DaemonSet 处理它（相同）" #: ../../source/mp/operator.rst:630 msgid "Security Considerations" -msgstr "" +msgstr "安全考虑事项" #: ../../source/mp/operator.rst:632 msgid "" "**hostIPC** exposes the host's IPC namespace (System V IPC, POSIX message" " queues) to the container. Any process in the container can interact " "with IPC resources from other processes on the same host." -msgstr "" +msgstr "**hostIPC** 将主机的 IPC 命名空间（System V IPC，POSIX 消息队列）暴露给容器。容器中的任何进程都可以与同一主机上其他进程的 IPC 资源进行交互。" #: ../../source/mp/operator.rst:636 msgid "Deploy only in trusted environments." -msgstr "" +msgstr "仅在受信任的环境中部署。" #: ../../source/mp/operator.rst:637 msgid "" "Clusters using Pod Security Standards must allow the ``privileged`` " "profile for the LMCache namespace -- the ``baseline`` and ``restricted`` " "profiles reject ``hostIPC``." -msgstr "" +msgstr "使用 Pod 安全标准的集群必须允许 LMCache 命名空间的 ``privileged`` 配置文件 -- ``baseline`` 和 ``restricted`` 配置文件拒绝 ``hostIPC``。" #: ../../source/mp/operator.rst:642 msgid "Development" -msgstr "" +msgstr "开发" #: ../../source/mp/operator.rst:654 msgid "Pushing a custom operator image:" -msgstr "" +msgstr "推送自定义操作符镜像：" #: ../../source/mp/operator.rst:665 msgid "If your cluster needs pull credentials:" -msgstr "" +msgstr "如果您的集群需要拉取凭据：" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/quickstart.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/quickstart.po index c78eea59937..4ea35dc52f7 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/quickstart.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/quickstart.po @@ -21,155 +21,155 @@ msgstr "" #: ../../source/mp/quickstart.rst:2 msgid "Quick Start" -msgstr "" +msgstr "快速开始" #: ../../source/mp/quickstart.rst:4 msgid "" "This page walks through the fastest ways to get LMCache multiprocess mode" " running -- locally, in Docker, and with the HTTP server variant." -msgstr "" +msgstr "本页介绍了快速启动 LMCache 多进程模式的几种方法——本地、在 Docker 中以及使用 HTTP 服务器变体。" #: ../../source/mp/quickstart.rst:8 msgid "Local Quick Start" -msgstr "" +msgstr "本地快速开始" #: ../../source/mp/quickstart.rst:10 msgid "**Step 1: Start the LMCache server**" -msgstr "" +msgstr "**步骤 1：启动 LMCache 服务器**" #: ../../source/mp/quickstart.rst:17 msgid "Expected log output:" -msgstr "" +msgstr "预期的日志输出：" #: ../../source/mp/quickstart.rst:24 msgid "" "The default ZMQ port is **5555** (use ``--port`` to change it). The HTTP " "frontend listens on **8080** by default (use ``--http-port`` to change " "it)." -msgstr "" +msgstr "默认的 ZMQ 端口是 **5555**（使用 ``--port`` 来更改）。HTTP 前端默认监听 **8080**（使用 ``--http-port`` 来更改）。" #: ../../source/mp/quickstart.rst:28 msgid "**Step 2: Start vLLM with the LMCache connector**" -msgstr "" +msgstr "**步骤 2：使用 LMCache 连接器启动 vLLM**" #: ../../source/mp/quickstart.rst:30 msgid "In a new terminal:" -msgstr "" +msgstr "在新的终端中：" #: ../../source/mp/quickstart.rst:39 msgid "" "This connects to the default LMCache port (5555) on localhost. If you " "changed the server port with ``--port``, pass it on the vLLM side via " "``kv_connector_extra_config``:" -msgstr "" +msgstr "这将连接到本地主机上的默认 LMCache 端口 (5555)。如果您使用 ``--port`` 更改了服务器端口，请通过 ``kv_connector_extra_config`` 在 vLLM 端传递它：" #: ../../source/mp/quickstart.rst:49 msgid "To connect to a remote host, also set ``lmcache.mp.host``:" -msgstr "" +msgstr "要连接到远程主机，还需设置 ``lmcache.mp.host``：" #: ../../source/mp/quickstart.rst:56 msgid "You should see on the **vLLM** side:" -msgstr "" +msgstr "您应该在 **vLLM** 端看到：" #: ../../source/mp/quickstart.rst:62 msgid "And on the **LMCache** side:" -msgstr "" +msgstr "在 **LMCache** 端：" #: ../../source/mp/quickstart.rst:68 msgid "**Step 3: Send a request**" -msgstr "" +msgstr "**步骤 3：发送请求**" #: ../../source/mp/quickstart.rst:80 msgid "First request -- tokens are **stored**:" -msgstr "" +msgstr "第一次请求 -- 令牌被 **存储**：" #: ../../source/mp/quickstart.rst:86 msgid "Second identical request -- tokens are **retrieved** from cache:" -msgstr "" +msgstr "第二个相同的请求 -- 令牌从缓存中 **检索**：" #: ../../source/mp/quickstart.rst:93 msgid "Docker Quick Start" -msgstr "" +msgstr "Docker 快速入门" #: ../../source/mp/quickstart.rst:95 msgid "**Step 1: Start the LMCache container**" -msgstr "" +msgstr "**步骤 1：启动 LMCache 容器**" #: ../../source/mp/quickstart.rst:107 msgid "" "``--network host`` lets the vLLM container reach the LMCache server on " "localhost. ``--ipc host`` is required for CUDA IPC shared memory." -msgstr "" +msgstr "``--network host`` 允许 vLLM 容器访问本地主机上的 LMCache 服务器。 ``--ipc host`` 是 CUDA IPC 共享内存所必需的。" #: ../../source/mp/quickstart.rst:110 msgid "**Step 2: Start the vLLM container**" -msgstr "" +msgstr "**步骤 2：启动 vLLM 容器**" #: ../../source/mp/quickstart.rst:123 msgid "" "Use the nightly images (``lmcache/standalone:nightly`` and ``lmcache" "/vllm-openai:latest-nightly``) as the MP-mode interfaces are actively " "evolving." -msgstr "" +msgstr "使用夜间构建镜像（``lmcache/standalone:nightly`` 和 ``lmcache/vllm-openai:latest-nightly``），因为 MP 模式接口正在积极演进。" #: ../../source/mp/quickstart.rst:127 msgid "**Step 3: Send requests** the same way as in the local quick start." -msgstr "" +msgstr "**步骤 3：发送请求** 与本地快速入门相同。" #: ../../source/mp/quickstart.rst:130 msgid "HTTP Server Quick Start" -msgstr "" +msgstr "HTTP 服务器快速入门" #: ../../source/mp/quickstart.rst:132 msgid "" "The HTTP server wraps the ZMQ server with a FastAPI frontend, adding HTTP" " management endpoints for health checking and cache administration." -msgstr "" +msgstr "HTTP 服务器使用 FastAPI 前端封装了 ZMQ 服务器，添加了用于健康检查和缓存管理的 HTTP 管理端点。" #: ../../source/mp/quickstart.rst:140 msgid "" "The HTTP server listens on ``0.0.0.0:8080`` by default (configurable with" " ``--http-host`` and ``--http-port``)." -msgstr "" +msgstr "HTTP 服务器默认监听 ``0.0.0.0:8080``（可通过 ``--http-host`` 和 ``--http-port`` 进行配置）。" #: ../../source/mp/quickstart.rst:143 msgid "**Endpoints:**" -msgstr "" +msgstr "**端点：**" #: ../../source/mp/quickstart.rst:149 msgid "Method" -msgstr "" +msgstr "方法" #: ../../source/mp/quickstart.rst:150 msgid "Path" -msgstr "" +msgstr "路径" #: ../../source/mp/quickstart.rst:151 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/mp/quickstart.rst:152 ../../source/mp/quickstart.rst:161 msgid "GET" -msgstr "" +msgstr "获取" #: ../../source/mp/quickstart.rst:153 msgid "``/healthcheck``" -msgstr "" +msgstr "``/healthcheck``" #: ../../source/mp/quickstart.rst:154 #, python-brace-format msgid "" "Returns ``{\"status\": \"healthy\"}`` when the engine is initialized and " "memory checks pass. Suitable for Kubernetes liveness/readiness probes." -msgstr "" +msgstr "当引擎初始化并且内存检查通过时，返回 ``{\"status\": \"healthy\"}``。适用于 Kubernetes 的存活/就绪探针。" #: ../../source/mp/quickstart.rst:156 msgid "POST" -msgstr "" +msgstr "POST" #: ../../source/mp/quickstart.rst:157 msgid "``/clear-cache``" -msgstr "" +msgstr "``/clear-cache``" #: ../../source/mp/quickstart.rst:158 #, python-brace-format @@ -177,25 +177,25 @@ msgid "" "Force-clears all KV cache data stored in L1 (CPU) memory, including " "objects with active read/write locks. Returns ``{\"status\": \"ok\"}`` on" " success." -msgstr "" +msgstr "强制清除存储在 L1 (CPU) 内存中的所有 KV Cache 数据，包括具有活动读/写锁的对象。成功时返回 ``{\\\"status\\\": \\\"ok\\\"}``。" #: ../../source/mp/quickstart.rst:162 msgid "``/status``" -msgstr "" +msgstr "``/status``" #: ../../source/mp/quickstart.rst:163 msgid "" "Returns detailed internal state of all MP components including L1 cache, " "L2 adapters, controllers, registered GPUs, and active sessions." -msgstr "" +msgstr "返回所有 MP 组件的详细内部状态，包括 L1 缓存、L2 适配器、控制器、注册的 GPU 和活动会话。" #: ../../source/mp/quickstart.rst:166 msgid "Examples:" -msgstr "" +msgstr "示例：" #: ../../source/mp/quickstart.rst:181 msgid "" "The ZMQ server runs on the same default port (5555) and accepts vLLM " "connections exactly as in the local quick start." -msgstr "" +msgstr "ZMQ 服务器在相同的默认端口（5555）上运行，并接受 vLLM 连接，正如本地快速入门中所示。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/serde.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/serde.po index 2229e1e79ad..9a5d2df41c5 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/serde.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/serde.po @@ -21,104 +21,104 @@ msgstr "" #: ../../source/mp/serde.rst:2 msgid "L2 Serde (Serialization / Deserialization)" -msgstr "" +msgstr "L2 序列化（Serialization / Deserialization）" #: ../../source/mp/serde.rst:4 msgid "" "LMCache supports a **per-adapter serde** that transforms KV cache data on" " its way to and from an L2 adapter. Typical uses: quantization (shrink " "storage footprint), compression, encryption." -msgstr "" +msgstr "LMCache 支持 **每个适配器的序列化/反序列化**，在 KV Cache 数据与 L2 适配器之间传输时进行转换。典型用途：量化（缩小存储占用）、压缩、加密。" #: ../../source/mp/serde.rst:14 msgid "When to use serde" -msgstr "" +msgstr "何时使用序列化与反序列化" #: ../../source/mp/serde.rst:16 msgid "" "**Save L2 storage or bandwidth.** fp8 quantization halves byte volume vs." " bf16 with minor accuracy loss — a good fit for disk / remote adapters." -msgstr "" +msgstr "**节省 L2 存储或带宽。** fp8 量化相比于 bf16 将字节体积减半，且准确度损失较小——非常适合磁盘/远程适配器。" #: ../../source/mp/serde.rst:19 msgid "" "**Encrypt at rest.** Wrap the raw bytes with authenticated encryption " "before they land on disk." -msgstr "" +msgstr "**静态加密。** 在原始字节写入磁盘之前，使用经过身份验证的加密进行包装。" #: ../../source/mp/serde.rst:21 msgid "" "**Custom compression.** Anything lossless (lz4/zstd) or lossy (CacheGen-" "style) can be plugged in via the ``Serializer`` / ``Deserializer`` ABCs." -msgstr "" +msgstr "**自定义压缩。** 任何无损（lz4/zstd）或有损（CacheGen 风格）都可以通过 ``Serializer`` / ``Deserializer`` ABC 插入。" #: ../../source/mp/serde.rst:25 msgid "" "Serde is **opt-in per adapter**: one ``--l2-adapter`` may use fp8 while " "another stores raw bytes. When omitted, the adapter behaves exactly as if" " serde did not exist (no extra allocations, no extra threads)." -msgstr "" +msgstr "Serde 是 **按适配器选择** 的：一个 ``--l2-adapter`` 可以使用 fp8，而另一个则存储原始字节。当省略时，适配器的行为就像 serde 不存在一样（没有额外的分配，没有额外的线程）。" #: ../../source/mp/serde.rst:31 msgid "Configuring serde on an L2 adapter" -msgstr "" +msgstr "在 L2 适配器上配置 serde" #: ../../source/mp/serde.rst:33 msgid "" "Add a ``\"serde\"`` sub-dict to any ``--l2-adapter`` JSON spec. The " "``type`` field selects a registered serde; remaining keys are forwarded " "to the serde factory." -msgstr "" +msgstr "向任何 ``--l2-adapter`` JSON 规范添加一个 ``\"serde\"`` 子字典。``type`` 字段选择一个已注册的 serde；其余键将转发到 serde 工厂。" #: ../../source/mp/serde.rst:48 msgid "Built-in serde types" -msgstr "" +msgstr "内置 serde 类型" #: ../../source/mp/serde.rst:52 msgid "``type``" -msgstr "" +msgstr "``type``" #: ../../source/mp/serde.rst:53 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/mp/serde.rst:54 msgid "Config fields" -msgstr "" +msgstr "配置字段" #: ../../source/mp/serde.rst:55 msgid "``fp8``" -msgstr "" +msgstr "``fp8``" #: ../../source/mp/serde.rst:56 msgid "" "Quantize each element to 8-bit float; dequantize on load. Lossy but " "highly compressible." -msgstr "" +msgstr "将每个元素量化为 8 位浮点数；在加载时反量化。虽然有损，但高度可压缩。" #: ../../source/mp/serde.rst:58 msgid "" "``fp8_dtype`` (default ``float8_e4m3fn``; also accepts ``float8_e5m2``), " "``max_workers`` (thread pool size, default 1)" -msgstr "" +msgstr "``fp8_dtype``（默认 ``float8_e4m3fn``；也接受 ``float8_e5m2``），``max_workers``（线程池大小，默认 1）" #: ../../source/mp/serde.rst:64 msgid "Writing a custom serde" -msgstr "" +msgstr "编写自定义序列化/反序列化" #: ../../source/mp/serde.rst:66 msgid "" "Implement the two sync ABCs (``Serializer``, ``Deserializer``) with your " "transform logic, then register a factory keyed on a name you pick:" -msgstr "" +msgstr "实现两个同步 ABC（``Serializer``，``Deserializer``），并使用您的转换逻辑，然后注册一个以您选择的名称为键的工厂：" #: ../../source/mp/serde.rst:98 msgid "Reference it from your adapter config:" -msgstr "" +msgstr "从你的适配器配置中引用它：" #: ../../source/mp/serde.rst:106 msgid "Notes" -msgstr "" +msgstr "备注" #: ../../source/mp/serde.rst:108 msgid "" @@ -126,7 +126,7 @@ msgid "" "upper bound on the actual serialized output — include any safety margin " "directly in the estimate (e.g., the built-in fp8 serializer returns ``1.5" " * num_elements``)." -msgstr "" +msgstr "**缓冲区大小。** ``estimate_serialized_size(layout)`` 必须返回实际序列化输出的上限 — 直接在估算中包含任何安全边际（例如，内置的 fp8 序列化器返回 ``1.5 * num_elements``）。" #: ../../source/mp/serde.rst:112 msgid "" @@ -134,18 +134,18 @@ msgid "" "deserialize), the whole submitted batch is reported as failed — partial " "success within one batch is not surfaced. Failed keys are cleaned up " "automatically." -msgstr "" +msgstr "**失败处理。** 如果任何步骤失败（序列化、存储、加载或反序列化），则整个提交的批次将被报告为失败——在一个批次内的部分成功不会被显示。失败的键会自动清理。" #: ../../source/mp/serde.rst:116 msgid "" "**Thread pool.** ``AsyncSerdeProcessor(max_workers=N)`` controls the pool" " size. Transforms that release the GIL (e.g., torch ops) benefit from ``N" " > 1``; pure-Python transforms do not." -msgstr "" +msgstr "**线程池。** ``AsyncSerdeProcessor(max_workers=N)`` 控制池的大小。释放全局解释器锁（GIL）的变换（例如，torch 操作）可以从 ``N > 1`` 中受益；纯 Python 变换则不然。" #: ../../source/mp/serde.rst:122 msgid "Example" -msgstr "" +msgstr "示例" #: ../../source/mp/serde.rst:124 msgid "" @@ -155,5 +155,5 @@ msgid "" ":file:`examples/serde/fp8/`. A pytest-based filesystem round-trip test " "(no vLLM required) is at " ":file:`tests/v1/distributed/serde/test_serde_fs_e2e.py`." -msgstr "" +msgstr "一个端到端的脚本，它在磁盘适配器上启动一个带有 fp8 的 lmcache 服务器，运行 vLLM，清除 L1，然后重新运行相同的请求以触发 L2 预取 + fp8 反序列化路径，位于 :file:`examples/serde/fp8/`。一个基于 pytest 的文件系统往返测试（不需要 vLLM）位于 :file:`tests/v1/distributed/serde/test_serde_fs_e2e.py`。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/mp/tracing_and_debugging.po b/docs/source/locale/zh_CN/LC_MESSAGES/mp/tracing_and_debugging.po index 397d01e3e86..2b8beb75877 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/mp/tracing_and_debugging.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/mp/tracing_and_debugging.po @@ -21,34 +21,34 @@ msgstr "" #: ../../source/mp/tracing_and_debugging.rst:2 msgid "Tracing and Debugging" -msgstr "" +msgstr "追踪和调试" #: ../../source/mp/tracing_and_debugging.rst:4 msgid "" "LMCache MP mode can record every ``StorageManager`` public-API call to a " "binary **trace file** and reissue those calls later against a fresh " "server via ``lmcache trace replay``. The feature is designed for:" -msgstr "" +msgstr "LMCache MP 模式可以将每个 ``StorageManager`` 公共 API 调用记录到一个二进制 **跟踪文件** 中，并通过 ``lmcache trace replay`` 在新服务器上重新发出这些调用。该功能旨在用于：" #: ../../source/mp/tracing_and_debugging.rst:8 msgid "" "**Regression hunting** — capture a production workload, then replay it " "against a build under investigation to reproduce a bug offline." -msgstr "" +msgstr "**回归排查** — 捕获生产工作负载，然后在调查中的构建上重放，以离线重现错误。" #: ../../source/mp/tracing_and_debugging.rst:10 msgid "" "**Performance characterization** — measure L1/L2 latency distributions " "under a realistic storage-level access pattern, without needing vLLM or a" " GPU." -msgstr "" +msgstr "**性能特征化** — 在真实的存储级访问模式下测量 L1/L2 延迟分布，无需 vLLM 或 GPU。" #: ../../source/mp/tracing_and_debugging.rst:13 msgid "" "**Configuration tuning** — replay the same trace against different L1 " "sizes, eviction policies, and L2 adapters to compare their behavior on " "identical input." -msgstr "" +msgstr "**配置调整** — 针对不同的 L1 大小、逐出策略和 L2 适配器重放相同的跟踪，以比较它们在相同输入下的行为。" #: ../../source/mp/tracing_and_debugging.rst:19 msgid "" @@ -56,101 +56,101 @@ msgid "" "spans). OTel tracing exports *live* spans to an OTLP endpoint for online " "observability; trace recording persists a replayable binary file for " "offline analysis. Both can be enabled simultaneously." -msgstr "" +msgstr "跟踪记录与 ``--enable-tracing``（OTel spans）是 **独立** 的。OTel 跟踪将 *实时* spans 导出到 OTLP 端点以进行在线可观察性；跟踪记录则持久化一个可重放的二进制文件以供离线分析。两者可以同时启用。" #: ../../source/mp/tracing_and_debugging.rst:28 msgid "Recording a Trace" -msgstr "" +msgstr "记录跟踪" #: ../../source/mp/tracing_and_debugging.rst:30 msgid "" "Recording is **off by default**. Enable it by adding ``--trace-level " "storage`` to ``lmcache server``:" -msgstr "" +msgstr "记录默认是**关闭的**。通过将``--trace-level storage``添加到``lmcache server``来启用它：" #: ../../source/mp/tracing_and_debugging.rst:48 msgid "" "Drive traffic through the server as usual (vLLM requests, benchmark " "scripts, etc.). The trace file is closed cleanly on ``SIGTERM`` via the " "EventBus stop path — no ``--stop-tracing`` command needed." -msgstr "" +msgstr "通过服务器正常驱动流量（vLLM 请求、基准测试脚本等）。在通过 EventBus 停止路径接收到 ``SIGTERM`` 时，跟踪文件会被干净地关闭 — 不需要 ``--stop-tracing`` 命令。" #: ../../source/mp/tracing_and_debugging.rst:52 msgid "**What is captured:**" -msgstr "" +msgstr "**捕获的内容：**" #: ../../source/mp/tracing_and_debugging.rst:54 msgid "" "The fully-qualified name of every decorated ``StorageManager`` call (e.g." " ``StorageManager.reserve_write``, " "``StorageManager.submit_prefetch_task``)." -msgstr "" +msgstr "每个被装饰的 ``StorageManager`` 调用的完全限定名称（例如 ``StorageManager.reserve_write``、``StorageManager.submit_prefetch_task``）。" #: ../../source/mp/tracing_and_debugging.rst:57 msgid "" "Each call's input arguments (``keys``, ``layout_desc``, ``mode``, " "``extra_count``, ``external_request_id``, …)." -msgstr "" +msgstr "每个调用的输入参数（``keys``, ``layout_desc``, ``mode``, ``extra_count``, ``external_request_id``, …）。" #: ../../source/mp/tracing_and_debugging.rst:59 msgid "Wall-clock and monotonic timestamps per call." -msgstr "" +msgstr "每次调用的墙钟时间和单调时间戳。" #: ../../source/mp/tracing_and_debugging.rst:60 msgid "" "A header with the file format version, trace schema version, start " "timestamps, and a SHA-256 digest of the active ``StorageManagerConfig`` " "so replay can flag mismatched configurations." -msgstr "" +msgstr "一个包含文件格式版本、跟踪模式版本、开始时间戳以及活动的 ``StorageManagerConfig`` 的 SHA-256 摘要的头部，以便重放可以标记不匹配的配置。" #: ../../source/mp/tracing_and_debugging.rst:65 msgid "**What is not captured:**" -msgstr "" +msgstr "**未捕获的内容：**" #: ../../source/mp/tracing_and_debugging.rst:67 msgid "" "**KV tensor bytes.** Replay exercises bookkeeping and controller logic; " "payloads at replay time are zeros. The trace file stays bounded even for " "long runs." -msgstr "" +msgstr "**KV 张量字节。** 重放练习的记账和控制逻辑；重放时的有效载荷为零。即使在长时间运行的情况下，跟踪文件也保持有限。" #: ../../source/mp/tracing_and_debugging.rst:70 msgid "" "Calls inside ``MPCacheEngine``, the message queue, or GPU-copy code. " "Those layers are out of scope for the ``storage`` trace level." -msgstr "" +msgstr "在 ``MPCacheEngine``、消息队列或 GPU 复制代码中的调用。这些层超出了 ``storage`` 跟踪级别的范围。" #: ../../source/mp/tracing_and_debugging.rst:73 msgid "**Overhead:**" -msgstr "" +msgstr "**开销：**" #: ../../source/mp/tracing_and_debugging.rst:75 msgid "Off: a single boolean check per ``StorageManager`` call. Effectively free." -msgstr "" +msgstr "关闭：每次 ``StorageManager`` 调用进行一次布尔检查。实际上是免费的。" #: ../../source/mp/tracing_and_debugging.rst:76 msgid "" "On: encoding and file I/O happen on the EventBus drain thread, off the " "request path. In practice this has no visible impact on request latency." -msgstr "" +msgstr "开启：编码和文件 I/O 在 EventBus 排水线程上进行，不在请求路径上。实际上，这对请求延迟没有明显影响。" #: ../../source/mp/tracing_and_debugging.rst:81 msgid "Inspecting a Trace" -msgstr "" +msgstr "检查追踪" #: ../../source/mp/tracing_and_debugging.rst:83 msgid "Before replaying, ``lmcache trace info`` prints a one-screen summary:" -msgstr "" +msgstr "在重放之前，``lmcache trace info`` 打印一个一屏的摘要：" #: ../../source/mp/tracing_and_debugging.rst:106 msgid "" "Use this to sanity-check that the trace you intend to replay covers the " "expected operation mix and duration." -msgstr "" +msgstr "使用此功能来检查您打算重放的跟踪是否涵盖了预期的操作组合和持续时间。" #: ../../source/mp/tracing_and_debugging.rst:110 msgid "Replaying a Trace" -msgstr "" +msgstr "重放跟踪" #: ../../source/mp/tracing_and_debugging.rst:112 msgid "" @@ -159,11 +159,11 @@ msgid "" "side config is **chosen by you**, not copied from the recording. This is " "the feature's main value — you can compare different L1/L2 setups on " "identical input." -msgstr "" +msgstr "``lmcache trace replay FILE`` 重新发出每个记录的调用，使用您提供的 CLI 标志构建的 **全新** ``StorageManager``。重放端的配置由 **您选择**，而不是从录音中复制。这是该功能的主要价值——您可以在相同的输入上比较不同的 L1/L2 设置。" #: ../../source/mp/tracing_and_debugging.rst:118 msgid "Minimal invocation:" -msgstr "" +msgstr "最小调用：" #: ../../source/mp/tracing_and_debugging.rst:125 msgid "" @@ -171,7 +171,7 @@ msgid "" "``lmcache server``. Any storage-manager flag accepted by the server also " "works here (``--l2-adapter``, ``--l1-use-lazy``, ``--l2-store-policy``, " "…); run ``lmcache trace replay --help`` for the full list." -msgstr "" +msgstr "``--l1-size-gb`` 和 ``--eviction-policy`` 是必需的，就像在 ``lmcache server`` 上一样。服务器接受的任何存储管理器标志在这里也有效（``--l2-adapter``、``--l1-use-lazy``、``--l2-store-policy`` 等）；运行 ``lmcache trace replay --help`` 获取完整列表。" #: ../../source/mp/tracing_and_debugging.rst:131 msgid "" @@ -183,97 +183,97 @@ msgid "" " recorded gaps races the internal queues and causes non-deterministic " "retrieve misses. If the replay host is slower than the recording host, " "the loop simply lags the recorded schedule." -msgstr "" +msgstr "**节奏控制。** 驱动程序始终遵循记录的调用间隔时间，通过休眠来使每次调度与其记录的 ``t_mono`` 偏移对齐。没有“尽可能快”的模式：``StorageManager`` 的读写是异步的，并且存在跨调用依赖关系（例如，检索可能依赖于先前的 L2 加载完成），因此压缩记录的间隔会导致内部队列竞争，从而导致非确定性的检索未命中。如果重放主机比记录主机慢，循环会简单地滞后于记录的时间表。" #: ../../source/mp/tracing_and_debugging.rst:140 msgid "" "**Output.** Every replay prints a terminal metrics table and writes a " "per-qualname CSV by default:" -msgstr "" +msgstr "**输出。** 每次重放默认都会打印一个终端指标表，并写入每个 qualname 的 CSV：" #: ../../source/mp/tracing_and_debugging.rst:160 msgid "Additional per-record output is controlled by:" -msgstr "" +msgstr "每条记录的附加输出由以下内容控制：" #: ../../source/mp/tracing_and_debugging.rst:166 #: ../../source/mp/tracing_and_debugging.rst:215 msgid "Flag" -msgstr "" +msgstr "标志" #: ../../source/mp/tracing_and_debugging.rst:167 msgid "Purpose" -msgstr "" +msgstr "目的" #: ../../source/mp/tracing_and_debugging.rst:168 msgid "``--output-dir DIR``" -msgstr "" +msgstr "``--output-dir DIR``" #: ../../source/mp/tracing_and_debugging.rst:169 msgid "Directory for aggregated summary files. Default: current dir." -msgstr "" +msgstr "聚合摘要文件的目录。默认：当前目录。" #: ../../source/mp/tracing_and_debugging.rst:170 msgid "``--no-csv``" -msgstr "" +msgstr "``--no-csv``" #: ../../source/mp/tracing_and_debugging.rst:171 msgid "Skip the ``trace_replay_ops.csv`` export." -msgstr "" +msgstr "跳过 ``trace_replay_ops.csv`` 导出。" #: ../../source/mp/tracing_and_debugging.rst:172 msgid "``--json``" -msgstr "" +msgstr "``--json``" #: ../../source/mp/tracing_and_debugging.rst:173 msgid "" "Also write ``trace_replay_summary.json`` (per-qualname count / mean / p50" " / p90 / p99 / min / max, plus total duration)." -msgstr "" +msgstr "还会写入 ``trace_replay_summary.json``（按资格名称的计数 / 平均值 / p50 / p90 / p99 / 最小值 / 最大值，以及总持续时间）。" #: ../../source/mp/tracing_and_debugging.rst:176 msgid "``--verbose``" -msgstr "" +msgstr "``--verbose``" #: ../../source/mp/tracing_and_debugging.rst:177 msgid "" "Print one ``[N/total] OK|FAIL (Xms)`` line per record to " "stdout in addition to the INFO log." -msgstr "" +msgstr "除了 INFO 日志外，还将每条记录打印一行 ``[N/total] OK|FAIL (Xms)`` 到 stdout。" #: ../../source/mp/tracing_and_debugging.rst:179 msgid "``--jsonl-out PATH``" -msgstr "" +msgstr "``--jsonl-out PATH``" #: ../../source/mp/tracing_and_debugging.rst:180 #, python-brace-format msgid "" "Write one JSON object per replayed record to ``PATH`` (``{qualname, " "latency_ms, failed}``) for post-hoc analysis." -msgstr "" +msgstr "将每个重放记录写入一个 JSON 对象到 ``PATH`` (``{qualname, latency_ms, failed}``)，以便进行事后分析。" #: ../../source/mp/tracing_and_debugging.rst:183 msgid "``-q`` / ``--quiet``" -msgstr "" +msgstr "``-q`` / ``--quiet``" #: ../../source/mp/tracing_and_debugging.rst:184 msgid "" "Suppress the terminal metrics table. The aggregated files are still " "written." -msgstr "" +msgstr "抑制终端指标表。聚合文件仍然会被写入。" #: ../../source/mp/tracing_and_debugging.rst:187 msgid "Even without ``--verbose``, the driver logs each dispatch at INFO:" -msgstr "" +msgstr "即使没有 ``--verbose``，驱动程序也会在 INFO 级别记录每个调度。" #: ../../source/mp/tracing_and_debugging.rst:195 msgid "" "Progress numbers come from a cheap pre-scan of the trace file, so you " "always see ``[N/total]`` rather than just a running counter." -msgstr "" +msgstr "进度数字来自对跟踪文件的廉价预扫描，因此您总是看到 ``[N/total]`` 而不仅仅是一个运行计数器。" #: ../../source/mp/tracing_and_debugging.rst:199 msgid "Monitoring During Replay" -msgstr "" +msgstr "重放期间的监控" #: ../../source/mp/tracing_and_debugging.rst:201 msgid "" @@ -282,113 +282,113 @@ msgid "" "lifecycle, eviction ticks, store/retrieve publishes, etc.) therefore flow" " through a live bus during replay and the standard subscribers — logging," " metrics, OTel tracing — can attach to them." -msgstr "" +msgstr "重放驱动程序在构造重放端 ``StorageManager`` 之前初始化了完整可观察性 EventBus。因此，内部事件（L1/L2 生命周期、逐出时钟、存储/检索发布等）在重放期间通过一个实时总线流动，标准订阅者——日志记录、指标、OTel 跟踪——可以附加到这些事件上。" #: ../../source/mp/tracing_and_debugging.rst:208 msgid "" "The same observability CLI flags that the server accepts are available on" " ``lmcache trace replay``:" -msgstr "" +msgstr "在 ``lmcache trace replay`` 上可用的与服务器接受的相同可观察性 CLI 标志：" #: ../../source/mp/tracing_and_debugging.rst:216 msgid "Effect" -msgstr "" +msgstr "效果" #: ../../source/mp/tracing_and_debugging.rst:217 msgid "``--disable-observability``" -msgstr "" +msgstr "``--disable-observability``" #: ../../source/mp/tracing_and_debugging.rst:218 msgid "Turn the EventBus off entirely. No subscribers fire." -msgstr "" +msgstr "完全关闭 EventBus。没有订阅者会触发。" #: ../../source/mp/tracing_and_debugging.rst:219 msgid "``--disable-metrics``" -msgstr "" +msgstr "``--disable-metrics``" #: ../../source/mp/tracing_and_debugging.rst:220 msgid "" "Skip OTel metrics init and metrics subscribers. Useful to avoid binding " "the Prometheus port when you only want logs." -msgstr "" +msgstr "跳过 OTel 指标初始化和指标订阅者。对于仅想要日志时，避免绑定 Prometheus 端口非常有用。" #: ../../source/mp/tracing_and_debugging.rst:222 msgid "``--disable-logging``" -msgstr "" +msgstr "``--disable-logging``" #: ../../source/mp/tracing_and_debugging.rst:223 msgid "Skip logging subscribers." -msgstr "" +msgstr "跳过日志订阅者。" #: ../../source/mp/tracing_and_debugging.rst:224 msgid "``--enable-tracing``" -msgstr "" +msgstr "``--enable-tracing``" #: ../../source/mp/tracing_and_debugging.rst:225 msgid "Enable OTel span subscribers. Requires ``--otlp-endpoint``." -msgstr "" +msgstr "启用 OTel span 订阅者。需要 ``--otlp-endpoint``。" #: ../../source/mp/tracing_and_debugging.rst:226 msgid "``--otlp-endpoint URL``" -msgstr "" +msgstr "``--otlp-endpoint URL``" #: ../../source/mp/tracing_and_debugging.rst:227 msgid "" "Export metrics/traces to an OTLP gRPC collector (e.g. " "``http://localhost:4317``). When unset, metrics fall back to the in-" "process Prometheus pull endpoint." -msgstr "" +msgstr "将指标/追踪导出到 OTLP gRPC 收集器（例如 ``http://localhost:4317``）。如果未设置，指标将回退到进程内的 Prometheus 拉取端点。" #: ../../source/mp/tracing_and_debugging.rst:230 msgid "``--prometheus-port PORT``" -msgstr "" +msgstr "``--prometheus-port PORT``" #: ../../source/mp/tracing_and_debugging.rst:231 msgid "" "Port for the Prometheus ``/metrics`` endpoint in pull mode. Default " "``9090``." -msgstr "" +msgstr "用于拉取模式下 Prometheus ``/metrics`` 端点的端口。默认值为 ``9090``。" #: ../../source/mp/tracing_and_debugging.rst:233 msgid "``--metrics-sample-rate FLOAT``" -msgstr "" +msgstr "``--metrics-sample-rate FLOAT``" #: ../../source/mp/tracing_and_debugging.rst:234 msgid "Sampling rate for lifecycle histograms. Counters always count all events." -msgstr "" +msgstr "生命周期直方图的采样率。计数器始终计算所有事件。" #: ../../source/mp/tracing_and_debugging.rst:237 msgid "Typical monitoring setups:" -msgstr "" +msgstr "典型的监控设置：" #: ../../source/mp/tracing_and_debugging.rst:239 msgid "**Raw log trail (SM/L1/L2 events to stdout):**" -msgstr "" +msgstr "**原始日志轨迹（SM/L1/L2 事件到 stdout）：**" #: ../../source/mp/tracing_and_debugging.rst:247 msgid "**Prometheus metrics in pull mode:**" -msgstr "" +msgstr "**以拉取模式的 Prometheus 指标：**" #: ../../source/mp/tracing_and_debugging.rst:256 msgid "**OTel metrics + traces to a collector:**" -msgstr "" +msgstr "**OTel 指标 + 跟踪到收集器：**" #: ../../source/mp/tracing_and_debugging.rst:267 msgid "" "The ``--trace-level`` and ``--trace-output`` flags are **recording-only**" " and are not accepted by ``lmcache trace replay``. A replay never writes " "a new trace file." -msgstr "" +msgstr "``--trace-level`` 和 ``--trace-output`` 标志是 **仅记录** 的，不被 ``lmcache trace replay`` 接受。重放不会写入新的追踪文件。" #: ../../source/mp/tracing_and_debugging.rst:272 msgid "Notes, Hints, and Caveats" -msgstr "" +msgstr "注意、提示和注意事项" #: ../../source/mp/tracing_and_debugging.rst:274 msgid "" "**Retrieve misses are expected when the replay environment differs.** At " "replay start, the CLI prints a visible warning banner:" -msgstr "" +msgstr "**当重放环境不同时，预期会出现未命中的情况。** 在重放开始时，CLI 会打印一个可见的警告横幅：" #: ../../source/mp/tracing_and_debugging.rst:283 msgid "" @@ -398,7 +398,7 @@ msgid "" "finished by the time the recorded retrieve was issued may still be in " "flight when the replayed retrieve fires. Treat retrieve-miss counts as a " "signal about the replay environment, **not** as a defect in the trace." -msgstr "" +msgstr "由于 KV 负载未被捕获，并且重放端的配置和主机速度可能与录制时不同，因此在录制时命中的检索调用在重放时可能会未命中——例如，在录制的检索发出时已完成的异步 L2 加载在重放的检索触发时可能仍在进行中。将未命中的检索计数视为重放环境的信号，**而不是**跟踪中的缺陷。" #: ../../source/mp/tracing_and_debugging.rst:291 msgid "" @@ -407,7 +407,7 @@ msgid "" "replay-side ``StorageManagerConfig`` differs from what was recorded — " "often exactly what you intended (comparing two configs on the same " "trace)." -msgstr "" +msgstr "**配置摘要不匹配是信息性的，而不是致命的。** 无论摘要是否匹配，重放始终会运行。 不匹配仅告诉您重放端的 ``StorageManagerConfig`` 与记录时不同——这通常正是您所期望的（在同一跟踪上比较两个配置）。" #: ../../source/mp/tracing_and_debugging.rst:297 msgid "" @@ -416,7 +416,7 @@ msgid "" "server — or running two replays at once — on the same port will fail. " "Either pass a different ``--prometheus-port`` or ``--disable-metrics`` on" " the secondary runs." -msgstr "" +msgstr "**Prometheus 端口绑定。** 服务器的 ``--prometheus-port`` 默认值为 ``9090``。在同一端口上与服务器并发运行 ``lmcache trace replay`` 或同时运行两个重放将会失败。请在第二次运行时传递不同的 ``--prometheus-port`` 或 ``--disable-metrics``。" #: ../../source/mp/tracing_and_debugging.rst:303 msgid "" @@ -424,14 +424,14 @@ msgid "" "thread, not the request-handling threads. The gate is a single boolean " "check when disabled (default), so production builds with recording off " "pay no measurable cost." -msgstr "" +msgstr "**跟踪记录开销。** 记录发生在 EventBus 排出线程上，而不是请求处理线程上。当禁用时（默认），门控是一个单一的布尔检查，因此关闭记录的生产构建不会产生可测量的成本。" #: ../../source/mp/tracing_and_debugging.rst:308 msgid "" "**Trace files are not encrypted.** Arguments such as ``ObjectKey`` chunk " "hashes are written in plaintext. Treat trace files with the same care as " "cache hash logs." -msgstr "" +msgstr "**跟踪文件未加密。** 参数如 ``ObjectKey`` 块哈希以明文形式写入。请像对待缓存哈希日志一样小心处理跟踪文件。" #: ../../source/mp/tracing_and_debugging.rst:312 msgid "" @@ -440,7 +440,7 @@ msgid "" "than silently producing garbage. Captured API surface changes (new " "arguments on a traced method, new codec tags) bump the schema version; " "framing changes bump the format version." -msgstr "" +msgstr "**向前兼容性。** 头部携带格式版本和追踪模式版本。读取器会拒绝未知版本的文件，而不是默默地产生垃圾。捕获的 API 表面变化（追踪方法上的新参数、新的编解码器标签）会提升模式版本；框架变化会提升格式版本。" #: ../../source/mp/tracing_and_debugging.rst:318 msgid "" @@ -448,22 +448,22 @@ msgid "" "**levels** (``mq``, ``gpu``). Adding a new traced method in an existing " "level requires only decorating it on the recording side and registering a" " handler on the replay side — no format changes." -msgstr "" +msgstr "**可扩展性。** 该格式旨在适应未来的追踪 **级别**（``mq``、``gpu``）。在现有级别中添加一个新的追踪方法只需在记录端进行装饰，并在重放端注册一个处理程序 — 无需格式更改。" #: ../../source/mp/tracing_and_debugging.rst:324 msgid "See Also" -msgstr "" +msgstr "另请参阅" #: ../../source/mp/tracing_and_debugging.rst:326 msgid "" ":ref:`trace-recording` — the short ``Trace Recording`` section in the " "Observability page focuses on the recording-side flags." -msgstr "" +msgstr ":ref:`trace-recording` — 可观察性页面中的短小 ``Trace Recording`` 部分专注于记录端标志。" #: ../../source/mp/tracing_and_debugging.rst:328 msgid "" "``docs/design/v1/mp_observability/trace.md`` in the source tree — full " "design doc: architecture, replay dispatcher, context-manager pairing, " "stats collector, and test matrix." -msgstr "" +msgstr "``docs/design/v1/mp_observability/trace.md`` 在源代码树中 — 完整设计文档：架构、重放调度器、上下文管理器配对、统计收集器和测试矩阵。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/encoder_cache.po b/docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/encoder_cache.po index 08406d1c978..3c1fddffecb 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/encoder_cache.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/encoder_cache.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/non_kv_cache/encoder_cache.rst:2 msgid "Encoder caching" -msgstr "" +msgstr "编码器缓存" #: ../../source/non_kv_cache/encoder_cache.rst:4 msgid "" @@ -30,7 +30,7 @@ msgid "" "share a multimodal input — the same image, video, or audio clip — the " "second request loads the encoder output from the cache and the encoder " "does not run." -msgstr "" +msgstr "**编码器缓存 (EC)** 存储多模态模型编码器阶段的输出，按 vLLM 的每个输入 ``mm_hash`` 进行索引。当两个请求共享一个多模态输入——相同的图像、视频或音频片段——第二个请求将从缓存中加载编码器输出，而编码器不会运行。" #: ../../source/non_kv_cache/encoder_cache.rst:10 msgid "" @@ -41,7 +41,7 @@ msgid "" " connector is modality-agnostic — it caches a tensor of shape " "``[num_tokens, hidden_dim]`` keyed by ``mm_hash``, without knowing which " "encoder produced it." -msgstr "" +msgstr "这适用于 vLLM 通过其编码器缓存扩展点暴露的任何模态：视觉编码器（用于图像和采样视频帧的 CLIP / ViT 风格塔）、音频编码器（用于原始波形的 Whisper 风格塔）以及诸如 Qwen2.5-Omni 的组合模态编码器。连接器是模态无关的——它缓存一个形状为 ``[num_tokens, hidden_dim]`` 的张量，键为 ``mm_hash``，而不知道是哪个编码器生成的。" #: ../../source/non_kv_cache/encoder_cache.rst:18 msgid "" @@ -50,27 +50,27 @@ msgid "" "vLLM side and an ``ECCacheEngine`` on the LMCache side; together they " "back the encoder cache with any of LMCache's storage backends (local CPU," " local disk, remote, NIXL)." -msgstr "" +msgstr "vLLM 通过 ``ECConnectorBase``（仅限 vLLM v1）暴露了编码器缓存扩展点。LMCache 在 vLLM 端提供了一个 ``LMCacheECConnector`` 适配器，在 LMCache 端提供了一个 ``ECCacheEngine``；它们共同支持使用 LMCache 的任何存储后端（本地 CPU、本地磁盘、远程、NIXL）来备份编码器缓存。" #: ../../source/non_kv_cache/encoder_cache.rst:25 msgid "Enabling it" -msgstr "" +msgstr "启用它" #: ../../source/non_kv_cache/encoder_cache.rst:27 msgid "Pass ``--ec-transfer-config`` to ``vllm serve``:" -msgstr "" +msgstr "将 ``--ec-transfer-config`` 传递给 ``vllm serve``：" #: ../../source/non_kv_cache/encoder_cache.rst:38 msgid "" "``ec_role`` choices: ``ec_producer`` (saves only), ``ec_consumer`` (reads" " only), ``ec_both`` (single-instance default)." -msgstr "" +msgstr "``ec_role`` 选项： ``ec_producer``（仅保存）， ``ec_consumer``（仅读取）， ``ec_both``（单实例默认）。" #: ../../source/non_kv_cache/encoder_cache.rst:41 msgid "" "Set ``LMCACHE_CONFIG_FILE`` to point at a YAML with at least one storage " "backend configured for EC:" -msgstr "" +msgstr "将 ``LMCACHE_CONFIG_FILE`` 设置为指向一个配置了至少一个 EC 存储后端的 YAML 文件：" #: ../../source/non_kv_cache/encoder_cache.rst:52 msgid "" @@ -79,7 +79,7 @@ msgid "" "(e.g. ``ec_max_local_disk_size: 64`` or " "``LMCACHE_EC_MAX_LOCAL_DISK_SIZE=64``). EC and KV always run with " "**separate** ``StorageManager`` instances so one cannot evict the other." -msgstr "" +msgstr "要独立于（单独的）KV Cache 调整 EC 存储大小，请在 YAML 中使用 ``ec_`` 前缀覆盖，或在环境中使用 ``LMCACHE_EC_`` 前缀（例如 ``ec_max_local_disk_size: 64`` 或 ``LMCACHE_EC_MAX_LOCAL_DISK_SIZE=64``）。EC 和 KV 始终在 **独立** 的 ``StorageManager`` 实例中运行，因此一个不能逐出另一个。" #: ../../source/non_kv_cache/encoder_cache.rst:59 msgid "" @@ -87,27 +87,27 @@ msgid "" "starts, but EC entries live in CPU memory only and do not survive process" " restart. Set ``local_disk`` (or ``ec_local_disk``) to a real path if you" " want cache persistence — there is no implicit on-disk default location." -msgstr "" +msgstr "如果您不设置 ``local_disk``（或其 EC 覆盖），引擎仍然会启动，但 EC 条目仅存在于 CPU 内存中，并且在进程重启时不会保留。如果您希望缓存持久化，请将 ``local_disk``（或 ``ec_local_disk``）设置为一个真实路径——没有隐式的磁盘默认位置。" #: ../../source/non_kv_cache/encoder_cache.rst:66 msgid "Verifying it's working" -msgstr "" +msgstr "验证它是否正常工作" #: ../../source/non_kv_cache/encoder_cache.rst:68 msgid "Three independent signals:" -msgstr "" +msgstr "三个独立信号：" #: ../../source/non_kv_cache/encoder_cache.rst:70 msgid "" "**vLLM metric.** ``loggers.py`` reports ``MM cache hit rate: X%`` after " "warm requests." -msgstr "" +msgstr "**vLLM 指标。** ``loggers.py`` 在预热请求后报告 ``MM 缓存命中率：X%``。" #: ../../source/non_kv_cache/encoder_cache.rst:72 msgid "" "**LMCache log line.** Cold (first-time) requests emit ``LMCache INFO: EC " "put: stored N bytes for mm_hash=H``. Warm requests emit no ``EC put``." -msgstr "" +msgstr "**LMCache 日志行。** 冷（首次）请求会发出 ``LMCache INFO: EC put: stored N bytes for mm_hash=H``。热请求不会发出 ``EC put``。" #: ../../source/non_kv_cache/encoder_cache.rst:75 msgid "" @@ -116,11 +116,11 @@ msgid "" "and is reused thereafter. The ``@1@0@`` prefix reflects sentinel " "``world_size=1, worker_id=0`` in the EC cache key, so all tensor-parallel" " ranks share one entry." -msgstr "" +msgstr "**磁盘文件。** 在 ``local_disk`` 下，形式为 ``@1@0@@.pt`` 的条目在第一次请求后出现，并在此后被重用。``@1@0@`` 前缀反映了 EC 缓存键中的哨兵 ``world_size=1, worker_id=0``，因此所有张量并行的 rank 共享一个条目。" #: ../../source/non_kv_cache/encoder_cache.rst:82 msgid "Design notes (user-visible)" -msgstr "" +msgstr "设计说明（用户可见）" #: ../../source/non_kv_cache/encoder_cache.rst:84 msgid "" @@ -128,7 +128,7 @@ msgid "" "across TP ranks, so the EC key uses ``world_size=1, worker_id=0`` " "regardless of the deployment's actual TP. Concurrent puts from N ranks " "land on the same key with identical contents." -msgstr "" +msgstr "**缓存键使用哨兵 TP 形状。** 编码器输出在 TP 排中复制，因此 EC 键使用 ``world_size=1, worker_id=0``，而不管部署的实际 TP。来自 N 个排名的并发放置会落在相同的键上，内容相同。" #: ../../source/non_kv_cache/encoder_cache.rst:89 msgid "" @@ -136,7 +136,7 @@ msgid "" " the encoder output dtype (``vllm_config.model_config.dtype``), not " "``metadata.kv_dtype``. Changing KV quantization does not invalidate EC " "entries." -msgstr "" +msgstr "**数据类型与 KV 量化解耦。** EC 缓存键的 dtype 字段是编码器输出的数据类型（``vllm_config.model_config.dtype``），而不是 ``metadata.kv_dtype``。更改 KV 量化不会使 EC 条目失效。" #: ../../source/non_kv_cache/encoder_cache.rst:93 msgid "" @@ -145,7 +145,7 @@ msgid "" "request-scoped). Sharing one allocator pool would let one cache evict the" " other in unpredictable ways. Per-cache sizing knobs (``ec_max_local_*``)" " are explicit instead." -msgstr "" +msgstr "**将 StorageManager 与 KV 分离。** KV 和 EC 的访问模式非常不同（KV 是分块/逐层/高吞吐量；EC 是单张张量/请求范围）。共享一个分配器池会导致一个缓存以不可预测的方式逐出另一个缓存。因此，针对每个缓存的大小调整参数（``ec_max_local_*``）是明确的。" #: ../../source/non_kv_cache/encoder_cache.rst:98 msgid "" @@ -155,17 +155,17 @@ msgid "" " scheduler-side ``has_cache_item`` needs a fully constructed " "``StorageManager`` and LMCache currently aborts disk-backend setup when " "``metadata.role == \"scheduler\"``." -msgstr "" +msgstr "**连接器角色固定为“worker”。** vLLM 的 ``ECConnectorBase`` 是双角色（调度器和工作者）。LMCache 连接器无论如何都会调用 ``create_lmcache_metadata(..., role=\\\"worker\\\")``，因为调度器端的 ``has_cache_item`` 需要一个完全构造的 ``StorageManager``，而 LMCache 当前在 ``metadata.role == \\\"scheduler\\\"`` 时会中止磁盘后端的设置。" #: ../../source/non_kv_cache/encoder_cache.rst:105 msgid "" "The full internal design (class layering, code paths, follow-up work) " "lives at :file:`docs/design/v1/encoder-cache.md` in the source tree." -msgstr "" +msgstr "完整的内部设计（类层次、代码路径、后续工作）位于源代码树中的 :file:`docs/design/v1/encoder-cache.md`。" #: ../../source/non_kv_cache/encoder_cache.rst:109 msgid "Benchmark" -msgstr "" +msgstr "基准测试" #: ../../source/non_kv_cache/encoder_cache.rst:111 msgid "" @@ -173,85 +173,85 @@ msgid "" "Instruct`` (bf16) and Big Buck Bunny (10:34, 720p, ≈ 60 MB MP4). Same " "chat-completion request sent 1 cold + N warm times against the same vLLM " "server." -msgstr "" +msgstr "在一台单个 H100 80GB 上进行实时测量，使用 ``Qwen/Qwen2.5-VL-7B-Instruct``（bf16）和《大雄兔的故事》（10:34，720p，≈ 60 MB MP4）。对同一 vLLM 服务器发送相同的聊天完成请求，1 次冷启动 + N 次热启动。" #: ../../source/non_kv_cache/encoder_cache.rst:116 msgid "" "Two configurations, varying only ``num_frames`` (how many frames vLLM " "samples from the video):" -msgstr "" +msgstr "两个配置，仅改变 ``num_frames``（vLLM 从视频中采样的帧数）：" #: ../../source/non_kv_cache/encoder_cache.rst:123 msgid "num_frames" -msgstr "" +msgstr "num_frames" #: ../../source/non_kv_cache/encoder_cache.rst:124 msgid "EC entry" -msgstr "" +msgstr "EC 条目" #: ../../source/non_kv_cache/encoder_cache.rst:125 msgid "Cold TTFT (s)" -msgstr "" +msgstr "冷 TTFT (秒)" #: ../../source/non_kv_cache/encoder_cache.rst:126 msgid "Warm TTFT mean (s)" -msgstr "" +msgstr "温暖 TTFT 平均值 (秒)" #: ../../source/non_kv_cache/encoder_cache.rst:127 msgid "Saved" -msgstr "" +msgstr "已保存" #: ../../source/non_kv_cache/encoder_cache.rst:128 msgid "Speedup" -msgstr "" +msgstr "加速" #: ../../source/non_kv_cache/encoder_cache.rst:129 msgid "32 (vLLM default)" -msgstr "" +msgstr "32 (vLLM 默认)" #: ../../source/non_kv_cache/encoder_cache.rst:130 msgid "34.3 MB" -msgstr "" +msgstr "34.3 MB" #: ../../source/non_kv_cache/encoder_cache.rst:131 msgid "3.923" -msgstr "" +msgstr "3.923" #: ../../source/non_kv_cache/encoder_cache.rst:132 msgid "3.125" -msgstr "" +msgstr "3.125" #: ../../source/non_kv_cache/encoder_cache.rst:133 msgid "798 ms" -msgstr "" +msgstr "798 毫秒" #: ../../source/non_kv_cache/encoder_cache.rst:134 msgid "**1.26×**" -msgstr "" +msgstr "**1.26×**" #: ../../source/non_kv_cache/encoder_cache.rst:135 msgid "128" -msgstr "" +msgstr "128" #: ../../source/non_kv_cache/encoder_cache.rst:136 msgid "130.8 MB" -msgstr "" +msgstr "130.8 MB" #: ../../source/non_kv_cache/encoder_cache.rst:137 msgid "5.895" -msgstr "" +msgstr "5.895" #: ../../source/non_kv_cache/encoder_cache.rst:138 msgid "3.375" -msgstr "" +msgstr "3.375" #: ../../source/non_kv_cache/encoder_cache.rst:139 msgid "2.52 s" -msgstr "" +msgstr "2.52 秒" #: ../../source/non_kv_cache/encoder_cache.rst:140 msgid "**1.75×**" -msgstr "" +msgstr "**1.75×**" #: ../../source/non_kv_cache/encoder_cache.rst:142 msgid "" @@ -262,15 +262,15 @@ msgid "" "the encoder is the dominant share of prefill (long videos at high frame " "counts, long audio clips, large images at high resolution) and smallest " "when text prefill dominates." -msgstr "" +msgstr "随着 ``num_frames`` 的增加，速度提升也会增加，因为编码器的工作负载与帧数呈线性扩展，而其余的 Prefill（对生成的多模态令牌和短文本提示的 LM 前向推理）则呈亚线性扩展。相同的原理适用于其他模态：当编码器占据 Prefill 的主导份额时（高帧数的长视频、长音频片段、高分辨率的大图像），收益最大；而当文本 Prefill 占主导时，收益最小。" #: ../../source/non_kv_cache/encoder_cache.rst:151 msgid "Reproducing" -msgstr "" +msgstr "再现" #: ../../source/non_kv_cache/encoder_cache.rst:153 msgid "Server (heavier-encoder configuration):" -msgstr "" +msgstr "服务器（重型编码器配置）：" #: ../../source/non_kv_cache/encoder_cache.rst:169 msgid "" @@ -278,5 +278,5 @@ msgid "" "multimodal payload. The benchmark measures TTFT (time to first token) " "because the encoder runs during prefill — any encoder savings show up " "there. Decode tokens-per-second is unaffected by EC." -msgstr "" +msgstr "客户端：任何重新发送相同多模态负载的流式 OpenAI 兼容客户端。基准测试测量 TTFT（首次令牌时间），因为编码器在 Prefill 期间运行——任何编码器节省都在此处显现。解码每秒令牌数不受 EC 影响。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/docker_deployment.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/docker_deployment.po index 1ad304238e4..da1a38499b1 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/docker_deployment.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/docker_deployment.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: LMCache \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2026-05-18 17:25+0000\n" +"POT-Creation-Date: 2026-05-19 20:13+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language: zh_CN\n" @@ -21,29 +21,29 @@ msgstr "" #: ../../source/production/docker_deployment.rst:4 msgid "Docker deployment" -msgstr "" +msgstr "Docker 部署" #: ../../source/production/docker_deployment.rst:6 msgid "**Prerequisites:** Docker Engine 27.0+" -msgstr "" +msgstr "**前提条件：** Docker Engine 27.0+" #: ../../source/production/docker_deployment.rst:8 msgid "See :ref:`installation_guide` for pulling images." -msgstr "" +msgstr "请参阅 :ref:`installation_guide` 以获取拉取镜像的信息。" #: ../../source/production/docker_deployment.rst:11 msgid "Running the container" -msgstr "" +msgstr "运行容器" #: ../../source/production/docker_deployment.rst:27 msgid "" "See the `docker run example " "`_ for more details." -msgstr "" +msgstr "有关更多详细信息，请参见 `docker run 示例 `_。" #: ../../source/production/docker_deployment.rst:30 msgid "ROCm (AMD)" -msgstr "" +msgstr "ROCm (AMD)" #: ../../source/production/docker_deployment.rst:32 msgid "" @@ -53,27 +53,36 @@ msgid "" "`__ for full " "instructions." -msgstr "" +msgstr "`AMD Infinity hub `__ 为 vLLM 提供了一个预构建的、优化的镜像，适用于 AMD Instinct™ MI300X。有关完整说明，请参见 `LLM 推理性能验证在 AMD Instinct MI300X 上 `__。" #: ../../source/production/docker_deployment.rst:37 msgid "" "Validated environment: ``rocm/vllm-" "dev:nightly_0624_rc2_0624_rc2_20250620``, MI300X, vLLM V1." -msgstr "" +msgstr "验证的环境：``rocm/vllm-dev:nightly_0624_rc2_0624_rc2_20250620``, MI300X, vLLM V1." #: ../../source/production/docker_deployment.rst:56 msgid "XPU (Intel)" -msgstr "" +msgstr "XPU（英特尔）" #: ../../source/production/docker_deployment.rst:58 msgid "" -"The `Intel vLLM XPU hub`__ offers a " -"prebuilt, optimized docker image designed for validating inference " +"The `Intel vLLM XPU hub `__ offers a" +" prebuilt, optimized docker image designed for validating inference " "performance on the Intel GPU accelerators like ARC770, B60/B70, and " "future products." -msgstr "" +msgstr "`Intel vLLM XPU hub `__ 提供了一个预构建的、优化的 Docker 镜像，旨在验证在 Intel GPU 加速器（如 ARC770、B60/B70 及未来产品）上的推理性能。" #: ../../source/production/docker_deployment.rst:60 msgid "Validated environment: ``intel/vllm:0.17.0-xpu``, Intel B60 GPU, vLLM V1." -msgstr "" +msgstr "验证环境：``intel/vllm:0.17.0-xpu``, Intel B60 GPU, vLLM V1。" + +#~ msgid "" +#~ "The `Intel vLLM XPU " +#~ "hub`__ offers a " +#~ "prebuilt, optimized docker image designed " +#~ "for validating inference performance on " +#~ "the Intel GPU accelerators like ARC770," +#~ " B60/B70, and future products." +#~ msgstr "" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/kubernetes_deployment.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/kubernetes_deployment.po index b65fd6efbbd..251ab80a301 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/kubernetes_deployment.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/kubernetes_deployment.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/production/kubernetes_deployment.rst:2 msgid "Kubernetes Deployment" -msgstr "" +msgstr "Kubernetes 部署" #: ../../source/production/kubernetes_deployment.rst:4 msgid "" @@ -29,30 +29,30 @@ msgid "" "using the `vLLM Production Stack `_ project. This is a specialized production-ready " "implementation for K8S-native cluster-wide deployment for vllm & lmcache." -msgstr "" +msgstr "对于集成 LMCache 的 vLLM 的 Kubernetes 部署，我们建议使用 `vLLM Production Stack `_ 项目。这是一个专门为 vllm 和 lmcache 提供的 K8S 原生集群范围的生产就绪实现。" #: ../../source/production/kubernetes_deployment.rst:6 msgid "" "For a quick start guide, please refer to the official `documentation " "`_" -msgstr "" +msgstr "有关快速入门指南，请参阅官方 `文档 `_。" #: ../../source/production/kubernetes_deployment.rst:8 msgid "" "and replace the Helm values file with (`values-05-cpu-offloading.yaml " "`_):" -msgstr "" +msgstr "并用 (`values-05-cpu-offloading.yaml `_) 替换 Helm 值文件：" #: ../../source/production/kubernetes_deployment.rst:35 msgid "OR" -msgstr "" +msgstr "或" #: ../../source/production/kubernetes_deployment.rst:37 msgid "" "refer to a detailed `step-by-step tutorial `_ on" " how to offload KV cache with LMCache in the production stack." -msgstr "" +msgstr "请参阅有关如何在生产环境中使用 LMCache 卸载 KV Cache 的详细 `逐步教程 `_。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/kv_cache_events.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/kv_cache_events.po index 134b07112c4..aa1eed57c5c 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/kv_cache_events.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/kv_cache_events.po @@ -21,85 +21,85 @@ msgstr "" #: ../../source/production/kv_cache_events.rst:4 msgid "KV Cache Events" -msgstr "" +msgstr "KV Cache 事件" #: ../../source/production/kv_cache_events.rst:6 msgid "" "KV cache events are actions or lifecycle events that occur when managing " "the KV cache during inference. These events can be used for KV-cache-" "aware routing." -msgstr "" +msgstr "KV Cache 事件是在推理过程中管理 KV Cache 时发生的操作或生命周期事件。这些事件可用于 KV Cache 感知路由。" #: ../../source/production/kv_cache_events.rst:8 msgid "LMCache supports KV cache events as follows:" -msgstr "" +msgstr "LMCache 支持以下 KV 缓存事件：" #: ../../source/production/kv_cache_events.rst:10 msgid "Generates storage KV cache events" -msgstr "" +msgstr "生成存储 KV Cache 事件" #: ../../source/production/kv_cache_events.rst:11 msgid "" "The events format is defined as per the `BlockStored class " "`_ in vLLM" -msgstr "" +msgstr "事件格式根据 vLLM 中的 `BlockStored class `_ 定义。" #: ../../source/production/kv_cache_events.rst:12 msgid "" "LMCache passes the events to SGLang or vLLM to publish them using their " "messaging system" -msgstr "" +msgstr "LMCache 将事件传递给 SGLang 或 vLLM，以通过它们的消息系统发布这些事件。" #: ../../source/production/kv_cache_events.rst:15 msgid "Prerequisites" -msgstr "" +msgstr "先决条件" #: ../../source/production/kv_cache_events.rst:17 msgid "The following prerequisites are required:" -msgstr "" +msgstr "以下先决条件是必需的：" #: ../../source/production/kv_cache_events.rst msgid "vLLM" -msgstr "" +msgstr "vLLM" #: ../../source/production/kv_cache_events.rst:23 msgid "vLLM v0.13.0+" -msgstr "" +msgstr "vLLM v0.13.0+" #: ../../source/production/kv_cache_events.rst:24 msgid "LMCache v0.3.11+" -msgstr "" +msgstr "LMCache v0.3.11+" #: ../../source/production/kv_cache_events.rst msgid "SGLang" -msgstr "" +msgstr "SGLang" #: ../../source/production/kv_cache_events.rst:28 msgid "SGLang vx.y.z+" -msgstr "" +msgstr "SGLang vx.y.z+" #: ../../source/production/kv_cache_events.rst:29 msgid "LMCache vx.y.z+" -msgstr "" +msgstr "LMCache vx.y.z+" #: ../../source/production/kv_cache_events.rst:32 msgid "How to Generate KV Cache events" -msgstr "" +msgstr "如何生成 KV Cache 事件" #: ../../source/production/kv_cache_events.rst:38 #: ../../source/production/kv_cache_events.rst:89 msgid "" "Before starting to generate KV events, you need to be aware of the " "following:" -msgstr "" +msgstr "在开始生成 KV 事件之前，您需要了解以下内容：" #: ../../source/production/kv_cache_events.rst:40 #: ../../source/production/kv_cache_events.rst:91 msgid "" "You need to enable ``enable_kv_events`` for LMCache as events are not " "generated by default." -msgstr "" +msgstr "您需要为 LMCache 启用 ``enable_kv_events``，因为默认情况下不会生成事件。" #: ../../source/production/kv_cache_events.rst:41 msgid "" @@ -108,7 +108,7 @@ msgid "" " hashes generated per worker are the same. If not then you will have " "duplicate events for the same operation as events are generated per " "worker." -msgstr "" +msgstr "如果在 vLLM 中运行多个工作进程，则需要使用非默认的哈希算法（在 LMCache 中设置 ``pre_caching_hash_algorithm``），以确保每个工作进程生成的哈希相同。如果不这样做，则会为相同操作生成重复事件，因为事件是按工作进程生成的。" #: ../../source/production/kv_cache_events.rst:42 msgid "" @@ -117,23 +117,23 @@ msgid "" "config``. See `vLLM KV Events configuration " "`_ for more " "details." -msgstr "" +msgstr "LMCache 将事件发送到 vLLM 进行发布。要启用事件发布，您需要设置 vLLM 配置选项 ``--kv-events-config``。有关更多详细信息，请参见 `vLLM KV Events configuration `_。" #: ../../source/production/kv_cache_events.rst:44 #: ../../source/production/kv_cache_events.rst:94 msgid "" "The steps that follow give an example of how KV events can be generated, " "published and consumed:" -msgstr "" +msgstr "接下来的步骤给出了如何生成、发布和消费 KV 事件的示例：" #: ../../source/production/kv_cache_events.rst:46 msgid "Start vLLM with LMCache and model ``Qwen/Qwen3-0.6B`` as follows:" -msgstr "" +msgstr "以如下方式启动 vLLM，使用 LMCache 和模型 ``Qwen/Qwen3-0.6B``：" #: ../../source/production/kv_cache_events.rst:54 #: ../../source/production/kv_cache_events.rst:107 msgid "Example of the LMCache configuration is as follows:" -msgstr "" +msgstr "LMCache 配置示例如下：" #: ../../source/production/kv_cache_events.rst:63 msgid "" @@ -142,35 +142,35 @@ msgid "" "the events. vLLM provides such a client example `KV Events Subscriber " "`_." " Run this python script in a separate terminal." -msgstr "" +msgstr "要处理 vLLM 发布的事件，您需要一个订阅发布者消息通道并能够消费事件的客户端。vLLM 提供了这样的客户端示例 `KV Events Subscriber `_。在单独的终端中运行此 Python 脚本。" #: ../../source/production/kv_cache_events.rst:65 #: ../../source/production/kv_cache_events.rst:119 msgid "Prompt the model:" -msgstr "" +msgstr "提示模型：" #: ../../source/production/kv_cache_events.rst:78 #: ../../source/production/kv_cache_events.rst:133 msgid "" "You should receive a message in the client (that you started in step 2.) " "window, similar to the following:" -msgstr "" +msgstr "您应该在客户端（您在第 2 步中启动的客户端）窗口中收到一条消息，类似于以下内容：" #: ../../source/production/kv_cache_events.rst:85 #: ../../source/production/kv_cache_events.rst:155 msgid "This is the event generated after the cache store operation." -msgstr "" +msgstr "这是在缓存存储操作后生成的事件。" #: ../../source/production/kv_cache_events.rst:92 msgid "" "LMCache sends the events to SGLang for publishing. To enable events to be" " published, you need to set the SGLang configuration setting ``--kv-" "events-config``." -msgstr "" +msgstr "LMCache 将事件发送到 SGLang 进行发布。要启用事件发布，您需要设置 SGLang 配置选项 ``--kv-events-config``。" #: ../../source/production/kv_cache_events.rst:96 msgid "Start SGLang with LMCache and model ``Qwen/Qwen3-0.6B`` as follows:" -msgstr "" +msgstr "使用 LMCache 和模型 ``Qwen/Qwen3-0.6B`` 启动 SGLang，如下所示：" #: ../../source/production/kv_cache_events.rst:117 msgid "" @@ -182,5 +182,5 @@ msgid "" "``medium`` and ``lora_name`` from the ``BlockStored`` class definition " "and ``medium`` from ``BlockRemoved`` class definition. Save the changes " "and run this updated python script in a separate terminal." -msgstr "" +msgstr "要处理 SGLang 发布的事件，您需要一个订阅发布者消息通道并能够消费事件的客户端。vLLM 提供了这样一个客户端示例 `KV Events Subscriber `_。要将此客户端用于 SGLang，您需要从 ``BlockStored`` 类定义中删除属性 ``medium`` 和 ``lora_name``，并从 ``BlockRemoved`` 类定义中删除 ``medium``。保存更改并在单独的终端中运行此更新的 Python 脚本。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/chunk_statistics.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/chunk_statistics.po index 290bbd57f3c..b6930b32f36 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/chunk_statistics.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/chunk_statistics.po @@ -21,560 +21,560 @@ msgstr "" #: ../../source/production/observability/chunk_statistics.rst:4 msgid "Chunk Statistics" -msgstr "" +msgstr "块统计" #: ../../source/production/observability/chunk_statistics.rst:6 msgid "" "The chunk statistics feature provides insights into KV cache chunk reuse " "patterns, helping you understand cache efficiency and optimize your " "deployment." -msgstr "" +msgstr "块统计功能提供了对 KV Cache 块重用模式的洞察，帮助您理解缓存效率并优化您的部署。" #: ../../source/production/observability/chunk_statistics.rst:9 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/production/observability/chunk_statistics.rst:11 msgid "" "Chunk statistics tracks and analyzes KV cache chunks to provide metrics " "on:" -msgstr "" +msgstr "块统计跟踪和分析 KV Cache 块，以提供以下指标：" #: ../../source/production/observability/chunk_statistics.rst:13 msgid "" "**Total chunks processed**: The total number of chunks that have been " "processed" -msgstr "" +msgstr "**处理的总块数**：已处理的块的总数" #: ../../source/production/observability/chunk_statistics.rst:14 msgid "**Unique chunks**: The number of distinct chunks encountered" -msgstr "" +msgstr "**唯一块**：遇到的不同块的数量" #: ../../source/production/observability/chunk_statistics.rst:15 msgid "**Duplicate chunks**: The number of repeated chunks" -msgstr "" +msgstr "**重复块**：重复块的数量" #: ../../source/production/observability/chunk_statistics.rst:16 msgid "" "**Reuse rate**: The ratio of duplicate chunks to total chunks, indicating" " cache efficiency" -msgstr "" +msgstr "**重用率**：重复块与总块的比例，指示缓存效率" #: ../../source/production/observability/chunk_statistics.rst:18 msgid "This information is valuable for:" -msgstr "" +msgstr "此信息对以下内容很有价值：" #: ../../source/production/observability/chunk_statistics.rst:20 msgid "Understanding cache hit patterns" -msgstr "" +msgstr "理解缓存命中模式" #: ../../source/production/observability/chunk_statistics.rst:21 msgid "Optimizing cache size and eviction policies" -msgstr "" +msgstr "优化缓存大小和逐出策略" #: ../../source/production/observability/chunk_statistics.rst:22 msgid "Analyzing workload characteristics" -msgstr "" +msgstr "分析工作负载特征" #: ../../source/production/observability/chunk_statistics.rst:23 msgid "Capacity planning for production deployments" -msgstr "" +msgstr "生产部署的容量规划" #: ../../source/production/observability/chunk_statistics.rst:26 msgid "Recording Strategies" -msgstr "" +msgstr "记录策略" #: ../../source/production/observability/chunk_statistics.rst:28 msgid "" "LMCache supports multiple recording strategies, each optimized for " "different use cases:" -msgstr "" +msgstr "LMCache 支持多种记录策略，每种策略都针对不同的使用案例进行了优化：" #: ../../source/production/observability/chunk_statistics.rst:31 msgid "Memory Bloom Filter Strategy" -msgstr "" +msgstr "内存布隆过滤器策略" #: ../../source/production/observability/chunk_statistics.rst:33 msgid "" "A memory-efficient strategy using Bloom filters for probabilistic " "duplicate detection." -msgstr "" +msgstr "一种使用布隆过滤器进行概率重复检测的内存高效策略。" #: ../../source/production/observability/chunk_statistics.rst:35 #: ../../source/production/observability/chunk_statistics.rst:64 msgid "**Advantages:**" -msgstr "" +msgstr "**优点：**" #: ../../source/production/observability/chunk_statistics.rst:37 msgid "Low memory footprint" -msgstr "" +msgstr "低内存占用" #: ../../source/production/observability/chunk_statistics.rst:38 msgid "Fast lookup operations" -msgstr "" +msgstr "快速查找操作" #: ../../source/production/observability/chunk_statistics.rst:39 msgid "Suitable for large-scale deployments" -msgstr "" +msgstr "适合大规模部署" #: ../../source/production/observability/chunk_statistics.rst:41 #: ../../source/production/observability/chunk_statistics.rst:70 msgid "**Configuration:**" -msgstr "" +msgstr "**配置：**" #: ../../source/production/observability/chunk_statistics.rst:51 #: ../../source/production/observability/chunk_statistics.rst:81 msgid "**Environment Variables:**" -msgstr "" +msgstr "**环境变量：**" #: ../../source/production/observability/chunk_statistics.rst:60 msgid "File Hash Strategy" -msgstr "" +msgstr "文件哈希策略" #: ../../source/production/observability/chunk_statistics.rst:62 msgid "" "A file-based strategy that writes chunk hashes to disk for exact tracking" " and offline analysis." -msgstr "" +msgstr "一种基于文件的策略，将块哈希写入磁盘，以便进行精确跟踪和离线分析。" #: ../../source/production/observability/chunk_statistics.rst:66 msgid "Exact duplicate detection (no false positives)" -msgstr "" +msgstr "精确重复检测（无误报）" #: ../../source/production/observability/chunk_statistics.rst:67 msgid "Persistent storage for offline analysis" -msgstr "" +msgstr "离线分析的持久存储" #: ../../source/production/observability/chunk_statistics.rst:68 msgid "Automatic file rotation and cleanup" -msgstr "" +msgstr "自动文件轮换和清理" #: ../../source/production/observability/chunk_statistics.rst:90 msgid "Quick Start Guide" -msgstr "" +msgstr "快速入门指南" #: ../../source/production/observability/chunk_statistics.rst:93 msgid "Step 1: Enable Chunk Statistics" -msgstr "" +msgstr "步骤 1：启用块统计信息" #: ../../source/production/observability/chunk_statistics.rst:95 msgid "Configure your LMCache instance with chunk statistics enabled:" -msgstr "" +msgstr "配置您的 LMCache 实例以启用块统计信息：" #: ../../source/production/observability/chunk_statistics.rst:97 msgid "**Using YAML Configuration:**" -msgstr "" +msgstr "**使用 YAML 配置：**" #: ../../source/production/observability/chunk_statistics.rst:118 msgid "**Using vLLM with LMCache:**" -msgstr "" +msgstr "**使用 vLLM 和 LMCache:**" #: ../../source/production/observability/chunk_statistics.rst:135 msgid "Step 2: Access Statistics" -msgstr "" +msgstr "步骤 2：访问统计信息" #: ../../source/production/observability/chunk_statistics.rst:137 msgid "Retrieve statistics through the internal API server:" -msgstr "" +msgstr "通过内部 API 服务器检索统计信息：" #: ../../source/production/observability/chunk_statistics.rst:144 msgid "**Example Response:**" -msgstr "" +msgstr "**示例响应：**" #: ../../source/production/observability/chunk_statistics.rst:187 msgid "Configuration Options" -msgstr "" +msgstr "配置选项" #: ../../source/production/observability/chunk_statistics.rst:190 msgid "Basic Configuration" -msgstr "" +msgstr "基本配置" #: ../../source/production/observability/chunk_statistics.rst:192 msgid "Basic Configuration Options" -msgstr "" +msgstr "基本配置选项" #: ../../source/production/observability/chunk_statistics.rst:196 #: ../../source/production/observability/chunk_statistics.rst:224 #: ../../source/production/observability/chunk_statistics.rst:243 msgid "Configuration Key" -msgstr "" +msgstr "配置键" #: ../../source/production/observability/chunk_statistics.rst:197 #: ../../source/production/observability/chunk_statistics.rst:225 #: ../../source/production/observability/chunk_statistics.rst:244 msgid "Default Value" -msgstr "" +msgstr "默认值" #: ../../source/production/observability/chunk_statistics.rst:198 #: ../../source/production/observability/chunk_statistics.rst:226 #: ../../source/production/observability/chunk_statistics.rst:245 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/production/observability/chunk_statistics.rst:199 msgid "``enable_chunk_statistics``" -msgstr "" +msgstr "``enable_chunk_statistics``" #: ../../source/production/observability/chunk_statistics.rst:200 #: ../../source/production/observability/chunk_statistics.rst:206 msgid "``false``" -msgstr "" +msgstr "``false``" #: ../../source/production/observability/chunk_statistics.rst:201 msgid "Enable chunk statistics tracking" -msgstr "" +msgstr "启用块统计跟踪" #: ../../source/production/observability/chunk_statistics.rst:202 msgid "``chunk_statistics_strategy``" -msgstr "" +msgstr "``chunk_statistics_strategy``" #: ../../source/production/observability/chunk_statistics.rst:203 msgid "``memory_bloom_filter``" -msgstr "" +msgstr "``memory_bloom_filter``" #: ../../source/production/observability/chunk_statistics.rst:204 msgid "Recording strategy: ``memory_bloom_filter`` or ``file_hash``" -msgstr "" +msgstr "记录策略：``memory_bloom_filter`` 或 ``file_hash``" #: ../../source/production/observability/chunk_statistics.rst:205 msgid "``chunk_statistics_auto_start_statistics``" -msgstr "" +msgstr "``chunk_statistics_auto_start_statistics``" #: ../../source/production/observability/chunk_statistics.rst:207 msgid "Automatically start statistics collection on initialization" -msgstr "" +msgstr "在初始化时自动开始统计收集" #: ../../source/production/observability/chunk_statistics.rst:208 msgid "``chunk_statistics_auto_exit_timeout_hours``" -msgstr "" +msgstr "``chunk_statistics_auto_exit_timeout_hours``" #: ../../source/production/observability/chunk_statistics.rst:209 msgid "``0.0``" -msgstr "" +msgstr "``0.0``" #: ../../source/production/observability/chunk_statistics.rst:210 msgid "Auto-stop after specified hours (0 = disabled)" -msgstr "" +msgstr "在指定小时后自动停止（0 = 禁用）" #: ../../source/production/observability/chunk_statistics.rst:211 msgid "``chunk_statistics_auto_exit_target_unique_chunks``" -msgstr "" +msgstr "``chunk_statistics_auto_exit_target_unique_chunks``" #: ../../source/production/observability/chunk_statistics.rst:212 msgid "``0``" -msgstr "" +msgstr "``0``" #: ../../source/production/observability/chunk_statistics.rst:213 msgid "Auto-stop after reaching target unique chunks (0 = disabled)" -msgstr "" +msgstr "达到目标唯一块后自动停止（0 = 禁用）" #: ../../source/production/observability/chunk_statistics.rst:216 msgid "Memory Bloom Filter Options" -msgstr "" +msgstr "内存布隆过滤器选项" #: ../../source/production/observability/chunk_statistics.rst:218 #: ../../source/production/observability/chunk_statistics.rst:237 msgid "Configure these options in the ``extra_config`` section:" -msgstr "" +msgstr "在 ``extra_config`` 部分配置这些选项：" #: ../../source/production/observability/chunk_statistics.rst:220 msgid "Bloom Filter Configuration" -msgstr "" +msgstr "布隆过滤器配置" #: ../../source/production/observability/chunk_statistics.rst:227 msgid "``chunk_statistics_mem_bf_expected_chunks``" -msgstr "" +msgstr "``chunk_statistics_mem_bf_expected_chunks``" #: ../../source/production/observability/chunk_statistics.rst:228 msgid "``20000000``" -msgstr "" +msgstr "``20000000``" #: ../../source/production/observability/chunk_statistics.rst:229 msgid "Expected number of chunks for capacity planning" -msgstr "" +msgstr "容量规划的预期块数" #: ../../source/production/observability/chunk_statistics.rst:230 msgid "``chunk_statistics_mem_bf_false_positive_rate``" -msgstr "" +msgstr "``chunk_statistics_mem_bf_false_positive_rate``" #: ../../source/production/observability/chunk_statistics.rst:231 msgid "``0.01``" -msgstr "" +msgstr "``0.01``" #: ../../source/production/observability/chunk_statistics.rst:232 msgid "Target false positive rate (1%)" -msgstr "" +msgstr "目标假阳性率 (1%)" #: ../../source/production/observability/chunk_statistics.rst:235 msgid "File Hash Options" -msgstr "" +msgstr "文件哈希选项" #: ../../source/production/observability/chunk_statistics.rst:239 msgid "File Hash Configuration" -msgstr "" +msgstr "文件哈希配置" #: ../../source/production/observability/chunk_statistics.rst:246 msgid "``chunk_statistics_file_output_dir``" -msgstr "" +msgstr "``chunk_statistics_file_output_dir``" #: ../../source/production/observability/chunk_statistics.rst:247 msgid "``/tmp/lmcache_chunk_statistics``" -msgstr "" +msgstr "``/tmp/lmcache_chunk_statistics``" #: ../../source/production/observability/chunk_statistics.rst:248 msgid "Directory for storing chunk hash files" -msgstr "" +msgstr "存储块哈希文件的目录" #: ../../source/production/observability/chunk_statistics.rst:249 msgid "``chunk_statistics_file_rotation_size``" -msgstr "" +msgstr "``chunk_statistics_file_rotation_size``" #: ../../source/production/observability/chunk_statistics.rst:250 msgid "``104857600``" -msgstr "" +msgstr "``104857600``" #: ../../source/production/observability/chunk_statistics.rst:251 msgid "File size threshold for rotation (bytes, default 100MB)" -msgstr "" +msgstr "文件大小阈值用于轮换（字节，默认 100MB）" #: ../../source/production/observability/chunk_statistics.rst:252 msgid "``chunk_statistics_file_max_count``" -msgstr "" +msgstr "``chunk_statistics_file_max_count``" #: ../../source/production/observability/chunk_statistics.rst:253 msgid "``100``" -msgstr "" +msgstr "``100``" #: ../../source/production/observability/chunk_statistics.rst:254 msgid "Maximum number of files to keep" -msgstr "" +msgstr "保留的最大文件数量" #: ../../source/production/observability/chunk_statistics.rst:257 msgid "Advanced Usage" -msgstr "" +msgstr "高级用法" #: ../../source/production/observability/chunk_statistics.rst:260 msgid "Programmatic Control" -msgstr "" +msgstr "程序化控制" #: ../../source/production/observability/chunk_statistics.rst:262 msgid "Control statistics collection programmatically through the internal API:" -msgstr "" +msgstr "通过内部 API 以编程方式控制统计信息收集：" #: ../../source/production/observability/chunk_statistics.rst:282 msgid "Auto-Stop Configuration" -msgstr "" +msgstr "自动停止配置" #: ../../source/production/observability/chunk_statistics.rst:284 msgid "Configure automatic stopping based on time or chunk count:" -msgstr "" +msgstr "根据时间或块数配置自动停止：" #: ../../source/production/observability/chunk_statistics.rst:291 msgid "Prometheus Metrics" -msgstr "" +msgstr "Prometheus 指标" #: ../../source/production/observability/chunk_statistics.rst:293 msgid "" "When using the internal API server, chunk statistics are exposed as " "Prometheus metrics:" -msgstr "" +msgstr "当使用内部 API 服务器时，块统计信息作为 Prometheus 指标公开：" #: ../../source/production/observability/chunk_statistics.rst:295 msgid "" "``lmcache_chunk_statistics_total_chunks``: Total number of chunks " "processed" -msgstr "" +msgstr "``lmcache_chunk_statistics_total_chunks``: 处理的块的总数" #: ../../source/production/observability/chunk_statistics.rst:296 msgid "``lmcache_chunk_statistics_unique_chunks``: Number of unique chunks" -msgstr "" +msgstr "``lmcache_chunk_statistics_unique_chunks``: 唯一块的数量" #: ../../source/production/observability/chunk_statistics.rst:297 msgid "``lmcache_chunk_statistics_reuse_rate``: Cache reuse rate (0.0 to 1.0)" -msgstr "" +msgstr "``lmcache_chunk_statistics_reuse_rate``: 缓存重用率 (0.0 到 1.0)" #: ../../source/production/observability/chunk_statistics.rst:298 msgid "" "``lmcache_chunk_statistics_bloom_filter_size_mb``: Bloom filter memory " "usage (MB)" -msgstr "" +msgstr "``lmcache_chunk_statistics_bloom_filter_size_mb``: 布隆过滤器内存使用量 (MB)" #: ../../source/production/observability/chunk_statistics.rst:299 msgid "" "``lmcache_chunk_statistics_bloom_filter_fill_rate``: Bloom filter fill " "rate (0.0 to 1.0)" -msgstr "" +msgstr "``lmcache_chunk_statistics_bloom_filter_fill_rate``: 布隆过滤器填充率 (0.0 到 1.0)" #: ../../source/production/observability/chunk_statistics.rst:300 msgid "``lmcache_chunk_statistics_file_count``: Number of hash files created" -msgstr "" +msgstr "``lmcache_chunk_statistics_file_count``: 创建的哈希文件数量" #: ../../source/production/observability/chunk_statistics.rst:301 msgid "``lmcache_chunk_statistics_current_file_size``: Current file size (bytes)" -msgstr "" +msgstr "``lmcache_chunk_statistics_current_file_size``: 当前文件大小（字节）" #: ../../source/production/observability/chunk_statistics.rst:304 msgid "Offline Analysis" -msgstr "" +msgstr "离线分析" #: ../../source/production/observability/chunk_statistics.rst:306 msgid "" "For the file hash strategy, you can perform detailed offline analysis of " "collected chunk hash data." -msgstr "" +msgstr "对于文件哈希策略，您可以对收集到的块哈希数据进行详细的离线分析。" #: ../../source/production/observability/chunk_statistics.rst:309 msgid "Using the Analysis Script" -msgstr "" +msgstr "使用分析脚本" #: ../../source/production/observability/chunk_statistics.rst:311 msgid "" "LMCache provides a comprehensive analysis script at " "``examples/chunk_statistics/analyze_chunk_hashes.py`` that supports " "multiple analysis modes." -msgstr "" +msgstr "LMCache 提供了一个全面的分析脚本，位于 ``examples/chunk_statistics/analyze_chunk_hashes.py``，支持多种分析模式。" #: ../../source/production/observability/chunk_statistics.rst:314 msgid "Best Practices" -msgstr "" +msgstr "最佳实践" #: ../../source/production/observability/chunk_statistics.rst:316 msgid "**Choose the Right Strategy:**" -msgstr "" +msgstr "**选择正确的策略：**" #: ../../source/production/observability/chunk_statistics.rst:318 msgid "Use **memory_bloom_filter** for real-time monitoring with minimal overhead" -msgstr "" +msgstr "使用 **memory_bloom_filter** 进行实时监控，且开销最小" #: ../../source/production/observability/chunk_statistics.rst:319 msgid "Use **file_hash** when exact tracking is required or for offline analysis" -msgstr "" +msgstr "使用 **file_hash** 当需要精确跟踪或进行离线分析时。" #: ../../source/production/observability/chunk_statistics.rst:321 msgid "**Tune Bloom Filter Parameters:**" -msgstr "" +msgstr "**调整布隆过滤器参数:**" #: ../../source/production/observability/chunk_statistics.rst:323 msgid "Set ``expected_chunks`` based on your workload size" -msgstr "" +msgstr "根据您的工作负载大小设置 ``expected_chunks``" #: ../../source/production/observability/chunk_statistics.rst:324 msgid "Lower ``false_positive_rate`` increases memory usage but improves accuracy" -msgstr "" +msgstr "降低 ``false_positive_rate`` 会增加内存使用，但提高准确性。" #: ../../source/production/observability/chunk_statistics.rst:326 msgid "**Monitor Memory Usage:**" -msgstr "" +msgstr "**监控内存使用情况：**" #: ../../source/production/observability/chunk_statistics.rst:328 msgid "" "Track ``bloom_filter_size_mb`` metric to ensure it fits in available " "memory" -msgstr "" +msgstr "跟踪 ``bloom_filter_size_mb`` 指标以确保其适合可用内存" #: ../../source/production/observability/chunk_statistics.rst:329 msgid "Adjust ``expected_chunks`` if memory usage is too high" -msgstr "" +msgstr "如果内存使用过高，请调整 ``expected_chunks``。" #: ../../source/production/observability/chunk_statistics.rst:331 msgid "**File Rotation:**" -msgstr "" +msgstr "**文件轮换：**" #: ../../source/production/observability/chunk_statistics.rst:333 msgid "" "Configure appropriate ``file_rotation_size`` to balance file size and " "count" -msgstr "" +msgstr "配置适当的 ``file_rotation_size`` 以平衡文件大小和数量" #: ../../source/production/observability/chunk_statistics.rst:334 msgid "Set ``file_max_count`` to prevent unlimited disk usage" -msgstr "" +msgstr "设置 ``file_max_count`` 以防止无限制的磁盘使用" #: ../../source/production/observability/chunk_statistics.rst:336 msgid "**Production Deployment:**" -msgstr "" +msgstr "**生产部署：**" #: ../../source/production/observability/chunk_statistics.rst:338 msgid "Enable auto-stop to prevent indefinite data collection" -msgstr "" +msgstr "启用自动停止以防止无限期的数据收集" #: ../../source/production/observability/chunk_statistics.rst:339 msgid "Use internal API server for centralized metrics collection" -msgstr "" +msgstr "使用内部 API 服务器进行集中式指标收集" #: ../../source/production/observability/chunk_statistics.rst:340 msgid "Integrate with your monitoring stack (Prometheus, Grafana, etc.)" -msgstr "" +msgstr "与您的监控栈集成（Prometheus、Grafana 等）" #: ../../source/production/observability/chunk_statistics.rst:343 msgid "Troubleshooting" -msgstr "" +msgstr "故障排除" #: ../../source/production/observability/chunk_statistics.rst:346 msgid "Statistics Not Updating" -msgstr "" +msgstr "统计未更新" #: ../../source/production/observability/chunk_statistics.rst:348 msgid "**Issue:** Statistics remain at zero or don't update." -msgstr "" +msgstr "**问题：** 统计数据保持为零或不更新。" #: ../../source/production/observability/chunk_statistics.rst:350 #: ../../source/production/observability/chunk_statistics.rst:361 #: ../../source/production/observability/chunk_statistics.rst:372 msgid "**Solutions:**" -msgstr "" +msgstr "**解决方案：**" #: ../../source/production/observability/chunk_statistics.rst:352 msgid "Verify ``enable_chunk_statistics`` is set to ``true``" -msgstr "" +msgstr "验证 ``enable_chunk_statistics`` 是否设置为 ``true``" #: ../../source/production/observability/chunk_statistics.rst:353 msgid "Check that statistics collection is started (auto-start or manual start)" -msgstr "" +msgstr "检查统计收集是否已启动（自动启动或手动启动）" #: ../../source/production/observability/chunk_statistics.rst:354 msgid "Ensure requests are being processed by the LMCache instance" -msgstr "" +msgstr "确保请求正在被 LMCache 实例处理" #: ../../source/production/observability/chunk_statistics.rst:357 msgid "High Memory Usage" -msgstr "" +msgstr "高内存使用率" #: ../../source/production/observability/chunk_statistics.rst:359 msgid "**Issue:** Bloom filter consuming too much memory." -msgstr "" +msgstr "**问题：** Bloom 过滤器消耗了过多的内存。" #: ../../source/production/observability/chunk_statistics.rst:363 msgid "Reduce ``chunk_statistics_mem_bf_expected_chunks``" -msgstr "" +msgstr "减少 ``chunk_statistics_mem_bf_expected_chunks``" #: ../../source/production/observability/chunk_statistics.rst:364 msgid "" "Increase ``chunk_statistics_mem_bf_false_positive_rate`` (trade accuracy " "for memory)" -msgstr "" +msgstr "增加 ``chunk_statistics_mem_bf_false_positive_rate`` （以牺牲准确性换取内存）" #: ../../source/production/observability/chunk_statistics.rst:365 msgid "Consider switching to ``file_hash`` strategy" -msgstr "" +msgstr "考虑切换到 ``file_hash`` 策略" #: ../../source/production/observability/chunk_statistics.rst:368 msgid "File System Full" -msgstr "" +msgstr "文件系统已满" #: ../../source/production/observability/chunk_statistics.rst:370 msgid "**Issue:** Disk space exhausted with file hash strategy." -msgstr "" +msgstr "**问题：** 磁盘空间耗尽，使用文件哈希策略。" #: ../../source/production/observability/chunk_statistics.rst:374 msgid "Reduce ``chunk_statistics_file_max_count``" -msgstr "" +msgstr "减少 ``chunk_statistics_file_max_count``" #: ../../source/production/observability/chunk_statistics.rst:375 msgid "Decrease ``chunk_statistics_file_rotation_size``" -msgstr "" +msgstr "减少 ``chunk_statistics_file_rotation_size``" #: ../../source/production/observability/chunk_statistics.rst:376 msgid "Implement external log rotation or archival" -msgstr "" +msgstr "实现外部日志轮换或归档" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/frontend.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/frontend.po index 2b12b6590f6..2622452f5f6 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/frontend.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/frontend.po @@ -21,156 +21,156 @@ msgstr "" #: ../../source/production/observability/frontend.rst:4 msgid "LMCache Frontend" -msgstr "" +msgstr "LMCache 前端" #: ../../source/production/observability/frontend.rst:6 msgid "" "LMCache Frontend is a monitoring and proxy service for LMCache clusters, " "providing a web interface for cluster management and HTTP request " "proxying to cluster nodes." -msgstr "" +msgstr "LMCache Frontend 是一个用于 LMCache 集群的监控和代理服务，提供集群管理的 Web 界面以及对集群节点的 HTTP 请求代理。" #: ../../source/production/observability/frontend.rst:10 msgid "LMCache Frontend Dashboard" -msgstr "" +msgstr "LMCache 前端仪表板" #: ../../source/production/observability/frontend.rst:16 msgid "Features" -msgstr "" +msgstr "功能" #: ../../source/production/observability/frontend.rst:18 msgid "**Cluster Monitoring**: Web-based dashboard for visualizing cluster status" -msgstr "" +msgstr "**集群监控**：基于网页的仪表板，用于可视化集群状态" #: ../../source/production/observability/frontend.rst:19 msgid "" "**Request Proxying**: HTTP proxy service to forward requests to any " "cluster node" -msgstr "" +msgstr "**请求代理**：HTTP 代理服务，用于将请求转发到任何集群节点" #: ../../source/production/observability/frontend.rst:20 msgid "" "**Flexible Configuration**: Support for both IP:port and Unix domain " "sockets" -msgstr "" +msgstr "**灵活配置**：支持 IP:端口 和 Unix 域套接字" #: ../../source/production/observability/frontend.rst:21 msgid "**Plugin System**: Integration with LMCache plugin framework" -msgstr "" +msgstr "**插件系统**：与 LMCache 插件框架的集成" #: ../../source/production/observability/frontend.rst:25 msgid "Installation" -msgstr "" +msgstr "安装" #: ../../source/production/observability/frontend.rst:27 msgid "Install from PyPI:" -msgstr "" +msgstr "从 PyPI 安装：" #: ../../source/production/observability/frontend.rst:33 msgid "Or install from source:" -msgstr "" +msgstr "或从源代码安装：" #: ../../source/production/observability/frontend.rst:43 msgid "Quick Start" -msgstr "" +msgstr "快速开始" #: ../../source/production/observability/frontend.rst:46 msgid "Starting the Service" -msgstr "" +msgstr "启动服务" #: ../../source/production/observability/frontend.rst:54 msgid "Command Line Options" -msgstr "" +msgstr "命令行选项" #: ../../source/production/observability/frontend.rst:60 msgid "Option" -msgstr "" +msgstr "选项" #: ../../source/production/observability/frontend.rst:61 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/production/observability/frontend.rst:62 msgid "``--port``" -msgstr "" +msgstr "``--port``" #: ../../source/production/observability/frontend.rst:63 msgid "Service port (default: 8000)" -msgstr "" +msgstr "服务端口（默认：8000）" #: ../../source/production/observability/frontend.rst:64 msgid "``--host``" -msgstr "" +msgstr "``--host``" #: ../../source/production/observability/frontend.rst:65 msgid "Bind host address (default: 0.0.0.0)" -msgstr "" +msgstr "绑定主机地址（默认：0.0.0.0）" #: ../../source/production/observability/frontend.rst:66 msgid "``--config``" -msgstr "" +msgstr "``--config``" #: ../../source/production/observability/frontend.rst:67 msgid "Path to configuration file" -msgstr "" +msgstr "配置文件路径" #: ../../source/production/observability/frontend.rst:68 msgid "``--nodes``" -msgstr "" +msgstr "``--nodes``" #: ../../source/production/observability/frontend.rst:69 msgid "Direct node configuration (JSON string)" -msgstr "" +msgstr "直接节点配置（JSON 字符串）" #: ../../source/production/observability/frontend.rst:72 msgid "" "After starting the service, access the dashboard at " "``http://localhost:8080/``." -msgstr "" +msgstr "启动服务后，可以通过访问 ``http://localhost:8080/`` 来查看仪表板。" #: ../../source/production/observability/frontend.rst:76 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/production/observability/frontend.rst:79 msgid "Node Configuration" -msgstr "" +msgstr "节点配置" #: ../../source/production/observability/frontend.rst:81 msgid "Create a ``config.json`` file with node definitions:" -msgstr "" +msgstr "创建一个 ``config.json`` 文件，包含节点定义：" #: ../../source/production/observability/frontend.rst:98 msgid "" "The ``port`` field can be configured as either an integer port number or " "a string path for Unix domain sockets." -msgstr "" +msgstr "``port`` 字段可以配置为整数端口号或 Unix 域套接字的字符串路径。" #: ../../source/production/observability/frontend.rst:103 msgid "LMCache Plugin Integration" -msgstr "" +msgstr "LMCache 插件集成" #: ../../source/production/observability/frontend.rst:105 msgid "" "To start the frontend via the LMCache plugin framework, add the following" " to your ``lmcache.yaml``:" -msgstr "" +msgstr "要通过 LMCache 插件框架启动前端，请将以下内容添加到您的 ``lmcache.yaml`` 中：" #: ../../source/production/observability/frontend.rst:119 msgid "Proxying Requests" -msgstr "" +msgstr "代理请求" #: ../../source/production/observability/frontend.rst:121 msgid "Proxy requests using the format:" -msgstr "" +msgstr "使用以下格式代理请求：" #: ../../source/production/observability/frontend.rst:127 msgid "Examples:" -msgstr "" +msgstr "示例：" #: ../../source/production/observability/frontend.rst:140 msgid "Contributing" -msgstr "" +msgstr "贡献" #: ../../source/production/observability/frontend.rst:142 msgid "" @@ -178,45 +178,45 @@ msgid "" "from the community! Whether you want to fix bugs, add new features, " "improve documentation, or share ideas, your contributions are " "appreciated." -msgstr "" +msgstr "LMCache 前端是一个开源项目，我们欢迎社区的贡献！无论您想修复错误、添加新功能、改善文档还是分享想法，我们都非常感谢您的贡献。" #: ../../source/production/observability/frontend.rst:146 msgid "Ways to contribute:" -msgstr "" +msgstr "贡献方式：" #: ../../source/production/observability/frontend.rst:148 msgid "" "**Report Issues**: Found a bug or have a feature request? Open an issue " "on GitHub." -msgstr "" +msgstr "**报告问题**：发现了一个错误或有功能请求？在 GitHub 上提交一个问题。" #: ../../source/production/observability/frontend.rst:150 msgid "" "**Submit Pull Requests**: Fork the repository, make your changes, and " "submit a PR." -msgstr "" +msgstr "**提交拉取请求**：分叉仓库，进行更改，并提交 PR。" #: ../../source/production/observability/frontend.rst:152 msgid "**Improve Documentation**: Help us make the docs clearer and more helpful." -msgstr "" +msgstr "**改善文档**：帮助我们使文档更清晰、更有帮助。" #: ../../source/production/observability/frontend.rst:153 msgid "" "**Share Feedback**: Let us know how you're using LMCache Frontend and " "what could be improved." -msgstr "" +msgstr "**分享反馈**：让我们知道您如何使用 LMCache 前端以及可以改进的地方。" #: ../../source/production/observability/frontend.rst:156 msgid "Join us in making LMCache Frontend better for everyone!" -msgstr "" +msgstr "加入我们，让 LMCache Frontend 变得更好！" #: ../../source/production/observability/frontend.rst:160 msgid "More Information" -msgstr "" +msgstr "更多信息" #: ../../source/production/observability/frontend.rst:162 msgid "" "For more details, visit the `LMCache Frontend GitHub repository " "`_." -msgstr "" +msgstr "有关更多详细信息，请访问 `LMCache Frontend GitHub 仓库 `_。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/health_monitor.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/health_monitor.po index 500d4a3ed7c..7fa1c01a018 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/health_monitor.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/health_monitor.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/production/observability/health_monitor.rst:4 msgid "Health Monitor" -msgstr "" +msgstr "健康监控器" #: ../../source/production/observability/health_monitor.rst:6 msgid "" @@ -29,467 +29,467 @@ msgid "" "continuously monitors the health of the cache engine and its components. " "This feature is essential for production deployments to detect and " "respond to failures in remote storage backends." -msgstr "" +msgstr "LMCache 包含一个全面的健康监测框架，持续监测缓存引擎及其组件的健康状况。此功能对于生产环境的部署至关重要，可以检测并响应远程存储后端的故障。" #: ../../source/production/observability/health_monitor.rst:9 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/production/observability/health_monitor.rst:11 msgid "The Health Monitor provides:" -msgstr "" +msgstr "健康监控器提供：" #: ../../source/production/observability/health_monitor.rst:13 msgid "" "**Automatic health checks**: Periodically monitors the health of all " "registered components" -msgstr "" +msgstr "**自动健康检查**：定期监控所有注册组件的健康状况" #: ../../source/production/observability/health_monitor.rst:14 msgid "" "**Extensible framework**: Easily add custom health checks for new " "components" -msgstr "" +msgstr "**可扩展框架**：轻松为新组件添加自定义健康检查" #: ../../source/production/observability/health_monitor.rst:15 msgid "" "**Remote backend monitoring**: Built-in support for monitoring remote " "storage backends via ping" -msgstr "" +msgstr "**远程后端监控**：内置支持通过 ping 监控远程存储后端" #: ../../source/production/observability/health_monitor.rst:16 msgid "" "**Degraded mode support**: Automatically blocks operations when the " "system is unhealthy" -msgstr "" +msgstr "**降级模式支持**：当系统不健康时自动阻止操作" #: ../../source/production/observability/health_monitor.rst:17 msgid "" "**Prometheus metrics integration**: Health status exposed via metrics " "endpoint" -msgstr "" +msgstr "**Prometheus 指标集成**：通过指标端点暴露健康状态" #: ../../source/production/observability/health_monitor.rst:20 msgid "Architecture" -msgstr "" +msgstr "架构" #: ../../source/production/observability/health_monitor.rst:22 msgid "The health monitoring system consists of three main components:" -msgstr "" +msgstr "健康监测系统由三个主要组件组成：" #: ../../source/production/observability/health_monitor.rst:24 msgid "**HealthCheck (Abstract Base Class)**" -msgstr "" +msgstr "**健康检查（抽象基类）**" #: ../../source/production/observability/health_monitor.rst:26 msgid "" "Base class for individual health checks. Each health check represents one" " aspect of system health." -msgstr "" +msgstr "单个健康检查的基类。每个健康检查代表系统健康的一个方面。" #: ../../source/production/observability/health_monitor.rst:28 msgid "**HealthMonitor**" -msgstr "" +msgstr "**健康监控器**" #: ../../source/production/observability/health_monitor.rst:30 msgid "" "The central monitor that orchestrates all health checks. It runs in a " "background thread and periodically executes all registered health checks." -msgstr "" +msgstr "中央监控器，负责协调所有健康检查。它在后台线程中运行，并定期执行所有注册的健康检查。" #: ../../source/production/observability/health_monitor.rst:32 msgid "**RemoteBackendHealthCheck**" -msgstr "" +msgstr "**RemoteBackendHealthCheck**" #: ../../source/production/observability/health_monitor.rst:34 msgid "" "Built-in health check for remote storage backends. It pings the remote " "connector to verify connectivity." -msgstr "" +msgstr "内置的远程存储后端健康检查。它会向远程连接器发送请求以验证连接性。" #: ../../source/production/observability/health_monitor.rst:37 msgid "Auto-Discovery" -msgstr "" +msgstr "自动发现" #: ../../source/production/observability/health_monitor.rst:39 msgid "" "The Health Monitor uses an auto-discovery mechanism to find and " "instantiate health checks:" -msgstr "" +msgstr "健康监控器使用自动发现机制来查找和实例化健康检查：" #: ../../source/production/observability/health_monitor.rst:41 msgid "" "At startup, the monitor scans the ``lmcache.v1.health_monitor.checks`` " "package" -msgstr "" +msgstr "在启动时，监视器扫描 ``lmcache.v1.health_monitor.checks`` 包" #: ../../source/production/observability/health_monitor.rst:42 msgid "All classes that inherit from ``HealthCheck`` are discovered" -msgstr "" +msgstr "所有继承自 ``HealthCheck`` 的类都会被发现" #: ../../source/production/observability/health_monitor.rst:43 msgid "Each check's ``create_from_engine()`` method is called to create instances" -msgstr "" +msgstr "每个检查的 ``create_from_engine()`` 方法被调用以创建实例" #: ../../source/production/observability/health_monitor.rst:44 msgid "The instances are registered with the monitor" -msgstr "" +msgstr "这些实例会被注册到监视器中。" #: ../../source/production/observability/health_monitor.rst:46 msgid "" "This design allows you to add new health checks by simply creating a new " "module in the checks package." -msgstr "" +msgstr "此设计允许您通过在检查包中简单地创建一个新模块来添加新的健康检查。" #: ../../source/production/observability/health_monitor.rst:49 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/production/observability/health_monitor.rst:51 msgid "" "Health monitor configuration is done through the ``extra_config`` section" " of your LMCache configuration:" -msgstr "" +msgstr "健康监控配置通过您的 LMCache 配置的 ``extra_config`` 部分完成：" #: ../../source/production/observability/health_monitor.rst:53 msgid "Health Monitor Configuration Options" -msgstr "" +msgstr "健康监控配置选项" #: ../../source/production/observability/health_monitor.rst:57 msgid "Configuration Key" -msgstr "" +msgstr "配置键" #: ../../source/production/observability/health_monitor.rst:58 msgid "Default Value" -msgstr "" +msgstr "默认值" #: ../../source/production/observability/health_monitor.rst:59 #: ../../source/production/observability/health_monitor.rst:148 #: ../../source/production/observability/health_monitor.rst:175 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/production/observability/health_monitor.rst:60 msgid "``ping_interval``" -msgstr "" +msgstr "``ping_interval``" #: ../../source/production/observability/health_monitor.rst:61 msgid "``30.0``" -msgstr "" +msgstr "``30.0``" #: ../../source/production/observability/health_monitor.rst:62 msgid "Interval (in seconds) between health check cycles" -msgstr "" +msgstr "健康检查周期之间的间隔（以秒为单位）" #: ../../source/production/observability/health_monitor.rst:63 msgid "``ping_timeout``" -msgstr "" +msgstr "``ping_timeout``" #: ../../source/production/observability/health_monitor.rst:64 msgid "``5.0``" -msgstr "" +msgstr "``5.0``" #: ../../source/production/observability/health_monitor.rst:65 msgid "Timeout (in seconds) for each ping operation" -msgstr "" +msgstr "每次 ping 操作的超时时间（以秒为单位）" #: ../../source/production/observability/health_monitor.rst:66 msgid "``get_blocking_failed_threshold``" -msgstr "" +msgstr "``get_blocking_failed_threshold``" #: ../../source/production/observability/health_monitor.rst:67 msgid "``10``" -msgstr "" +msgstr "``10``" #: ../../source/production/observability/health_monitor.rst:68 msgid "Max number of get_blocking failed count in check interval" -msgstr "" +msgstr "检查时间间隔内获取阻塞失败的最大次数" #: ../../source/production/observability/health_monitor.rst:69 msgid "``waiting_time_for_recovery``" -msgstr "" +msgstr "``waiting_time_for_recovery``" #: ../../source/production/observability/health_monitor.rst:70 msgid "``300.0``" -msgstr "" +msgstr "``300.0``" #: ../../source/production/observability/health_monitor.rst:71 msgid "Waiting time (in seconds) for recovery if get_blocking failed" -msgstr "" +msgstr "如果 get_blocking 失败，恢复的等待时间（以秒为单位）" #: ../../source/production/observability/health_monitor.rst:75 msgid "How It Works" -msgstr "" +msgstr "它是如何工作的" #: ../../source/production/observability/health_monitor.rst:78 msgid "Runtime Behavior" -msgstr "" +msgstr "运行时行为" #: ../../source/production/observability/health_monitor.rst:80 msgid "The health monitor runs in a background thread:" -msgstr "" +msgstr "健康监控在后台线程中运行：" #: ../../source/production/observability/health_monitor.rst:82 msgid "Every ``ping_interval`` seconds, all health checks are executed" -msgstr "" +msgstr "每 ``ping_interval`` 秒，所有健康检查都会执行" #: ../../source/production/observability/health_monitor.rst:83 msgid "If any check fails, the system is marked as unhealthy" -msgstr "" +msgstr "如果任何检查失败，系统将被标记为不健康。" #: ../../source/production/observability/health_monitor.rst:84 msgid "When unhealthy, store/retrieve operations are blocked with a warning log" -msgstr "" +msgstr "当系统不健康时，存储/检索操作会被阻止，并记录警告日志。" #: ../../source/production/observability/health_monitor.rst:85 msgid "" "Once all checks pass again, the system is marked as healthy and " "operations resume" -msgstr "" +msgstr "一旦所有检查再次通过，系统将被标记为健康，操作将恢复。" #: ../../source/production/observability/health_monitor.rst:88 msgid "Initialization Failure Handling" -msgstr "" +msgstr "初始化失败处理" #: ../../source/production/observability/health_monitor.rst:90 msgid "When initialization or post-initialization fails irrecoverably:" -msgstr "" +msgstr "当初始化或后初始化不可恢复地失败时：" #: ../../source/production/observability/health_monitor.rst:92 msgid "The system is marked with ``_init_failed = True``" -msgstr "" +msgstr "系统被标记为 ``_init_failed = True``" #: ../../source/production/observability/health_monitor.rst:93 msgid "``is_healthy()`` method returns ``False`` permanently" -msgstr "" +msgstr "``is_healthy()`` 方法永久返回 ``False``" #: ../../source/production/observability/health_monitor.rst:94 msgid "" "Health monitoring thread will not start (if initialization fails before " "it starts)" -msgstr "" +msgstr "健康监控线程将不会启动（如果初始化在启动之前失败）" #: ../../source/production/observability/health_monitor.rst:95 msgid "The system operates in degraded mode (recompute-only)" -msgstr "" +msgstr "系统以降级模式运行（仅重计算）" #: ../../source/production/observability/health_monitor.rst:97 msgid "" "This ensures that irrecoverable initialization errors don't cause " "cascading failures and the system can gracefully fall back to " "recomputation." -msgstr "" +msgstr "这确保不可恢复的初始化错误不会导致级联故障，并且系统可以优雅地回退到重计算。" #: ../../source/production/observability/health_monitor.rst:101 msgid "Graceful Degradation" -msgstr "" +msgstr "优雅降级" #: ../../source/production/observability/health_monitor.rst:103 msgid "When the health monitor detects an unhealthy state:" -msgstr "" +msgstr "当健康监测器检测到不健康状态时：" #: ../../source/production/observability/health_monitor.rst:105 msgid "**Store operations**: Skipped with a warning message" -msgstr "" +msgstr "**存储操作**：跳过并显示警告信息" #: ../../source/production/observability/health_monitor.rst:106 msgid "**Retrieve operations**: Return empty results with a warning message" -msgstr "" +msgstr "**检索操作**：返回空结果并带有警告信息" #: ../../source/production/observability/health_monitor.rst:107 msgid "**Lookup operations**: Return 0 (no cache hits) with a warning message" -msgstr "" +msgstr "**查找操作**：返回 0（没有缓存命中）并附带警告信息" #: ../../source/production/observability/health_monitor.rst:109 msgid "This prevents cascading failures when remote backends are unavailable." -msgstr "" +msgstr "这可以防止在远程后端不可用时发生级联故障。" #: ../../source/production/observability/health_monitor.rst:112 msgid "Built-in Health Checks" -msgstr "" +msgstr "内置健康检查" #: ../../source/production/observability/health_monitor.rst:115 msgid "RemoteBackendHealthCheck" -msgstr "" +msgstr "RemoteBackendHealthCheck" #: ../../source/production/observability/health_monitor.rst:117 msgid "" "This check monitors the connectivity to remote storage backends (e.g., " "Redis, Valkey)." -msgstr "" +msgstr "此检查监控与远程存储后端（例如，Redis、Valkey）的连接性。" #: ../../source/production/observability/health_monitor.rst:119 msgid "**What it checks:**" -msgstr "" +msgstr "**检查内容：**" #: ../../source/production/observability/health_monitor.rst:121 msgid "Pings the remote connector to verify it's reachable" -msgstr "" +msgstr "对远程连接器进行 ping 测试以验证其可达性" #: ../../source/production/observability/health_monitor.rst:122 msgid "Measures ping latency" -msgstr "" +msgstr "测量 ping 延迟" #: ../../source/production/observability/health_monitor.rst:123 msgid "Reports error codes for failures" -msgstr "" +msgstr "报告失败的错误代码" #: ../../source/production/observability/health_monitor.rst:125 msgid "**When it's active:**" -msgstr "" +msgstr "**当它处于活动状态时：**" #: ../../source/production/observability/health_monitor.rst:127 msgid "Only when a remote backend is configured (``remote_url`` is set)" -msgstr "" +msgstr "仅在配置了远程后端时（``remote_url`` 已设置）" #: ../../source/production/observability/health_monitor.rst:128 msgid "Only if the connector supports the ``ping()`` operation" -msgstr "" +msgstr "仅当连接器支持 ``ping()`` 操作时" #: ../../source/production/observability/health_monitor.rst:130 msgid "**Metrics reported:**" -msgstr "" +msgstr "**报告的指标：**" #: ../../source/production/observability/health_monitor.rst:132 msgid "``lmcache:remote_ping_latency``: Latest ping latency (milliseconds)" -msgstr "" +msgstr "``lmcache:remote_ping_latency``: 最新的 ping 延迟（毫秒）" #: ../../source/production/observability/health_monitor.rst:133 msgid "``lmcache:remote_ping_error_code``: Latest error code (0 = success)" -msgstr "" +msgstr "``lmcache:remote_ping_error_code``: 最新错误代码 (0 = 成功)" #: ../../source/production/observability/health_monitor.rst:134 msgid "``lmcache:remote_ping_errors``: Total number of ping errors" -msgstr "" +msgstr "``lmcache:remote_ping_errors``: 总的 ping 错误数量" #: ../../source/production/observability/health_monitor.rst:135 msgid "``lmcache:remote_ping_successes``: Total number of successful pings" -msgstr "" +msgstr "``lmcache:remote_ping_successes``: 成功 ping 的总次数" #: ../../source/production/observability/health_monitor.rst:138 msgid "Prometheus Metrics" -msgstr "" +msgstr "Prometheus 指标" #: ../../source/production/observability/health_monitor.rst:140 msgid "The health monitor exposes metrics through the Prometheus endpoint:" -msgstr "" +msgstr "健康监控器通过 Prometheus 端点公开指标：" #: ../../source/production/observability/health_monitor.rst:142 msgid "Health Monitor Metrics" -msgstr "" +msgstr "健康监控指标" #: ../../source/production/observability/health_monitor.rst:146 msgid "Metric Name" -msgstr "" +msgstr "指标名称" #: ../../source/production/observability/health_monitor.rst:147 msgid "Type" -msgstr "" +msgstr "类型" #: ../../source/production/observability/health_monitor.rst:149 msgid "``lmcache:is_healthy``" -msgstr "" +msgstr "``lmcache:is_healthy``" #: ../../source/production/observability/health_monitor.rst:150 #: ../../source/production/observability/health_monitor.rst:153 #: ../../source/production/observability/health_monitor.rst:156 msgid "Gauge" -msgstr "" +msgstr "仪表盘" #: ../../source/production/observability/health_monitor.rst:151 msgid "Overall system health status (1 = healthy, 0 = unhealthy)" -msgstr "" +msgstr "整体系统健康状态（1 = 健康，0 = 不健康）" #: ../../source/production/observability/health_monitor.rst:152 msgid "``lmcache:remote_ping_latency``" -msgstr "" +msgstr "``lmcache:remote_ping_latency``" #: ../../source/production/observability/health_monitor.rst:154 msgid "Latest ping latency to remote backends (milliseconds)" -msgstr "" +msgstr "最新的远程后端 ping 延迟（毫秒）" #: ../../source/production/observability/health_monitor.rst:155 msgid "``lmcache:remote_ping_error_code``" -msgstr "" +msgstr "``lmcache:remote_ping_error_code``" #: ../../source/production/observability/health_monitor.rst:157 msgid "Latest ping error code (0 = success, -1 = timeout, -2 = generic error)" -msgstr "" +msgstr "最新的 ping 错误代码（0 = 成功，-1 = 超时，-2 = 通用错误）" #: ../../source/production/observability/health_monitor.rst:158 msgid "``lmcache:remote_ping_errors``" -msgstr "" +msgstr "``lmcache:remote_ping_errors``" #: ../../source/production/observability/health_monitor.rst:159 #: ../../source/production/observability/health_monitor.rst:162 msgid "Counter" -msgstr "" +msgstr "计数器" #: ../../source/production/observability/health_monitor.rst:160 msgid "Total number of ping errors to remote backends" -msgstr "" +msgstr "远程后端的总 ping 错误数量" #: ../../source/production/observability/health_monitor.rst:161 msgid "``lmcache:remote_ping_successes``" -msgstr "" +msgstr "``lmcache:remote_ping_successes``" #: ../../source/production/observability/health_monitor.rst:163 msgid "Total number of successful pings to remote backends" -msgstr "" +msgstr "成功 ping 远程后端的总次数" #: ../../source/production/observability/health_monitor.rst:166 msgid "Error Codes" -msgstr "" +msgstr "错误代码" #: ../../source/production/observability/health_monitor.rst:168 msgid "The health check system uses the following error codes:" -msgstr "" +msgstr "健康检查系统使用以下错误代码：" #: ../../source/production/observability/health_monitor.rst:170 msgid "Health Check Error Codes" -msgstr "" +msgstr "健康检查错误代码" #: ../../source/production/observability/health_monitor.rst:174 msgid "Code" -msgstr "" +msgstr "代码" #: ../../source/production/observability/health_monitor.rst:176 msgid "``0``" -msgstr "" +msgstr "``0``" #: ../../source/production/observability/health_monitor.rst:177 msgid "Success - the health check passed" -msgstr "" +msgstr "成功 - 健康检查通过" #: ../../source/production/observability/health_monitor.rst:178 msgid "``-1``" -msgstr "" +msgstr "``-1``" #: ../../source/production/observability/health_monitor.rst:179 msgid "Timeout - the ping operation exceeded the configured timeout" -msgstr "" +msgstr "超时 - ping 操作超过了配置的超时时间" #: ../../source/production/observability/health_monitor.rst:180 msgid "``-2``" -msgstr "" +msgstr "``-2``" #: ../../source/production/observability/health_monitor.rst:181 msgid "Generic error - an unexpected error occurred during the health check" -msgstr "" +msgstr "通用错误 - 在健康检查期间发生了意外错误" #: ../../source/production/observability/health_monitor.rst:184 msgid "Extending the Health Monitor" -msgstr "" +msgstr "扩展健康监控器" #: ../../source/production/observability/health_monitor.rst:186 msgid "" "You can add custom health checks by creating a new module in the " "``lmcache/v1/health_monitor/checks/`` directory." -msgstr "" +msgstr "您可以通过在 ``lmcache/v1/health_monitor/checks/`` 目录中创建新模块来添加自定义健康检查。" #: ../../source/production/observability/health_monitor.rst:188 msgid "" "The custom check will be automatically discovered and registered when " "LMCache starts." -msgstr "" +msgstr "当 LMCache 启动时，自定义检查将被自动发现并注册。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/index.po index 118b87314fa..1e584c25851 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/index.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/index.po @@ -21,5 +21,5 @@ msgstr "" #: ../../source/production/observability/index.rst:4 msgid "Observability" -msgstr "" +msgstr "可观察性" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/internal_api_server.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/internal_api_server.po index ed984b28a8d..b52b4ec1dd8 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/internal_api_server.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/internal_api_server.po @@ -21,118 +21,118 @@ msgstr "" #: ../../source/production/observability/internal_api_server.rst:4 msgid "Internal API Server Metrics" -msgstr "" +msgstr "内部 API 服务器指标" #: ../../source/production/observability/internal_api_server.rst:6 msgid "" "Another approach to retrieve LMCache metrics is to use the internal API " "server." -msgstr "" +msgstr "另一种获取 LMCache 指标的方法是使用内部 API 服务器。" #: ../../source/production/observability/internal_api_server.rst:9 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/production/observability/internal_api_server.rst:11 msgid "" "The internal API server exposes Prometheus-compatible metrics endpoints " "in your LMCache deployment." -msgstr "" +msgstr "内部 API 服务器在您的 LMCache 部署中公开与 Prometheus 兼容的指标端点。" #: ../../source/production/observability/internal_api_server.rst:14 msgid "Quick Start Guide" -msgstr "" +msgstr "快速入门指南" #: ../../source/production/observability/internal_api_server.rst:17 msgid "Step 1: Enable Internal API Server" -msgstr "" +msgstr "步骤 1：启用内部 API 服务器" #: ../../source/production/observability/internal_api_server.rst:19 msgid "Configure your vLLM instance to enable the internal API server:" -msgstr "" +msgstr "配置您的 vLLM 实例以启用内部 API 服务器：" #: ../../source/production/observability/internal_api_server.rst:28 msgid "Step 2: Access Metrics Endpoint" -msgstr "" +msgstr "步骤 2：访问指标端点" #: ../../source/production/observability/internal_api_server.rst:30 msgid "Retrieve metrics from the worker's endpoint:" -msgstr "" +msgstr "从工作节点的端点检索指标：" #: ../../source/production/observability/internal_api_server.rst:37 msgid "Port Configuration" -msgstr "" +msgstr "端口配置" #: ../../source/production/observability/internal_api_server.rst:39 msgid "" "The following environment variables are used implicitly with their " "default values:" -msgstr "" +msgstr "以下环境变量隐式使用其默认值：" #: ../../source/production/observability/internal_api_server.rst:41 msgid "Default Port Configuration" -msgstr "" +msgstr "默认端口配置" #: ../../source/production/observability/internal_api_server.rst:45 msgid "Environment Variable" -msgstr "" +msgstr "环境变量" #: ../../source/production/observability/internal_api_server.rst:46 msgid "Default Value" -msgstr "" +msgstr "默认值" #: ../../source/production/observability/internal_api_server.rst:47 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/production/observability/internal_api_server.rst:48 msgid "``LMCACHE_INTERNAL_API_SERVER_HOST``" -msgstr "" +msgstr "``LMCACHE_INTERNAL_API_SERVER_HOST``" #: ../../source/production/observability/internal_api_server.rst:49 msgid "``0.0.0.0``" -msgstr "" +msgstr "``0.0.0.0``" #: ../../source/production/observability/internal_api_server.rst:50 msgid "Host address for the internal API server to bind to." -msgstr "" +msgstr "内部 API 服务器绑定的主机地址。" #: ../../source/production/observability/internal_api_server.rst:51 msgid "``LMCACHE_INTERNAL_API_SERVER_PORT_START``" -msgstr "" +msgstr "``LMCACHE_INTERNAL_API_SERVER_PORT_START``" #: ../../source/production/observability/internal_api_server.rst:52 msgid "``6999``" -msgstr "" +msgstr "``6999``" #: ../../source/production/observability/internal_api_server.rst:53 msgid "Starting port number, e.g.:" -msgstr "" +msgstr "起始端口号，例如：" #: ../../source/production/observability/internal_api_server.rst:55 msgid "Scheduler: port_start + 0 (6999)" -msgstr "" +msgstr "调度程序：port_start + 0 (6999)" #: ../../source/production/observability/internal_api_server.rst:56 msgid "Worker 0: port_start + 1 (7000)" -msgstr "" +msgstr "工作线程 0: port_start + 1 (7000)" #: ../../source/production/observability/internal_api_server.rst:57 msgid "Worker 1: port_start + 2 (7001)" -msgstr "" +msgstr "Worker 1: port_start + 2 (7001)" #: ../../source/production/observability/internal_api_server.rst:60 msgid "Therefore, the metrics endpoint curl command above uses port 7000." -msgstr "" +msgstr "因此，上述指标端点的 curl 命令使用端口 7000。" #: ../../source/production/observability/internal_api_server.rst:63 msgid "Advanced Usage" -msgstr "" +msgstr "高级用法" #: ../../source/production/observability/internal_api_server.rst:65 msgid "" "For comprehensive testing and configuration options, refer to " ":ref:`testing_internal_api_server` for detailed examples and best " "practices." -msgstr "" +msgstr "有关全面的测试和配置选项，请参阅 :ref:`testing_internal_api_server` 以获取详细示例和最佳实践。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/metrics.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/metrics.po index a54606dacb1..be20252b224 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/metrics.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/metrics.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/production/observability/metrics.rst:4 msgid "Metrics Reference" -msgstr "" +msgstr "指标参考" #: ../../source/production/observability/metrics.rst:6 msgid "" @@ -29,22 +29,22 @@ msgid "" " performance, cache efficiency, and system health. These metrics are " "exposed via the vLLM ``/metrics`` endpoint when LMCache is integrated " "with vLLM, or via the LMCache internal API server." -msgstr "" +msgstr "LMCache 通过 Prometheus 提供全面的指标，以帮助您监控性能、缓存效率和系统健康。这些指标在 LMCache 与 vLLM 集成时通过 vLLM ``/metrics`` 端点公开，或者通过 LMCache 内部 API 服务器公开。" #: ../../source/production/observability/metrics.rst:9 msgid "Available Metrics" -msgstr "" +msgstr "可用指标" #: ../../source/production/observability/metrics.rst:11 msgid "" "The following tables list all available LMCache metrics organized by " "category." -msgstr "" +msgstr "以下表格列出了按类别组织的所有可用 LMCache 指标。" #: ../../source/production/observability/metrics.rst:14 #: ../../source/production/observability/metrics.rst:16 msgid "Core Request Metrics" -msgstr "" +msgstr "核心请求指标" #: ../../source/production/observability/metrics.rst:20 #: ../../source/production/observability/metrics.rst:40 @@ -60,7 +60,7 @@ msgstr "" #: ../../source/production/observability/metrics.rst:333 #: ../../source/production/observability/metrics.rst:371 msgid "Metric Name" -msgstr "" +msgstr "指标名称" #: ../../source/production/observability/metrics.rst:21 #: ../../source/production/observability/metrics.rst:41 @@ -76,7 +76,7 @@ msgstr "" #: ../../source/production/observability/metrics.rst:334 #: ../../source/production/observability/metrics.rst:372 msgid "Type" -msgstr "" +msgstr "类型" #: ../../source/production/observability/metrics.rst:22 #: ../../source/production/observability/metrics.rst:42 @@ -92,11 +92,11 @@ msgstr "" #: ../../source/production/observability/metrics.rst:335 #: ../../source/production/observability/metrics.rst:373 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/production/observability/metrics.rst:23 msgid "``lmcache:num_retrieve_requests``" -msgstr "" +msgstr "``lmcache:num_retrieve_requests``" #: ../../source/production/observability/metrics.rst:24 #: ../../source/production/observability/metrics.rst:27 @@ -124,97 +124,97 @@ msgstr "" #: ../../source/production/observability/metrics.rst:282 #: ../../source/production/observability/metrics.rst:285 msgid "Counter" -msgstr "" +msgstr "计数器" #: ../../source/production/observability/metrics.rst:25 msgid "Total number of retrieve requests sent to LMCache." -msgstr "" +msgstr "发送到 LMCache 的检索请求总数。" #: ../../source/production/observability/metrics.rst:26 msgid "``lmcache:num_store_requests``" -msgstr "" +msgstr "``lmcache:num_store_requests``" #: ../../source/production/observability/metrics.rst:28 msgid "Total number of store requests sent to LMCache." -msgstr "" +msgstr "发送到 LMCache 的存储请求总数。" #: ../../source/production/observability/metrics.rst:29 msgid "``lmcache:num_lookup_requests``" -msgstr "" +msgstr "``lmcache:num_lookup_requests``" #: ../../source/production/observability/metrics.rst:31 msgid "Total number of lookup requests sent to LMCache." -msgstr "" +msgstr "发送到 LMCache 的查找请求总数。" #: ../../source/production/observability/metrics.rst:34 #: ../../source/production/observability/metrics.rst:36 msgid "Token Metrics" -msgstr "" +msgstr "令牌指标" #: ../../source/production/observability/metrics.rst:43 msgid "``lmcache:num_requested_tokens``" -msgstr "" +msgstr "``lmcache:num_requested_tokens``" #: ../../source/production/observability/metrics.rst:45 msgid "Total number of tokens requested for retrieval." -msgstr "" +msgstr "请求检索的总令牌数。" #: ../../source/production/observability/metrics.rst:46 msgid "``lmcache:num_hit_tokens``" -msgstr "" +msgstr "``lmcache:num_hit_tokens``" #: ../../source/production/observability/metrics.rst:48 msgid "Total number of tokens hit in LMCache during retrieval." -msgstr "" +msgstr "在检索过程中，LMCache 中命中的总令牌数。" #: ../../source/production/observability/metrics.rst:49 msgid "``lmcache:num_stored_tokens``" -msgstr "" +msgstr "``lmcache:num_stored_tokens``" #: ../../source/production/observability/metrics.rst:51 msgid "Total number of tokens stored in LMCache." -msgstr "" +msgstr "LMCache 中存储的令牌总数。" #: ../../source/production/observability/metrics.rst:52 msgid "``lmcache:num_lookup_tokens``" -msgstr "" +msgstr "``lmcache:num_lookup_tokens``" #: ../../source/production/observability/metrics.rst:54 msgid "Total number of tokens requested in lookup operations." -msgstr "" +msgstr "查找操作中请求的令牌总数。" #: ../../source/production/observability/metrics.rst:55 msgid "``lmcache:num_lookup_hits``" -msgstr "" +msgstr "``lmcache:num_lookup_hits``" #: ../../source/production/observability/metrics.rst:57 msgid "Total number of tokens hit in lookup operations." -msgstr "" +msgstr "查找操作中命中的总令牌数。" #: ../../source/production/observability/metrics.rst:58 msgid "``lmcache:num_vllm_hit_tokens``" -msgstr "" +msgstr "``lmcache:num_vllm_hit_tokens``" #: ../../source/production/observability/metrics.rst:60 msgid "Number of hit tokens in vLLM." -msgstr "" +msgstr "vLLM 中的命中令牌数量。" #: ../../source/production/observability/metrics.rst:61 msgid "``lmcache:num_prompt_tokens``" -msgstr "" +msgstr "``lmcache:num_prompt_tokens``" #: ../../source/production/observability/metrics.rst:63 msgid "Number of prompt tokens in LMCache." -msgstr "" +msgstr "LMCache 中的提示令牌数量。" #: ../../source/production/observability/metrics.rst:66 #: ../../source/production/observability/metrics.rst:68 msgid "Hit Rate Metrics" -msgstr "" +msgstr "命中率指标" #: ../../source/production/observability/metrics.rst:75 msgid "``lmcache:retrieve_hit_rate``" -msgstr "" +msgstr "``lmcache:retrieve_hit_rate``" #: ../../source/production/observability/metrics.rst:76 #: ../../source/production/observability/metrics.rst:79 @@ -252,23 +252,23 @@ msgstr "" #: ../../source/production/observability/metrics.rst:390 #: ../../source/production/observability/metrics.rst:393 msgid "Gauge" -msgstr "" +msgstr "仪表" #: ../../source/production/observability/metrics.rst:77 msgid "The hit rate for retrieve requests since last log." -msgstr "" +msgstr "自上次日志以来，检索请求的命中率。" #: ../../source/production/observability/metrics.rst:78 msgid "``lmcache:lookup_hit_rate``" -msgstr "" +msgstr "``lmcache:lookup_hit_rate``" #: ../../source/production/observability/metrics.rst:80 msgid "The hit rate for lookup requests since last log." -msgstr "" +msgstr "自上次日志以来查找请求的命中率。" #: ../../source/production/observability/metrics.rst:81 msgid "``lmcache:request_cache_hit_rate``" -msgstr "" +msgstr "``lmcache:request_cache_hit_rate``" #: ../../source/production/observability/metrics.rst:82 #: ../../source/production/observability/metrics.rst:99 @@ -291,609 +291,609 @@ msgstr "" #: ../../source/production/observability/metrics.rst:288 #: ../../source/production/observability/metrics.rst:291 msgid "Histogram" -msgstr "" +msgstr "直方图" #: ../../source/production/observability/metrics.rst:83 msgid "Distribution of hit rates per request." -msgstr "" +msgstr "每个请求的命中率分布。" #: ../../source/production/observability/metrics.rst:84 msgid "``lmcache:lookup_0_hit_requests``" -msgstr "" +msgstr "``lmcache:lookup_0_hit_requests``" #: ../../source/production/observability/metrics.rst:86 msgid "Total number of lookup requests with zero hits." -msgstr "" +msgstr "查找请求的总数，其中没有命中。" #: ../../source/production/observability/metrics.rst:89 #: ../../source/production/observability/metrics.rst:91 msgid "Performance & Latency Metrics" -msgstr "" +msgstr "性能与延迟指标" #: ../../source/production/observability/metrics.rst:98 msgid "``lmcache:time_to_retrieve``" -msgstr "" +msgstr "``lmcache:time_to_retrieve``" #: ../../source/production/observability/metrics.rst:100 msgid "Time taken to retrieve from the cache (seconds)." -msgstr "" +msgstr "从缓存中检索所花费的时间（秒）。" #: ../../source/production/observability/metrics.rst:101 msgid "``lmcache:time_to_store``" -msgstr "" +msgstr "``lmcache:time_to_store``" #: ../../source/production/observability/metrics.rst:103 msgid "Time taken to store to the cache (seconds)." -msgstr "" +msgstr "存储到缓存所需的时间（秒）。" #: ../../source/production/observability/metrics.rst:104 msgid "``lmcache:time_to_lookup``" -msgstr "" +msgstr "``lmcache:time_to_lookup``" #: ../../source/production/observability/metrics.rst:106 msgid "Time taken to perform a lookup in the cache (seconds)." -msgstr "" +msgstr "在缓存中执行查找所花费的时间（秒）。" #: ../../source/production/observability/metrics.rst:107 msgid "``lmcache:retrieve_speed``" -msgstr "" +msgstr "``lmcache:retrieve_speed``" #: ../../source/production/observability/metrics.rst:109 msgid "Retrieval speed (tokens per second)." -msgstr "" +msgstr "检索速度（每秒令牌数）。" #: ../../source/production/observability/metrics.rst:110 msgid "``lmcache:store_speed``" -msgstr "" +msgstr "``lmcache:store_speed``" #: ../../source/production/observability/metrics.rst:112 msgid "Storage speed (tokens per second)." -msgstr "" +msgstr "存储速度（每秒令牌数）。" #: ../../source/production/observability/metrics.rst:113 msgid "``lmcache:num_slow_retrieval_by_time``" -msgstr "" +msgstr "``lmcache:num_slow_retrieval_by_time``" #: ../../source/production/observability/metrics.rst:115 msgid "Total number of slow retrievals exceeding the time threshold." -msgstr "" +msgstr "超过时间阈值的慢检索总数。" #: ../../source/production/observability/metrics.rst:116 msgid "``lmcache:num_slow_retrieval_by_speed``" -msgstr "" +msgstr "``lmcache:num_slow_retrieval_by_speed``" #: ../../source/production/observability/metrics.rst:118 msgid "Total number of slow retrievals below the speed threshold." -msgstr "" +msgstr "低于速度阈值的慢检索总数。" #: ../../source/production/observability/metrics.rst:121 msgid "Detailed Profiling Metrics" -msgstr "" +msgstr "详细剖析指标" #: ../../source/production/observability/metrics.rst:123 msgid "Profiling Metrics" -msgstr "" +msgstr "剖析指标" #: ../../source/production/observability/metrics.rst:130 msgid "``lmcache:retrieve_process_tokens_time``" -msgstr "" +msgstr "``lmcache:retrieve_process_tokens_time``" #: ../../source/production/observability/metrics.rst:132 msgid "Time to process tokens in retrieve (seconds)." -msgstr "" +msgstr "处理检索中令牌的时间（秒）。" #: ../../source/production/observability/metrics.rst:133 msgid "``lmcache:retrieve_broadcast_time``" -msgstr "" +msgstr "``lmcache:retrieve_broadcast_time``" #: ../../source/production/observability/metrics.rst:135 msgid "Time to broadcast memory objects in retrieve (seconds)." -msgstr "" +msgstr "在检索中广播内存对象的时间（秒）。" #: ../../source/production/observability/metrics.rst:136 msgid "``lmcache:retrieve_to_gpu_time``" -msgstr "" +msgstr "``lmcache:retrieve_to_gpu_time``" #: ../../source/production/observability/metrics.rst:138 msgid "Time to move data to GPU in retrieve (seconds)." -msgstr "" +msgstr "将数据移动到 GPU 的检索时间（秒）。" #: ../../source/production/observability/metrics.rst:139 msgid "``lmcache:store_process_tokens_time``" -msgstr "" +msgstr "``lmcache:store_process_tokens_time``" #: ../../source/production/observability/metrics.rst:141 msgid "Time to process tokens in store (seconds)." -msgstr "" +msgstr "存储中处理令牌的时间（秒）。" #: ../../source/production/observability/metrics.rst:142 msgid "``lmcache:store_from_gpu_time``" -msgstr "" +msgstr "``lmcache:store_from_gpu_time``" #: ../../source/production/observability/metrics.rst:144 msgid "Time to move data from GPU in store (seconds)." -msgstr "" +msgstr "从 GPU 移动数据到存储的时间（秒）。" #: ../../source/production/observability/metrics.rst:145 msgid "``lmcache:store_put_time``" -msgstr "" +msgstr "``lmcache:store_put_time``" #: ../../source/production/observability/metrics.rst:147 msgid "Time to put data to storage in store (seconds)." -msgstr "" +msgstr "将数据存储到存储中的时间（秒）。" #: ../../source/production/observability/metrics.rst:148 msgid "``lmcache:remote_backend_batched_get_blocking_time``" -msgstr "" +msgstr "``lmcache:remote_backend_batched_get_blocking_time``" #: ../../source/production/observability/metrics.rst:150 msgid "Time spent waiting for data from remote backend (seconds)." -msgstr "" +msgstr "等待从远程后端获取数据的时间（秒）。" #: ../../source/production/observability/metrics.rst:151 msgid "``lmcache:instrumented_connector_batched_get_time``" -msgstr "" +msgstr "``lmcache:instrumented_connector_batched_get_time``" #: ../../source/production/observability/metrics.rst:153 msgid "Time spent in the connector layer (seconds)." -msgstr "" +msgstr "连接器层花费的时间（秒）。" #: ../../source/production/observability/metrics.rst:156 msgid "Cache Usage & Lifecycle Metrics" -msgstr "" +msgstr "缓存使用与生命周期指标" #: ../../source/production/observability/metrics.rst:158 msgid "Cache Usage Metrics" -msgstr "" +msgstr "缓存使用指标" #: ../../source/production/observability/metrics.rst:165 msgid "``lmcache:local_cache_usage``" -msgstr "" +msgstr "``lmcache:local_cache_usage``" #: ../../source/production/observability/metrics.rst:167 msgid "Local cache usage in bytes." -msgstr "" +msgstr "本地缓存使用量（以字节为单位）。" #: ../../source/production/observability/metrics.rst:168 msgid "``lmcache:remote_cache_usage``" -msgstr "" +msgstr "``lmcache:remote_cache_usage``" #: ../../source/production/observability/metrics.rst:170 msgid "Remote cache usage in bytes." -msgstr "" +msgstr "远程缓存使用量（以字节为单位）。" #: ../../source/production/observability/metrics.rst:171 msgid "``lmcache:local_storage_usage``" -msgstr "" +msgstr "``lmcache:local_storage_usage``" #: ../../source/production/observability/metrics.rst:173 msgid "Local storage usage in bytes." -msgstr "" +msgstr "本地存储使用量（以字节为单位）。" #: ../../source/production/observability/metrics.rst:174 msgid "``lmcache:request_cache_lifespan``" -msgstr "" +msgstr "``lmcache:request_cache_lifespan``" #: ../../source/production/observability/metrics.rst:176 msgid "Distribution of request cache lifespan in minutes." -msgstr "" +msgstr "请求缓存生命周期的分布（以分钟为单位）。" #: ../../source/production/observability/metrics.rst:179 msgid "Remote Backend & Network Metrics" -msgstr "" +msgstr "远程后端与网络指标" #: ../../source/production/observability/metrics.rst:181 msgid "Remote Backend Metrics" -msgstr "" +msgstr "远程后端指标" #: ../../source/production/observability/metrics.rst:188 msgid "``lmcache:num_remote_read_requests``" -msgstr "" +msgstr "``lmcache:num_remote_read_requests``" #: ../../source/production/observability/metrics.rst:190 msgid "Total number of read requests to remote backends." -msgstr "" +msgstr "对远程后端的读取请求总数。" #: ../../source/production/observability/metrics.rst:191 msgid "``lmcache:num_remote_read_bytes``" -msgstr "" +msgstr "``lmcache:num_remote_read_bytes``" #: ../../source/production/observability/metrics.rst:193 msgid "Total number of bytes read from remote backends." -msgstr "" +msgstr "从远程后端读取的字节总数。" #: ../../source/production/observability/metrics.rst:194 msgid "``lmcache:num_remote_write_requests``" -msgstr "" +msgstr "``lmcache:num_remote_write_requests``" #: ../../source/production/observability/metrics.rst:196 msgid "Total number of write requests to remote backends." -msgstr "" +msgstr "对远程后端的写请求总数。" #: ../../source/production/observability/metrics.rst:197 msgid "``lmcache:num_remote_write_bytes``" -msgstr "" +msgstr "``lmcache:num_remote_write_bytes``" #: ../../source/production/observability/metrics.rst:199 msgid "Total number of bytes written to remote backends." -msgstr "" +msgstr "写入远程后端的字节总数。" #: ../../source/production/observability/metrics.rst:200 msgid "``lmcache:remote_time_to_get``" -msgstr "" +msgstr "``lmcache:remote_time_to_get``" #: ../../source/production/observability/metrics.rst:202 msgid "Time taken to get data from remote backends (ms)." -msgstr "" +msgstr "从远程后端获取数据所花费的时间（毫秒）。" #: ../../source/production/observability/metrics.rst:203 msgid "``lmcache:remote_time_to_put``" -msgstr "" +msgstr "``lmcache:remote_time_to_put``" #: ../../source/production/observability/metrics.rst:205 msgid "Time taken to put data to remote backends (ms)." -msgstr "" +msgstr "将数据放入远程后端所花费的时间（毫秒）。" #: ../../source/production/observability/metrics.rst:206 msgid "``lmcache:remote_time_to_get_sync``" -msgstr "" +msgstr "``lmcache:remote_time_to_get_sync``" #: ../../source/production/observability/metrics.rst:208 msgid "Time taken to get data from remote backends synchronously (ms)." -msgstr "" +msgstr "从远程后端同步获取数据所花费的时间（毫秒）。" #: ../../source/production/observability/metrics.rst:209 msgid "``lmcache:remote_ping_latency``" -msgstr "" +msgstr "``lmcache:remote_ping_latency``" #: ../../source/production/observability/metrics.rst:211 msgid "Latest ping latency to remote backends (ms)." -msgstr "" +msgstr "最新的远程后端 ping 延迟（毫秒）。" #: ../../source/production/observability/metrics.rst:212 msgid "``lmcache:remote_ping_errors``" -msgstr "" +msgstr "``lmcache:remote_ping_errors``" #: ../../source/production/observability/metrics.rst:214 msgid "Total number of ping errors to remote backends." -msgstr "" +msgstr "远程后端的总 ping 错误数量。" #: ../../source/production/observability/metrics.rst:215 msgid "``lmcache:remote_ping_successes``" -msgstr "" +msgstr "``lmcache:remote_ping_successes``" #: ../../source/production/observability/metrics.rst:217 msgid "Total number of successful pings to remote backends." -msgstr "" +msgstr "成功 ping 远程后端的总次数。" #: ../../source/production/observability/metrics.rst:218 msgid "``lmcache:remote_ping_error_code``" -msgstr "" +msgstr "``lmcache:remote_ping_error_code``" #: ../../source/production/observability/metrics.rst:220 msgid "Latest ping error code to remote backends." -msgstr "" +msgstr "远程后端的最新 ping 错误代码。" #: ../../source/production/observability/metrics.rst:223 #: ../../source/production/observability/metrics.rst:225 msgid "Local CPU Backend Metrics" -msgstr "" +msgstr "本地 CPU 后端指标" #: ../../source/production/observability/metrics.rst:232 msgid "``lmcache:local_cpu_evict_count``" -msgstr "" +msgstr "``lmcache:local_cpu_evict_count``" #: ../../source/production/observability/metrics.rst:234 msgid "Total number of evictions in local CPU backend." -msgstr "" +msgstr "本地 CPU 后端的总逐出次数。" #: ../../source/production/observability/metrics.rst:235 msgid "``lmcache:local_cpu_evict_keys_count``" -msgstr "" +msgstr "``lmcache:local_cpu_evict_keys_count``" #: ../../source/production/observability/metrics.rst:237 msgid "Total number of evicted keys in local CPU backend." -msgstr "" +msgstr "本地 CPU 后端中被逐出的键的总数。" #: ../../source/production/observability/metrics.rst:238 msgid "``lmcache:local_cpu_evict_failed_count``" -msgstr "" +msgstr "``lmcache:local_cpu_evict_failed_count``" #: ../../source/production/observability/metrics.rst:240 msgid "Total number of failed evictions in local CPU backend." -msgstr "" +msgstr "本地 CPU 后端中失败的逐出总数。" #: ../../source/production/observability/metrics.rst:241 msgid "``lmcache:local_cpu_hot_cache_count``" -msgstr "" +msgstr "``lmcache:local_cpu_hot_cache_count``" #: ../../source/production/observability/metrics.rst:243 msgid "Current number of items in the hot cache." -msgstr "" +msgstr "热缓存中当前项目的数量。" #: ../../source/production/observability/metrics.rst:244 msgid "``lmcache:local_cpu_keys_in_request_count``" -msgstr "" +msgstr "``lmcache:local_cpu_keys_in_request_count``" #: ../../source/production/observability/metrics.rst:246 msgid "Current number of keys being processed in requests." -msgstr "" +msgstr "当前正在处理的请求中的键的数量。" #: ../../source/production/observability/metrics.rst:249 #: ../../source/production/observability/metrics.rst:251 msgid "Memory Management Metrics" -msgstr "" +msgstr "内存管理指标" #: ../../source/production/observability/metrics.rst:258 msgid "``lmcache:active_memory_objs_count``" -msgstr "" +msgstr "``lmcache:active_memory_objs_count``" #: ../../source/production/observability/metrics.rst:260 msgid "The number of currently active memory objects." -msgstr "" +msgstr "当前活动内存对象的数量。" #: ../../source/production/observability/metrics.rst:261 msgid "``lmcache:pinned_memory_objs_count``" -msgstr "" +msgstr "``lmcache:pinned_memory_objs_count``" #: ../../source/production/observability/metrics.rst:263 msgid "The number of currently pinned memory objects." -msgstr "" +msgstr "当前固定内存对象的数量。" #: ../../source/production/observability/metrics.rst:264 msgid "``lmcache:forced_unpin_count``" -msgstr "" +msgstr "``lmcache:forced_unpin_count``" #: ../../source/production/observability/metrics.rst:266 msgid "Total number of forced unpins due to timeout." -msgstr "" +msgstr "由于超时而强制逐出的总数。" #: ../../source/production/observability/metrics.rst:267 msgid "``lmcache:pin_monitor_pinned_objects_count``" -msgstr "" +msgstr "``lmcache:pin_monitor_pinned_objects_count``" #: ../../source/production/observability/metrics.rst:269 msgid "The number of pinned objects tracked by the PinMonitor." -msgstr "" +msgstr "PinMonitor 追踪的固定对象数量。" #: ../../source/production/observability/metrics.rst:272 #: ../../source/production/observability/metrics.rst:274 msgid "P2P Transfer Metrics" -msgstr "" +msgstr "P2P 转移指标" #: ../../source/production/observability/metrics.rst:281 msgid "``lmcache:num_p2p_requests``" -msgstr "" +msgstr "``lmcache:num_p2p_requests``" #: ../../source/production/observability/metrics.rst:283 msgid "Total number of P2P transfer requests." -msgstr "" +msgstr "P2P 转移请求的总数。" #: ../../source/production/observability/metrics.rst:284 msgid "``lmcache:num_p2p_transferred_tokens``" -msgstr "" +msgstr "``lmcache:num_p2p_transferred_tokens``" #: ../../source/production/observability/metrics.rst:286 msgid "Total number of tokens transferred via P2P." -msgstr "" +msgstr "通过 P2P 传输的令牌总数。" #: ../../source/production/observability/metrics.rst:287 msgid "``lmcache:p2p_time_to_transfer``" -msgstr "" +msgstr "``lmcache:p2p_time_to_transfer``" #: ../../source/production/observability/metrics.rst:289 msgid "Time taken for P2P transfers (seconds)." -msgstr "" +msgstr "P2P 传输所需时间（秒）。" #: ../../source/production/observability/metrics.rst:290 msgid "``lmcache:p2p_transfer_speed``" -msgstr "" +msgstr "``lmcache:p2p_transfer_speed``" #: ../../source/production/observability/metrics.rst:292 msgid "P2P transfer speed (tokens per second)." -msgstr "" +msgstr "P2P 传输速度（每秒令牌数）。" #: ../../source/production/observability/metrics.rst:295 msgid "Health & Internal System Metrics" -msgstr "" +msgstr "健康与内部系统指标" #: ../../source/production/observability/metrics.rst:297 msgid "Health & Internal Metrics" -msgstr "" +msgstr "健康与内部指标" #: ../../source/production/observability/metrics.rst:304 msgid "``lmcache:lmcache_is_healthy``" -msgstr "" +msgstr "``lmcache:lmcache_is_healthy``" #: ../../source/production/observability/metrics.rst:306 msgid "Overall health status of LMCache (1 = healthy, 0 = unhealthy)." -msgstr "" +msgstr "LMCache 的整体健康状态（1 = 健康，0 = 不健康）。" #: ../../source/production/observability/metrics.rst:307 msgid "``lmcache:interval_get_blocking_failed_count``" -msgstr "" +msgstr "``lmcache:interval_get_blocking_failed_count``" #: ../../source/production/observability/metrics.rst:309 msgid "Number of failed blocking get operations in the current interval." -msgstr "" +msgstr "当前时间间隔内失败的阻塞获取操作数量。" #: ../../source/production/observability/metrics.rst:310 msgid "``lmcache:kv_msg_queue_size``" -msgstr "" +msgstr "``lmcache:kv_msg_queue_size``" #: ../../source/production/observability/metrics.rst:312 msgid "Size of the KV message queue in the BatchedMessageSender." -msgstr "" +msgstr "批量消息发送器中 KV 消息队列的大小。" #: ../../source/production/observability/metrics.rst:313 msgid "``lmcache:remote_put_task_num``" -msgstr "" +msgstr "``lmcache:remote_put_task_num``" #: ../../source/production/observability/metrics.rst:315 msgid "Number of pending remote put tasks." -msgstr "" +msgstr "待处理的远程放置任务数量。" #: ../../source/production/observability/metrics.rst:316 msgid "``lmcache:storage_events_ongoing_count``" -msgstr "" +msgstr "``lmcache:storage_events_ongoing_count``" #: ../../source/production/observability/metrics.rst:318 msgid "Number of storage events currently in progress." -msgstr "" +msgstr "当前进行中的存储事件数量。" #: ../../source/production/observability/metrics.rst:319 msgid "``lmcache:storage_events_done_count``" -msgstr "" +msgstr "``lmcache:storage_events_done_count``" #: ../../source/production/observability/metrics.rst:321 msgid "Number of storage events completed successfully." -msgstr "" +msgstr "成功完成的存储事件数量。" #: ../../source/production/observability/metrics.rst:322 msgid "``lmcache:storage_events_not_found_count``" -msgstr "" +msgstr "``lmcache:storage_events_not_found_count``" #: ../../source/production/observability/metrics.rst:324 msgid "Number of storage events where the requested data was not found." -msgstr "" +msgstr "未找到请求数据的存储事件数量。" #: ../../source/production/observability/metrics.rst:327 #: ../../source/production/observability/metrics.rst:329 msgid "Chunk Statistics Metrics" -msgstr "" +msgstr "块统计指标" #: ../../source/production/observability/metrics.rst:336 msgid "``lmcache:chunk_statistics_enabled``" -msgstr "" +msgstr "``lmcache:chunk_statistics_enabled``" #: ../../source/production/observability/metrics.rst:338 msgid "" "Whether chunk statistics collection is enabled (1 = enabled, 0 = " "disabled)." -msgstr "" +msgstr "是否启用块统计收集（1 = 启用，0 = 禁用）。" #: ../../source/production/observability/metrics.rst:339 msgid "``lmcache:chunk_statistics_total_requests``" -msgstr "" +msgstr "``lmcache:chunk_statistics_total_requests``" #: ../../source/production/observability/metrics.rst:341 msgid "Total number of requests processed by chunk statistics." -msgstr "" +msgstr "按块统计处理的请求总数。" #: ../../source/production/observability/metrics.rst:342 msgid "``lmcache:chunk_statistics_total_chunks``" -msgstr "" +msgstr "``lmcache:chunk_statistics_total_chunks``" #: ../../source/production/observability/metrics.rst:344 msgid "Total number of chunks processed." -msgstr "" +msgstr "处理的块总数。" #: ../../source/production/observability/metrics.rst:345 msgid "``lmcache:chunk_statistics_unique_chunks``" -msgstr "" +msgstr "``lmcache:chunk_statistics_unique_chunks``" #: ../../source/production/observability/metrics.rst:347 msgid "Estimated number of unique chunks encountered." -msgstr "" +msgstr "估计遇到的唯一块的数量。" #: ../../source/production/observability/metrics.rst:348 msgid "``lmcache:chunk_statistics_reuse_rate``" -msgstr "" +msgstr "``lmcache:chunk_statistics_reuse_rate``" #: ../../source/production/observability/metrics.rst:350 msgid "Chunk reuse rate (0.0 to 1.0)." -msgstr "" +msgstr "块重用率（0.0 到 1.0）。" #: ../../source/production/observability/metrics.rst:351 msgid "``lmcache:chunk_statistics_bloom_filter_size_mb``" -msgstr "" +msgstr "``lmcache:chunk_statistics_bloom_filter_size_mb``" #: ../../source/production/observability/metrics.rst:353 msgid "Memory usage of the Bloom filter in megabytes." -msgstr "" +msgstr "布隆过滤器的内存使用量（以兆字节为单位）。" #: ../../source/production/observability/metrics.rst:354 msgid "``lmcache:chunk_statistics_bloom_filter_fill_rate``" -msgstr "" +msgstr "``lmcache:chunk_statistics_bloom_filter_fill_rate``" #: ../../source/production/observability/metrics.rst:356 msgid "Fill rate of the Bloom filter (0.0 to 1.0)." -msgstr "" +msgstr "布隆过滤器的填充率（0.0 到 1.0）。" #: ../../source/production/observability/metrics.rst:357 msgid "``lmcache:chunk_statistics_file_count``" -msgstr "" +msgstr "``lmcache:chunk_statistics_file_count``" #: ../../source/production/observability/metrics.rst:359 msgid "Number of files created when using the ``file_hash`` strategy." -msgstr "" +msgstr "使用 ``file_hash`` 策略时创建的文件数量。" #: ../../source/production/observability/metrics.rst:360 msgid "``lmcache:chunk_statistics_current_file_size``" -msgstr "" +msgstr "``lmcache:chunk_statistics_current_file_size``" #: ../../source/production/observability/metrics.rst:362 msgid "Current size of the active statistics file in bytes." -msgstr "" +msgstr "活动统计文件的当前大小（以字节为单位）。" #: ../../source/production/observability/metrics.rst:365 #: ../../source/production/observability/metrics.rst:367 msgid "Connector Metrics" -msgstr "" +msgstr "连接器指标" #: ../../source/production/observability/metrics.rst:374 msgid "``lmcache:scheduler_unfinished_requests_count``" -msgstr "" +msgstr "``lmcache:scheduler_unfinished_requests_count``" #: ../../source/production/observability/metrics.rst:376 msgid "Current count of unfinished requests in the scheduler." -msgstr "" +msgstr "调度器中未完成请求的当前计数。" #: ../../source/production/observability/metrics.rst:377 msgid "``lmcache:connector_load_specs_count``" -msgstr "" +msgstr "``lmcache:connector_load_specs_count``" #: ../../source/production/observability/metrics.rst:379 msgid "Number of load specifications currently in the connector." -msgstr "" +msgstr "连接器中当前的加载规范数量。" #: ../../source/production/observability/metrics.rst:380 msgid "``lmcache:connector_request_trackers_count``" -msgstr "" +msgstr "``lmcache:connector_request_trackers_count``" #: ../../source/production/observability/metrics.rst:382 msgid "Number of active request trackers in the connector." -msgstr "" +msgstr "连接器中活动请求跟踪器的数量。" #: ../../source/production/observability/metrics.rst:383 msgid "``lmcache:connector_kv_caches_count``" -msgstr "" +msgstr "``lmcache:connector_kv_caches_count``" #: ../../source/production/observability/metrics.rst:385 msgid "Number of KV caches currently managed by the connector." -msgstr "" +msgstr "连接器当前管理的 KV Cache 数量。" #: ../../source/production/observability/metrics.rst:386 msgid "``lmcache:connector_layerwise_retrievers_count``" -msgstr "" +msgstr "``lmcache:connector_layerwise_retrievers_count``" #: ../../source/production/observability/metrics.rst:388 msgid "Number of layer-wise retrievers active in the connector." -msgstr "" +msgstr "连接器中活动的逐层检索器数量。" #: ../../source/production/observability/metrics.rst:389 msgid "``lmcache:connector_invalid_block_ids_count``" -msgstr "" +msgstr "``lmcache:connector_invalid_block_ids_count``" #: ../../source/production/observability/metrics.rst:391 msgid "Number of invalid block IDs encountered by the connector." -msgstr "" +msgstr "连接器遇到的无效块 ID 的数量。" #: ../../source/production/observability/metrics.rst:392 msgid "``lmcache:connector_requests_priority_count``" -msgstr "" +msgstr "``lmcache:connector_requests_priority_count``" #: ../../source/production/observability/metrics.rst:394 msgid "Number of requests prioritized by the connector." -msgstr "" +msgstr "连接器优先处理的请求数量。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/periodic_thread_api.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/periodic_thread_api.po index 6032dcf7a8e..67f808524d1 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/periodic_thread_api.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/periodic_thread_api.po @@ -21,17 +21,17 @@ msgstr "" #: ../../source/production/observability/periodic_thread_api.rst:4 msgid "Periodic Thread Monitoring API" -msgstr "" +msgstr "周期性线程监控 API" #: ../../source/production/observability/periodic_thread_api.rst:6 msgid "" "This document describes the Periodic Thread Monitoring API feature " "introduced in LMCache." -msgstr "" +msgstr "本文描述了在 LMCache 中引入的周期性线程监控 API 功能。" #: ../../source/production/observability/periodic_thread_api.rst:10 msgid "Overview" -msgstr "" +msgstr "概述" #: ../../source/production/observability/periodic_thread_api.rst:12 msgid "" @@ -39,194 +39,194 @@ msgid "" " inspect the status of all periodic background threads running in the " "LMCache system. This is useful for debugging, health checking, and " "operational monitoring." -msgstr "" +msgstr "周期性线程监控 API 提供 HTTP 端点，以监控和检查在 LMCache 系统中运行的所有周期性后台线程的状态。这对于调试、健康检查和操作监控非常有用。" #: ../../source/production/observability/periodic_thread_api.rst:18 msgid "API Endpoints" -msgstr "" +msgstr "API 端点" #: ../../source/production/observability/periodic_thread_api.rst:20 msgid "Three API endpoints are available under the internal API server:" -msgstr "" +msgstr "在内部 API 服务器下提供三个 API 端点：" #: ../../source/production/observability/periodic_thread_api.rst:23 msgid "GET /periodic-threads" -msgstr "" +msgstr "GET /periodic-threads" #: ../../source/production/observability/periodic_thread_api.rst:25 msgid "Returns information about all registered periodic threads." -msgstr "" +msgstr "返回所有注册的周期线程的信息。" #: ../../source/production/observability/periodic_thread_api.rst:27 msgid "**Query Parameters:**" -msgstr "" +msgstr "**查询参数：**" #: ../../source/production/observability/periodic_thread_api.rst:33 msgid "Parameter" -msgstr "" +msgstr "参数" #: ../../source/production/observability/periodic_thread_api.rst:34 msgid "Default" -msgstr "" +msgstr "默认" #: ../../source/production/observability/periodic_thread_api.rst:35 #: ../../source/production/observability/periodic_thread_api.rst:121 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/production/observability/periodic_thread_api.rst:36 msgid "``level``" -msgstr "" +msgstr "``level``" #: ../../source/production/observability/periodic_thread_api.rst:37 msgid "(none)" -msgstr "" +msgstr "（无）" #: ../../source/production/observability/periodic_thread_api.rst:38 msgid "Filter by thread level (``critical``, ``high``, ``medium``, ``low``)" -msgstr "" +msgstr "按线程级别过滤（``critical``, ``high``, ``medium``, ``low``）" #: ../../source/production/observability/periodic_thread_api.rst:39 msgid "``running_only``" -msgstr "" +msgstr "``running_only``" #: ../../source/production/observability/periodic_thread_api.rst:40 #: ../../source/production/observability/periodic_thread_api.rst:43 msgid "``false``" -msgstr "" +msgstr "``false``" #: ../../source/production/observability/periodic_thread_api.rst:41 msgid "Only show running threads" -msgstr "" +msgstr "仅显示正在运行的线程" #: ../../source/production/observability/periodic_thread_api.rst:42 msgid "``active_only``" -msgstr "" +msgstr "``active_only``" #: ../../source/production/observability/periodic_thread_api.rst:44 msgid "Only show active threads" -msgstr "" +msgstr "仅显示活动线程" #: ../../source/production/observability/periodic_thread_api.rst:46 #: ../../source/production/observability/periodic_thread_api.rst:79 #: ../../source/production/observability/periodic_thread_api.rst:101 msgid "**Response Example:**" -msgstr "" +msgstr "**响应示例：**" #: ../../source/production/observability/periodic_thread_api.rst:75 #, python-brace-format msgid "GET /periodic-threads/{thread_name}" -msgstr "" +msgstr "GET /periodic-threads/{thread_name}" #: ../../source/production/observability/periodic_thread_api.rst:77 msgid "Returns detailed information about a specific periodic thread." -msgstr "" +msgstr "返回有关特定周期线程的详细信息。" #: ../../source/production/observability/periodic_thread_api.rst:96 msgid "GET /periodic-threads-health" -msgstr "" +msgstr "GET /periodic-threads-health" #: ../../source/production/observability/periodic_thread_api.rst:98 msgid "" "Quick health check for periodic threads. Returns whether all critical and" " high-level threads are active." -msgstr "" +msgstr "对周期性线程的快速健康检查。返回所有关键和高级线程是否处于活动状态。" #: ../../source/production/observability/periodic_thread_api.rst:112 msgid "Thread Levels" -msgstr "" +msgstr "线程级别" #: ../../source/production/observability/periodic_thread_api.rst:114 msgid "Periodic threads are categorized by importance level:" -msgstr "" +msgstr "周期性线程按重要性级别分类：" #: ../../source/production/observability/periodic_thread_api.rst:120 msgid "Level" -msgstr "" +msgstr "级别" #: ../../source/production/observability/periodic_thread_api.rst:122 msgid "``critical``" -msgstr "" +msgstr "``critical``" #: ../../source/production/observability/periodic_thread_api.rst:123 msgid "Essential for system operation" -msgstr "" +msgstr "系统运行所必需" #: ../../source/production/observability/periodic_thread_api.rst:124 msgid "``high``" -msgstr "" +msgstr "``high``" #: ../../source/production/observability/periodic_thread_api.rst:125 msgid "Important for performance" -msgstr "" +msgstr "对性能很重要" #: ../../source/production/observability/periodic_thread_api.rst:126 msgid "``medium``" -msgstr "" +msgstr "``medium``" #: ../../source/production/observability/periodic_thread_api.rst:127 msgid "Standard background tasks" -msgstr "" +msgstr "标准后台任务" #: ../../source/production/observability/periodic_thread_api.rst:128 msgid "``low``" -msgstr "" +msgstr "``low``" #: ../../source/production/observability/periodic_thread_api.rst:129 msgid "Optional/auxiliary tasks" -msgstr "" +msgstr "可选/辅助任务" #: ../../source/production/observability/periodic_thread_api.rst:131 msgid "" "The health check endpoint specifically monitors ``critical`` and ``high``" " level threads to determine system health." -msgstr "" +msgstr "健康检查端点专门监控 ``critical`` 和 ``high`` 级别的线程以确定系统健康状况。" #: ../../source/production/observability/periodic_thread_api.rst:135 msgid "IrrecoverableException Handling" -msgstr "" +msgstr "不可恢复异常处理" #: ../../source/production/observability/periodic_thread_api.rst:137 msgid "" "Periodic threads now properly handle ``IrrecoverableException``. When " "such an exception is raised during thread execution:" -msgstr "" +msgstr "周期性线程现在正确处理 ``IrrecoverableException``。当在线程执行期间引发此类异常时：" #: ../../source/production/observability/periodic_thread_api.rst:140 msgid "The exception is logged with full traceback" -msgstr "" +msgstr "该异常的完整回溯将被记录。" #: ../../source/production/observability/periodic_thread_api.rst:141 msgid "The thread run is marked as failed" -msgstr "" +msgstr "线程运行被标记为失败" #: ../../source/production/observability/periodic_thread_api.rst:142 msgid "The thread **stops its execution loop** instead of continuing" -msgstr "" +msgstr "线程 **停止其执行循环** 而不是继续执行。" #: ../../source/production/observability/periodic_thread_api.rst:144 msgid "" "This prevents threads from endlessly retrying operations that cannot " "succeed." -msgstr "" +msgstr "这可以防止线程无休止地重试无法成功的操作。" #: ../../source/production/observability/periodic_thread_api.rst:147 msgid "Usage Examples" -msgstr "" +msgstr "使用示例" #: ../../source/production/observability/periodic_thread_api.rst:150 msgid "Check Overall Thread Health" -msgstr "" +msgstr "检查整体线程健康状况" #: ../../source/production/observability/periodic_thread_api.rst:157 msgid "List All Running Threads" -msgstr "" +msgstr "列出所有正在运行的线程" #: ../../source/production/observability/periodic_thread_api.rst:164 msgid "Get Critical Threads Only" -msgstr "" +msgstr "仅获取关键线程" #: ../../source/production/observability/periodic_thread_api.rst:171 msgid "Check Specific Thread Status" -msgstr "" +msgstr "检查特定线程状态" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/vllm_endpoint.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/vllm_endpoint.po index 745f29f5f3c..19ca9c7cc8f 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/vllm_endpoint.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/observability/vllm_endpoint.po @@ -21,7 +21,7 @@ msgstr "" #: ../../source/production/observability/vllm_endpoint.rst:4 msgid "Metrics by vLLM API" -msgstr "" +msgstr "通过 vLLM API 的指标" #: ../../source/production/observability/vllm_endpoint.rst:6 msgid "" @@ -29,84 +29,84 @@ msgid "" " in-depth monitoring of cache performance and behavior. This section " "outlines how to enable and configure observability from embedded vLLM " "``/metrics`` API endpoint." -msgstr "" +msgstr "LMCache 通过 Prometheus 端点提供详细的指标，允许对缓存性能和行为进行深入监控。本节概述了如何启用和配置来自嵌入式 vLLM ``/metrics`` API 端点的可观察性。" #: ../../source/production/observability/vllm_endpoint.rst:11 msgid "Quick Start Guide" -msgstr "" +msgstr "快速入门指南" #: ../../source/production/observability/vllm_endpoint.rst:14 msgid "1) On vLLM/LMCache side" -msgstr "" +msgstr "1) 在 vLLM/LMCache 端" #: ../../source/production/observability/vllm_endpoint.rst:16 msgid "" "In v1, vLLM and LMCache run in separate processes, so you have to use " "multi‑process Prometheus." -msgstr "" +msgstr "在 v1 中，vLLM 和 LMCache 运行在不同的进程中，因此您必须使用多进程 Prometheus。" #: ../../source/production/observability/vllm_endpoint.rst:18 msgid "" "The ``PROMETHEUS_MULTIPROC_DIR`` environment variable must be the same in" " both processes, as a IPC directory." -msgstr "" +msgstr "``PROMETHEUS_MULTIPROC_DIR`` 环境变量在两个进程中必须相同，作为 IPC 目录。" #: ../../source/production/observability/vllm_endpoint.rst:26 msgid "" "Once the HTTP server is running, you can access the LMCache metrics at " "the ``/metrics`` endpoint." -msgstr "" +msgstr "一旦 HTTP 服务器运行，您可以在 ``/metrics`` 端点访问 LMCache 指标。" #: ../../source/production/observability/vllm_endpoint.rst:35 msgid "" "And you will also find some ``.db`` files in the " "``$PROMETHEUS_MULTIPROC_DIR`` directory." -msgstr "" +msgstr "您还会在 ``$PROMETHEUS_MULTIPROC_DIR`` 目录中找到一些 ``.db`` 文件。" #: ../../source/production/observability/vllm_endpoint.rst:39 msgid "2) Prometheus Configuration" -msgstr "" +msgstr "2) Prometheus 配置" #: ../../source/production/observability/vllm_endpoint.rst:41 msgid "" "To scrape the LMCache metrics with a Prometheus server, add the following" " job to your ``prometheus.yml`` configuration, or equivalent " "configuration to scrape the metrics endpoint:" -msgstr "" +msgstr "要使用 Prometheus 服务器抓取 LMCache 指标，请将以下作业添加到您的 ``prometheus.yml`` 配置中，或等效配置以抓取指标端点：" #: ../../source/production/observability/vllm_endpoint.rst:53 msgid "Available Metrics" -msgstr "" +msgstr "可用指标" #: ../../source/production/observability/vllm_endpoint.rst:55 msgid "" "LMCache exposes a variety of metrics to monitor its performance. The " "following table lists all available metrics organized by category:" -msgstr "" +msgstr "LMCache 提供多种指标以监控其性能。下表列出了按类别组织的所有可用指标：" #: ../../source/production/observability/vllm_endpoint.rst:57 msgid "LMCache Metrics" -msgstr "" +msgstr "LMCache 指标" #: ../../source/production/observability/vllm_endpoint.rst:61 msgid "Metric Name" -msgstr "" +msgstr "指标名称" #: ../../source/production/observability/vllm_endpoint.rst:62 msgid "Type" -msgstr "" +msgstr "类型" #: ../../source/production/observability/vllm_endpoint.rst:63 msgid "Description" -msgstr "" +msgstr "描述" #: ../../source/production/observability/vllm_endpoint.rst:64 msgid "**Core Request Metrics**" -msgstr "" +msgstr "**核心请求指标**" #: ../../source/production/observability/vllm_endpoint.rst:67 msgid "``lmcache:num_retrieve_requests``" -msgstr "" +msgstr "``lmcache:num_retrieve_requests``" #: ../../source/production/observability/vllm_endpoint.rst:68 #: ../../source/production/observability/vllm_endpoint.rst:71 @@ -126,75 +126,75 @@ msgstr "" #: ../../source/production/observability/vllm_endpoint.rst:173 #: ../../source/production/observability/vllm_endpoint.rst:176 msgid "Counter" -msgstr "" +msgstr "计数器" #: ../../source/production/observability/vllm_endpoint.rst:69 msgid "Total number of retrieve requests" -msgstr "" +msgstr "检索请求的总数" #: ../../source/production/observability/vllm_endpoint.rst:70 msgid "``lmcache:num_store_requests``" -msgstr "" +msgstr "``lmcache:num_store_requests``" #: ../../source/production/observability/vllm_endpoint.rst:72 msgid "Total number of store requests" -msgstr "" +msgstr "存储请求的总数" #: ../../source/production/observability/vllm_endpoint.rst:73 msgid "``lmcache:num_lookup_requests``" -msgstr "" +msgstr "``lmcache:num_lookup_requests``" #: ../../source/production/observability/vllm_endpoint.rst:75 msgid "Total number of lookup requests" -msgstr "" +msgstr "查找请求的总数" #: ../../source/production/observability/vllm_endpoint.rst:76 msgid "``lmcache:num_requested_tokens``" -msgstr "" +msgstr "``lmcache:num_requested_tokens``" #: ../../source/production/observability/vllm_endpoint.rst:78 msgid "Total number of tokens requested for retrieval" -msgstr "" +msgstr "检索请求的总令牌数" #: ../../source/production/observability/vllm_endpoint.rst:79 msgid "``lmcache:num_hit_tokens``" -msgstr "" +msgstr "``lmcache:num_hit_tokens``" #: ../../source/production/observability/vllm_endpoint.rst:81 msgid "Total number of cache hit tokens from retrieval" -msgstr "" +msgstr "从检索中获得的缓存命中令牌的总数" #: ../../source/production/observability/vllm_endpoint.rst:82 msgid "``lmcache:num_lookup_tokens``" -msgstr "" +msgstr "``lmcache:num_lookup_tokens``" #: ../../source/production/observability/vllm_endpoint.rst:84 msgid "Total number of tokens requested in lookup operations" -msgstr "" +msgstr "查找操作中请求的令牌总数" #: ../../source/production/observability/vllm_endpoint.rst:85 msgid "``lmcache:num_lookup_hits``" -msgstr "" +msgstr "``lmcache:num_lookup_hits``" #: ../../source/production/observability/vllm_endpoint.rst:87 msgid "Total number of tokens hit in lookup operations" -msgstr "" +msgstr "查找操作中命中的总令牌数" #: ../../source/production/observability/vllm_endpoint.rst:88 msgid "``lmcache:num_vllm_hit_tokens``" -msgstr "" +msgstr "``lmcache:num_vllm_hit_tokens``" #: ../../source/production/observability/vllm_endpoint.rst:90 msgid "Number of hit tokens in vLLM" -msgstr "" +msgstr "vLLM 中的命中令牌数量" #: ../../source/production/observability/vllm_endpoint.rst:91 msgid "**Hit Rate Metrics**" -msgstr "" +msgstr "**命中率指标**" #: ../../source/production/observability/vllm_endpoint.rst:94 msgid "``lmcache:retrieve_hit_rate``" -msgstr "" +msgstr "``lmcache:retrieve_hit_rate``" #: ../../source/production/observability/vllm_endpoint.rst:95 #: ../../source/production/observability/vllm_endpoint.rst:98 @@ -208,55 +208,55 @@ msgstr "" #: ../../source/production/observability/vllm_endpoint.rst:188 #: ../../source/production/observability/vllm_endpoint.rst:191 msgid "Gauge" -msgstr "" +msgstr "仪表" #: ../../source/production/observability/vllm_endpoint.rst:96 msgid "The hit rate for retrieve requests" -msgstr "" +msgstr "检索请求的命中率" #: ../../source/production/observability/vllm_endpoint.rst:97 msgid "``lmcache:lookup_hit_rate``" -msgstr "" +msgstr "``lmcache:lookup_hit_rate``" #: ../../source/production/observability/vllm_endpoint.rst:99 msgid "The hit rate for lookup requests" -msgstr "" +msgstr "查找请求的命中率" #: ../../source/production/observability/vllm_endpoint.rst:100 msgid "**Cache Usage Metrics**" -msgstr "" +msgstr "**缓存使用指标**" #: ../../source/production/observability/vllm_endpoint.rst:103 msgid "``lmcache:local_cache_usage``" -msgstr "" +msgstr "``lmcache:local_cache_usage``" #: ../../source/production/observability/vllm_endpoint.rst:105 msgid "Local cache usage in bytes" -msgstr "" +msgstr "本地缓存使用量（字节）" #: ../../source/production/observability/vllm_endpoint.rst:106 msgid "``lmcache:remote_cache_usage``" -msgstr "" +msgstr "``lmcache:remote_cache_usage``" #: ../../source/production/observability/vllm_endpoint.rst:108 msgid "Remote cache usage in bytes" -msgstr "" +msgstr "远程缓存使用量（字节）" #: ../../source/production/observability/vllm_endpoint.rst:109 msgid "``lmcache:local_storage_usage``" -msgstr "" +msgstr "``lmcache:local_storage_usage``" #: ../../source/production/observability/vllm_endpoint.rst:111 msgid "Local storage usage in bytes" -msgstr "" +msgstr "本地存储使用量（字节）" #: ../../source/production/observability/vllm_endpoint.rst:112 msgid "**Performance Metrics**" -msgstr "" +msgstr "**性能指标**" #: ../../source/production/observability/vllm_endpoint.rst:115 msgid "``lmcache:time_to_retrieve``" -msgstr "" +msgstr "``lmcache:time_to_retrieve``" #: ../../source/production/observability/vllm_endpoint.rst:116 #: ../../source/production/observability/vllm_endpoint.rst:119 @@ -266,193 +266,193 @@ msgstr "" #: ../../source/production/observability/vllm_endpoint.rst:146 #: ../../source/production/observability/vllm_endpoint.rst:149 msgid "Histogram" -msgstr "" +msgstr "直方图" #: ../../source/production/observability/vllm_endpoint.rst:117 msgid "Time taken to retrieve from the cache (seconds)" -msgstr "" +msgstr "从缓存中检索所需的时间（秒）" #: ../../source/production/observability/vllm_endpoint.rst:118 msgid "``lmcache:time_to_store``" -msgstr "" +msgstr "``lmcache:time_to_store``" #: ../../source/production/observability/vllm_endpoint.rst:120 msgid "Time taken to store to the cache (seconds)" -msgstr "" +msgstr "存储到缓存所需的时间（秒）" #: ../../source/production/observability/vllm_endpoint.rst:121 msgid "``lmcache:retrieve_speed``" -msgstr "" +msgstr "``lmcache:retrieve_speed``" #: ../../source/production/observability/vllm_endpoint.rst:123 msgid "Retrieval speed (tokens per second)" -msgstr "" +msgstr "检索速度（每秒令牌数）" #: ../../source/production/observability/vllm_endpoint.rst:124 msgid "``lmcache:store_speed``" -msgstr "" +msgstr "``lmcache:store_speed``" #: ../../source/production/observability/vllm_endpoint.rst:126 msgid "Storage speed (tokens per second)" -msgstr "" +msgstr "存储速度（每秒令牌数）" #: ../../source/production/observability/vllm_endpoint.rst:127 msgid "**Remote Backend Metrics**" -msgstr "" +msgstr "**远程后端指标**" #: ../../source/production/observability/vllm_endpoint.rst:130 msgid "``lmcache:num_remote_read_requests``" -msgstr "" +msgstr "``lmcache:num_remote_read_requests``" #: ../../source/production/observability/vllm_endpoint.rst:132 msgid "Total number of read requests to remote backends" -msgstr "" +msgstr "对远程后端的读取请求总数" #: ../../source/production/observability/vllm_endpoint.rst:133 msgid "``lmcache:num_remote_read_bytes``" -msgstr "" +msgstr "``lmcache:num_remote_read_bytes``" #: ../../source/production/observability/vllm_endpoint.rst:135 msgid "Total number of bytes read from remote backends" -msgstr "" +msgstr "从远程后端读取的字节总数" #: ../../source/production/observability/vllm_endpoint.rst:136 msgid "``lmcache:num_remote_write_requests``" -msgstr "" +msgstr "``lmcache:num_remote_write_requests``" #: ../../source/production/observability/vllm_endpoint.rst:138 msgid "Total number of write requests to remote backends" -msgstr "" +msgstr "远程后端写请求的总数" #: ../../source/production/observability/vllm_endpoint.rst:139 msgid "``lmcache:num_remote_write_bytes``" -msgstr "" +msgstr "``lmcache:num_remote_write_bytes``" #: ../../source/production/observability/vllm_endpoint.rst:141 msgid "Total number of bytes written to remote backends" -msgstr "" +msgstr "写入远程后端的字节总数" #: ../../source/production/observability/vllm_endpoint.rst:142 msgid "``lmcache:remote_time_to_get``" -msgstr "" +msgstr "``lmcache:remote_time_to_get``" #: ../../source/production/observability/vllm_endpoint.rst:144 msgid "Time taken to get data from remote backends (milliseconds)" -msgstr "" +msgstr "从远程后端获取数据所花费的时间（毫秒）" #: ../../source/production/observability/vllm_endpoint.rst:145 msgid "``lmcache:remote_time_to_put``" -msgstr "" +msgstr "``lmcache:remote_time_to_put``" #: ../../source/production/observability/vllm_endpoint.rst:147 msgid "Time taken to put data to remote backends (milliseconds)" -msgstr "" +msgstr "将数据放入远程后端所需的时间（毫秒）" #: ../../source/production/observability/vllm_endpoint.rst:148 msgid "``lmcache:remote_time_to_get_sync``" -msgstr "" +msgstr "``lmcache:remote_time_to_get_sync``" #: ../../source/production/observability/vllm_endpoint.rst:150 msgid "Time taken to get data from remote backends synchronously (milliseconds)" -msgstr "" +msgstr "从远程后端同步获取数据所需的时间（毫秒）" #: ../../source/production/observability/vllm_endpoint.rst:151 msgid "**Network Monitoring Metrics**" -msgstr "" +msgstr "**网络监控指标**" #: ../../source/production/observability/vllm_endpoint.rst:154 msgid "``lmcache:remote_ping_latency``" -msgstr "" +msgstr "``lmcache:remote_ping_latency``" #: ../../source/production/observability/vllm_endpoint.rst:156 msgid "Latest ping latency to remote backends (milliseconds)" -msgstr "" +msgstr "最新的远程后端 ping 延迟（毫秒）" #: ../../source/production/observability/vllm_endpoint.rst:157 msgid "``lmcache:remote_ping_errors``" -msgstr "" +msgstr "``lmcache:remote_ping_errors``" #: ../../source/production/observability/vllm_endpoint.rst:159 msgid "Number of ping errors to remote backends" -msgstr "" +msgstr "远程后端的 ping 错误数量" #: ../../source/production/observability/vllm_endpoint.rst:160 msgid "``lmcache:remote_ping_successes``" -msgstr "" +msgstr "``lmcache:remote_ping_successes``" #: ../../source/production/observability/vllm_endpoint.rst:162 msgid "Number of ping successes to remote backends" -msgstr "" +msgstr "远程后端的 ping 成功次数" #: ../../source/production/observability/vllm_endpoint.rst:163 msgid "``lmcache:remote_ping_error_code``" -msgstr "" +msgstr "``lmcache:remote_ping_error_code``" #: ../../source/production/observability/vllm_endpoint.rst:165 msgid "Latest ping error code to remote backends" -msgstr "" +msgstr "最新的远程后端 ping 错误代码" #: ../../source/production/observability/vllm_endpoint.rst:166 msgid "**Local CPU Backend Metrics**" -msgstr "" +msgstr "**本地 CPU 后端指标**" #: ../../source/production/observability/vllm_endpoint.rst:169 msgid "``lmcache:local_cpu_evict_count``" -msgstr "" +msgstr "``lmcache:local_cpu_evict_count``" #: ../../source/production/observability/vllm_endpoint.rst:171 msgid "Total number of evictions in local CPU backend" -msgstr "" +msgstr "本地 CPU 后端的总逐出次数" #: ../../source/production/observability/vllm_endpoint.rst:172 msgid "``lmcache:local_cpu_evict_keys_count``" -msgstr "" +msgstr "``lmcache:local_cpu_evict_keys_count``" #: ../../source/production/observability/vllm_endpoint.rst:174 msgid "Total number of evicted keys in local CPU backend" -msgstr "" +msgstr "本地 CPU 后端中被逐出的键的总数" #: ../../source/production/observability/vllm_endpoint.rst:175 msgid "``lmcache:local_cpu_evict_failed_count``" -msgstr "" +msgstr "``lmcache:local_cpu_evict_failed_count``" #: ../../source/production/observability/vllm_endpoint.rst:177 msgid "Total number of failed evictions in local CPU backend" -msgstr "" +msgstr "本地 CPU 后端中失败的逐出总数" #: ../../source/production/observability/vllm_endpoint.rst:178 msgid "``lmcache:local_cpu_hot_cache_count``" -msgstr "" +msgstr "``lmcache:local_cpu_hot_cache_count``" #: ../../source/production/observability/vllm_endpoint.rst:180 msgid "The size of the hot cache" -msgstr "" +msgstr "热缓存的大小" #: ../../source/production/observability/vllm_endpoint.rst:181 msgid "``lmcache:local_cpu_keys_in_request_count``" -msgstr "" +msgstr "``lmcache:local_cpu_keys_in_request_count``" #: ../../source/production/observability/vllm_endpoint.rst:183 msgid "The size of the keys in request" -msgstr "" +msgstr "请求中键的大小" #: ../../source/production/observability/vllm_endpoint.rst:184 msgid "**Memory Management Metrics**" -msgstr "" +msgstr "**内存管理指标**" #: ../../source/production/observability/vllm_endpoint.rst:187 msgid "``lmcache:active_memory_objs_count``" -msgstr "" +msgstr "``lmcache:active_memory_objs_count``" #: ../../source/production/observability/vllm_endpoint.rst:189 msgid "The number of active memory objects" -msgstr "" +msgstr "活动内存对象的数量" #: ../../source/production/observability/vllm_endpoint.rst:190 msgid "``lmcache:pinned_memory_objs_count``" -msgstr "" +msgstr "``lmcache:pinned_memory_objs_count``" #: ../../source/production/observability/vllm_endpoint.rst:192 msgid "The number of pinned memory objects" -msgstr "" +msgstr "固定内存对象的数量" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/production/performance_tuning.po b/docs/source/locale/zh_CN/LC_MESSAGES/production/performance_tuning.po index d17648e5ada..f2386b067a3 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/production/performance_tuning.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/production/performance_tuning.po @@ -21,17 +21,17 @@ msgstr "" #: ../../source/production/performance_tuning.rst:4 msgid "Performance Tuning" -msgstr "" +msgstr "性能调优" #: ../../source/production/performance_tuning.rst:6 msgid "" "This guide covers key LMCache configuration options that can help you " "optimize performance in production deployments." -msgstr "" +msgstr "本指南涵盖了关键的 LMCache 配置选项，这些选项可以帮助您优化生产环境中的性能。" #: ../../source/production/performance_tuning.rst:10 msgid "Minimum Retrieve Tokens" -msgstr "" +msgstr "最小检索令牌" #: ../../source/production/performance_tuning.rst:12 msgid "" @@ -39,91 +39,91 @@ msgid "" "into GPU memory to avoid recomputation. However, if only a small number " "of tokens are hit, the overhead of loading them from the cache may " "outweigh the benefit of skipping recomputation." -msgstr "" +msgstr "当 LMCache 找到部分 KV Cache 命中时，它会将缓存的令牌加载到显存中，以避免重计算。然而，如果命中的令牌数量很少，从缓存中加载它们的开销可能会超过跳过重计算的好处。" #: ../../source/production/performance_tuning.rst:17 msgid "" "The ``min_retrieve_tokens`` setting lets you set a threshold: if the " "number of tokens that need to be loaded is below this value, LMCache will" " skip the retrieve and let the inference engine recompute them instead." -msgstr "" +msgstr "``min_retrieve_tokens`` 设置允许您设置一个阈值：如果需要加载的令牌数量低于此值，LMCache 将跳过检索，并让推理引擎重新计算它们。" #: ../../source/production/performance_tuning.rst:24 msgid "" "Even when retrieve is skipped, LMCache still records the hit tokens " "internally so that it does **not** re-store chunks that already exist in " "the cache." -msgstr "" +msgstr "即使跳过了查找，LMCache 仍然在内部记录命中的令牌，以便它**不**会重新存储已经存在于缓存中的块。" #: ../../source/production/performance_tuning.rst:29 msgid "When to Use" -msgstr "" +msgstr "何时使用" #: ../../source/production/performance_tuning.rst:31 msgid "Consider setting ``min_retrieve_tokens`` when:" -msgstr "" +msgstr "考虑在以下情况下设置 ``min_retrieve_tokens``：" #: ../../source/production/performance_tuning.rst:33 msgid "" "For the backend you are using, the transfer latency is noticeable for " "small payloads." -msgstr "" +msgstr "对于您使用的后端，小负载的传输延迟是显著的。" #: ../../source/production/performance_tuning.rst:35 msgid "" "Your workload has many requests with **low cache hit ratios**, where " "recomputation is faster than cache loading." -msgstr "" +msgstr "您的工作负载有许多请求，**缓存命中率低**，在这种情况下，重计算比缓存加载更快。" #: ../../source/production/performance_tuning.rst:37 msgid "You want to reduce unnecessary I/O for marginal cache hits." -msgstr "" +msgstr "您希望减少边际缓存命中的不必要 I/O。" #: ../../source/production/performance_tuning.rst:39 msgid "You can increase or decrease based on your latency observations." -msgstr "" +msgstr "您可以根据您的延迟观察来增加或减少。" #: ../../source/production/performance_tuning.rst:42 msgid "Configuration" -msgstr "" +msgstr "配置" #: ../../source/production/performance_tuning.rst:44 msgid "**YAML configuration file:**" -msgstr "" +msgstr "**YAML 配置文件：**" #: ../../source/production/performance_tuning.rst:52 msgid "**Environment variable:**" -msgstr "" +msgstr "**环境变量：**" #: ../../source/production/performance_tuning.rst:59 msgid "Working Example" -msgstr "" +msgstr "工作示例" #: ../../source/production/performance_tuning.rst:61 msgid "Create an LMCache configuration file ``lmcache_config.yaml``:" -msgstr "" +msgstr "创建一个 LMCache 配置文件 ``lmcache_config.yaml``：" #: ../../source/production/performance_tuning.rst:70 msgid "Start vLLM with LMCache:" -msgstr "" +msgstr "使用 LMCache 启动 vLLM：" #: ../../source/production/performance_tuning.rst:81 msgid "Send a request to populate the cache:" -msgstr "" +msgstr "发送请求以填充缓存：" #: ../../source/production/performance_tuning.rst:94 msgid "Send a similar request that partially reuses the cached prefix:" -msgstr "" +msgstr "发送一个类似的请求，部分重用缓存的前缀：" #: ../../source/production/performance_tuning.rst:107 msgid "" "Check the server logs. If the number of loadable hit tokens is below " "1024, you will see a log message like:" -msgstr "" +msgstr "检查服务器日志。如果可加载的命中令牌数量低于 1024，您将看到类似于以下的日志消息：" #: ../../source/production/performance_tuning.rst:115 msgid "" "This confirms that the small cache hit was skipped in favor of " "recomputation, avoiding unnecessary transfer overhead." -msgstr "" +msgstr "这确认了小缓存命中被跳过，以便进行重计算，从而避免不必要的传输开销。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/devstral.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/devstral.po index 5bcaa29aa84..6fedfde74c7 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/devstral.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/devstral.po @@ -21,104 +21,104 @@ msgstr "" #: ../../source/recipes/devstral.rst:4 msgid "MistralForCausalLM" -msgstr "" +msgstr "MistralForCausalLM" #: ../../source/recipes/devstral.rst:7 msgid "Validated models" -msgstr "" +msgstr "验证过的模型" #: ../../source/recipes/devstral.rst:9 msgid "" "`mistralai/Devstral-2-123B-Instruct-2512 " "`_" -msgstr "" +msgstr "`mistralai/Devstral-2-123B-Instruct-2512 `_" #: ../../source/recipes/devstral.rst msgid "vLLM" -msgstr "" +msgstr "vLLM" #: ../../source/recipes/devstral.rst:16 msgid "" "**Engine documentation:** `Mistral / Devstral in vLLM supported models " "`_ (architecture ``MistralForCausalLM``)." -msgstr "" +msgstr "**引擎文档：** `Mistral / Devstral 在 vLLM 支持的模型 `_ （架构 ``MistralForCausalLM``）。" #: ../../source/recipes/devstral.rst:21 msgid "**Status:** Validated with LMCache." -msgstr "" +msgstr "**状态:** 已通过 LMCache 验证。" #: ../../source/recipes/devstral.rst:23 msgid "Start the LMCache MP server:" -msgstr "" +msgstr "启动 LMCache MP 服务器：" #: ../../source/recipes/devstral.rst:31 msgid "Start vLLM with the LMCache MP connector:" -msgstr "" +msgstr "启动带有 LMCache MP 连接器的 vLLM：" #: ../../source/recipes/devstral.rst:44 msgid "" "Adjust ``--tensor-parallel-size`` to match your hardware. For the generic" " LMCache + vLLM wiring (ports, remote hosts, in-process mode), see " ":doc:`../mp/quickstart`." -msgstr "" +msgstr "调整 ``--tensor-parallel-size`` 以匹配您的硬件。有关通用 LMCache + vLLM 连接（端口、远程主机、进程内模式），请参见 :doc:`../mp/quickstart`。" #: ../../source/recipes/devstral.rst:48 msgid "" "If there are any issues with vLLM setup, please refer to the `vLLM " "Recipes `_ " "for more details." -msgstr "" +msgstr "如果在 vLLM 设置中遇到任何问题，请参考 `vLLM Recipes `_ 获取更多详细信息。" #: ../../source/recipes/devstral.rst msgid "SGLang" -msgstr "" +msgstr "SGLang" #: ../../source/recipes/devstral.rst:54 msgid "**Status:** Not validated with LMCache." -msgstr "" +msgstr "**状态：** 未通过 LMCache 验证。" #: ../../source/recipes/devstral.rst msgid "TRT-LLM" -msgstr "" +msgstr "TRT-LLM" #: ../../source/recipes/devstral.rst:58 msgid "**Status:** Not supported. LMCache TRT-LLM integration is in progress." -msgstr "" +msgstr "**状态：** 不支持。LMCache TRT-LLM 集成正在进行中。" #: ../../source/recipes/devstral.rst:61 msgid "CacheBlend support" -msgstr "" +msgstr "CacheBlend 支持" #: ../../source/recipes/devstral.rst:64 msgid "Compression support" -msgstr "" +msgstr "压缩支持" #: ../../source/recipes/devstral.rst:70 msgid "Method" -msgstr "" +msgstr "方法" #: ../../source/recipes/devstral.rst:71 msgid "Status" -msgstr "" +msgstr "状态" #: ../../source/recipes/devstral.rst:72 msgid "Notes" -msgstr "" +msgstr "备注" #: ../../source/recipes/devstral.rst:73 msgid ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" -msgstr "" +msgstr ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" #: ../../source/recipes/devstral.rst:74 msgid "Not validated" -msgstr "" +msgstr "未验证" #: ../../source/recipes/devstral.rst:78 msgid "Caveats" -msgstr "" +msgstr "注意事项" #: ../../source/recipes/devstral.rst:80 msgid "None known." -msgstr "" +msgstr "没有已知的问题。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gemma4.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gemma4.po index 6a78548a101..e7435c648b0 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gemma4.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gemma4.po @@ -21,102 +21,102 @@ msgstr "" #: ../../source/recipes/gemma4.rst:4 msgid "Gemma4ForConditionalGeneration" -msgstr "" +msgstr "Gemma4ForConditionalGeneration" #: ../../source/recipes/gemma4.rst:7 msgid "Validated models" -msgstr "" +msgstr "验证过的模型" #: ../../source/recipes/gemma4.rst:9 msgid "`google/gemma-4-31B-it `_" -msgstr "" +msgstr "`google/gemma-4-31B-it `_" #: ../../source/recipes/gemma4.rst msgid "vLLM" -msgstr "" +msgstr "vLLM" #: ../../source/recipes/gemma4.rst:16 msgid "" "**Engine documentation:** `Gemma 4 in vLLM supported models " "`_ (architecture ``Gemma4ForConditionalGeneration``)." -msgstr "" +msgstr "**引擎文档:** `Gemma 4 在 vLLM 支持的模型 `_ (架构 ``Gemma4ForConditionalGeneration``)。" #: ../../source/recipes/gemma4.rst:21 msgid "**Status:** Validated with LMCache." -msgstr "" +msgstr "**状态:** 已通过 LMCache 验证。" #: ../../source/recipes/gemma4.rst:23 msgid "Start the LMCache MP server:" -msgstr "" +msgstr "启动 LMCache MP 服务器：" #: ../../source/recipes/gemma4.rst:31 msgid "Start vLLM with the LMCache MP connector:" -msgstr "" +msgstr "启动 vLLM 与 LMCache MP 连接器：" #: ../../source/recipes/gemma4.rst:42 msgid "" "Adjust ``--tensor-parallel-size`` to match your hardware. For the generic" " LMCache + vLLM wiring (ports, remote hosts, in-process mode), see " ":doc:`../mp/quickstart`." -msgstr "" +msgstr "调整 ``--tensor-parallel-size`` 以匹配您的硬件。有关通用 LMCache + vLLM 连接（端口、远程主机、进程内模式），请参见 :doc:`../mp/quickstart`。" #: ../../source/recipes/gemma4.rst:46 msgid "" "If there are any issues with vLLM setup, please refer to the `vLLM " "Recipes `_ " "for more details." -msgstr "" +msgstr "如果在 vLLM 设置中遇到任何问题，请参考 `vLLM Recipes `_ 以获取更多详细信息。" #: ../../source/recipes/gemma4.rst msgid "SGLang" -msgstr "" +msgstr "SGLang" #: ../../source/recipes/gemma4.rst:52 msgid "**Status:** Not validated with LMCache." -msgstr "" +msgstr "**状态：** 未通过 LMCache 验证。" #: ../../source/recipes/gemma4.rst msgid "TRT-LLM" -msgstr "" +msgstr "TRT-LLM" #: ../../source/recipes/gemma4.rst:56 msgid "**Status:** Not supported. LMCache TRT-LLM integration is in progress." -msgstr "" +msgstr "**状态：** 不支持。LMCache TRT-LLM 集成正在进行中。" #: ../../source/recipes/gemma4.rst:59 msgid "CacheBlend support" -msgstr "" +msgstr "CacheBlend 支持" #: ../../source/recipes/gemma4.rst:62 msgid "Compression support" -msgstr "" +msgstr "压缩支持" #: ../../source/recipes/gemma4.rst:68 msgid "Method" -msgstr "" +msgstr "方法" #: ../../source/recipes/gemma4.rst:69 msgid "Status" -msgstr "" +msgstr "状态" #: ../../source/recipes/gemma4.rst:70 msgid "Notes" -msgstr "" +msgstr "笔记" #: ../../source/recipes/gemma4.rst:71 msgid ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" -msgstr "" +msgstr ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" #: ../../source/recipes/gemma4.rst:72 msgid "Not validated" -msgstr "" +msgstr "未验证" #: ../../source/recipes/gemma4.rst:76 msgid "Caveats" -msgstr "" +msgstr "注意事项" #: ../../source/recipes/gemma4.rst:78 msgid "None known." -msgstr "" +msgstr "没有已知的问题。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gpt_oss.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gpt_oss.po index 7a9a0087a84..c690dd2fdb0 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gpt_oss.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/gpt_oss.po @@ -21,110 +21,110 @@ msgstr "" #: ../../source/recipes/gpt_oss.rst:4 msgid "GptOssForCausalLM" -msgstr "" +msgstr "GptOssForCausalLM" #: ../../source/recipes/gpt_oss.rst:7 msgid "Validated models" -msgstr "" +msgstr "验证模型" #: ../../source/recipes/gpt_oss.rst:9 msgid "`openai/gpt-oss-120b `_" -msgstr "" +msgstr "`openai/gpt-oss-120b `_" #: ../../source/recipes/gpt_oss.rst:10 msgid "`openai/gpt-oss-20b `_" -msgstr "" +msgstr "`openai/gpt-oss-20b `_" #: ../../source/recipes/gpt_oss.rst msgid "vLLM" -msgstr "" +msgstr "vLLM" #: ../../source/recipes/gpt_oss.rst:17 msgid "" "**Engine documentation:** `GPT-OSS in vLLM supported models " "`_ (architecture ``GptOssForCausalLM``)." -msgstr "" +msgstr "**引擎文档：** `vLLM 支持的模型中的 GPT-OSS `_ （架构 ``GptOssForCausalLM``）。" #: ../../source/recipes/gpt_oss.rst:22 msgid "**Status:** Validated with LMCache." -msgstr "" +msgstr "**状态:** 已通过 LMCache 验证。" #: ../../source/recipes/gpt_oss.rst:24 msgid "Start the LMCache MP server:" -msgstr "" +msgstr "启动 LMCache MP 服务器：" #: ../../source/recipes/gpt_oss.rst:32 msgid "**gpt-oss-120b** (2 GPUs):" -msgstr "" +msgstr "**gpt-oss-120b** (2 GPUs):" #: ../../source/recipes/gpt_oss.rst:45 msgid "**gpt-oss-20b** (1 GPU):" -msgstr "" +msgstr "**gpt-oss-20b** (1 GPU):" #: ../../source/recipes/gpt_oss.rst:57 msgid "" "Adjust ``--tensor-parallel-size`` to match your hardware. For the generic" " LMCache + vLLM wiring (ports, remote hosts, in-process mode), see " ":doc:`../mp/quickstart`." -msgstr "" +msgstr "调整 ``--tensor-parallel-size`` 以匹配您的硬件。有关通用 LMCache + vLLM 连接（端口、远程主机、进程内模式），请参见 :doc:`../mp/quickstart`。" #: ../../source/recipes/gpt_oss.rst:61 msgid "" "If there are any issues with vLLM setup, please refer to the `vLLM " "Recipes `_ " "for more details." -msgstr "" +msgstr "如果在 vLLM 设置中遇到任何问题，请参考 `vLLM Recipes `_ 以获取更多详细信息。" #: ../../source/recipes/gpt_oss.rst msgid "SGLang" -msgstr "" +msgstr "SGLang" #: ../../source/recipes/gpt_oss.rst:67 msgid "**Status:** Not validated with LMCache." -msgstr "" +msgstr "**状态：** 未通过 LMCache 验证。" #: ../../source/recipes/gpt_oss.rst msgid "TRT-LLM" -msgstr "" +msgstr "TRT-LLM" #: ../../source/recipes/gpt_oss.rst:71 msgid "**Status:** Not supported. LMCache TRT-LLM integration is in progress." -msgstr "" +msgstr "**状态：** 不支持。LMCache TRT-LLM 集成正在进行中。" #: ../../source/recipes/gpt_oss.rst:74 msgid "CacheBlend support" -msgstr "" +msgstr "CacheBlend 支持" #: ../../source/recipes/gpt_oss.rst:77 msgid "Compression support" -msgstr "" +msgstr "压缩支持" #: ../../source/recipes/gpt_oss.rst:83 msgid "Method" -msgstr "" +msgstr "方法" #: ../../source/recipes/gpt_oss.rst:84 msgid "Status" -msgstr "" +msgstr "状态" #: ../../source/recipes/gpt_oss.rst:85 msgid "Notes" -msgstr "" +msgstr "笔记" #: ../../source/recipes/gpt_oss.rst:86 msgid ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" -msgstr "" +msgstr ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" #: ../../source/recipes/gpt_oss.rst:87 msgid "Not validated" -msgstr "" +msgstr "未验证" #: ../../source/recipes/gpt_oss.rst:91 msgid "Caveats" -msgstr "" +msgstr "注意事项" #: ../../source/recipes/gpt_oss.rst:93 msgid "None known." -msgstr "" +msgstr "没有已知的问题。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/index.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/index.po index 92880121cef..318ecd5bed2 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/index.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/index.po @@ -21,32 +21,32 @@ msgstr "" #: ../../source/recipes/index.rst:4 msgid "Recipes" -msgstr "" +msgstr "食谱" #: ../../source/recipes/index.rst:6 msgid "" "This section lists model architectures that have been validated end-to-" "end with LMCache, with a recipe page per architecture covering only the " "LMCache-specific configuration that diverges from defaults." -msgstr "" +msgstr "本节列出了经过 LMCache 端到端验证的模型架构，每个架构都有一个配方页面，涵盖与默认设置不同的仅 LMCache 特定配置。" #: ../../source/recipes/index.rst:10 msgid "" "Engine-side documentation (how to serve the model itself) lives with the " "serving engine. Recipe pages link out rather than duplicate." -msgstr "" +msgstr "引擎端文档（如何服务模型本身）与服务引擎一起存在。食谱页面链接而不是重复。" #: ../../source/recipes/index.rst:14 msgid "Recipe page contents" -msgstr "" +msgstr "食谱页面内容" #: ../../source/recipes/index.rst:16 msgid "Each recipe page is intentionally minimal:" -msgstr "" +msgstr "每个食谱页面都故意保持简约：" #: ../../source/recipes/index.rst:18 msgid "**Validated models** -- exact HF repo IDs that have been tested." -msgstr "" +msgstr "**验证过的模型** -- 已测试的确切 HF 仓库 ID。" #: ../../source/recipes/index.rst:19 msgid "" @@ -54,21 +54,21 @@ msgid "" "Each tab links to the engine's own documentation for the model and shows " "the exact ``lmcache server`` and engine launch commands. Tabs for engines" " that are not yet validated state so explicitly." -msgstr "" +msgstr "**引擎选项卡** -- 每个服务引擎 (vLLM, SGLang, TRT-LLM) 一个选项卡。每个选项卡链接到该引擎的模型文档，并显示确切的 ``lmcache server`` 和引擎启动命令。尚未验证的引擎的选项卡会明确说明。" #: ../../source/recipes/index.rst:23 msgid "**CacheBlend support** -- validation status (may be empty)." -msgstr "" +msgstr "**CacheBlend 支持** -- 验证状态（可能为空）。" #: ../../source/recipes/index.rst:24 msgid "" "**Compression support** -- table of compression methods (CacheGen, etc.) " "with per-method validation status. Extensible: new methods get a row." -msgstr "" +msgstr "**压缩支持** -- 压缩方法（CacheGen 等）的表格，包含每种方法的验证状态。可扩展：新方法将获得一行。" #: ../../source/recipes/index.rst:26 msgid "**Caveats** -- known limitations, if any." -msgstr "" +msgstr "**注意事项** -- 已知的限制（如果有的话）。" #: ../../source/recipes/index.rst:28 msgid "" @@ -76,49 +76,49 @@ msgid "" "mode, sending a first request), see :doc:`../getting_started/quickstart` " "and :doc:`../mp/quickstart`. Recipes assume those pages as a " "prerequisite." -msgstr "" +msgstr "有关通用 LMCache + 引擎连接（端口、远程主机、进程内模式、发送第一个请求），请参阅 :doc:`../getting_started/quickstart` 和 :doc:`../mp/quickstart`。食谱假设这些页面是先决条件。" #: ../../source/recipes/index.rst:33 msgid "Supported architectures" -msgstr "" +msgstr "支持的架构" #: ../../source/recipes/index.rst:39 msgid "Architecture" -msgstr "" +msgstr "架构" #: ../../source/recipes/index.rst:40 msgid "Example HF model" -msgstr "" +msgstr "示例 HF 模型" #: ../../source/recipes/index.rst:41 msgid "vLLM" -msgstr "" +msgstr "vLLM" #: ../../source/recipes/index.rst:42 msgid "SGLang" -msgstr "" +msgstr "SGLang" #: ../../source/recipes/index.rst:43 msgid "TRT-LLM" -msgstr "" +msgstr "TRT-LLM" #: ../../source/recipes/index.rst:44 msgid "Recipe" -msgstr "" +msgstr "食谱" #: ../../source/recipes/index.rst:45 msgid "``MiniMaxM2ForCausalLM``" -msgstr "" +msgstr "``MiniMaxM2ForCausalLM``" #: ../../source/recipes/index.rst:46 msgid "``MiniMaxAI/MiniMax-M2``" -msgstr "" +msgstr "``MiniMaxAI/MiniMax-M2``" #: ../../source/recipes/index.rst:47 ../../source/recipes/index.rst:53 #: ../../source/recipes/index.rst:59 ../../source/recipes/index.rst:65 #: ../../source/recipes/index.rst:71 msgid "✓" -msgstr "" +msgstr "✓" #: ../../source/recipes/index.rst:48 ../../source/recipes/index.rst:49 #: ../../source/recipes/index.rst:54 ../../source/recipes/index.rst:55 @@ -126,86 +126,86 @@ msgstr "" #: ../../source/recipes/index.rst:66 ../../source/recipes/index.rst:67 #: ../../source/recipes/index.rst:72 ../../source/recipes/index.rst:73 msgid "—" -msgstr "" +msgstr "—" #: ../../source/recipes/index.rst:50 msgid ":doc:`minimax_m2`" -msgstr "" +msgstr ":doc:`minimax_m2`" #: ../../source/recipes/index.rst:51 msgid "``Gemma4ForConditionalGeneration``" -msgstr "" +msgstr "``Gemma4ForConditionalGeneration``" #: ../../source/recipes/index.rst:52 msgid "``google/gemma-4-31B-it``" -msgstr "" +msgstr "``google/gemma-4-31B-it``" #: ../../source/recipes/index.rst:56 msgid ":doc:`gemma4`" -msgstr "" +msgstr ":doc:`gemma4`" #: ../../source/recipes/index.rst:57 msgid "``MistralForCausalLM``" -msgstr "" +msgstr "``MistralForCausalLM``" #: ../../source/recipes/index.rst:58 msgid "``mistralai/Devstral-2-123B-Instruct-2512``" -msgstr "" +msgstr "``mistralai/Devstral-2-123B-Instruct-2512``" #: ../../source/recipes/index.rst:62 msgid ":doc:`devstral`" -msgstr "" +msgstr ":doc:`devstral`" #: ../../source/recipes/index.rst:63 msgid "``GptOssForCausalLM``" -msgstr "" +msgstr "``GptOssForCausalLM``" #: ../../source/recipes/index.rst:64 msgid "``openai/gpt-oss-120b``" -msgstr "" +msgstr "``openai/gpt-oss-120b``" #: ../../source/recipes/index.rst:68 msgid ":doc:`gpt_oss`" -msgstr "" +msgstr ":gpt_oss:" #: ../../source/recipes/index.rst:69 msgid "``Qwen3MoeForCausalLM``" -msgstr "" +msgstr "``Qwen3MoeForCausalLM``" #: ../../source/recipes/index.rst:70 msgid "``Qwen/Qwen3-235B-A22B``" -msgstr "" +msgstr "``Qwen/Qwen3-235B-A22B``" #: ../../source/recipes/index.rst:74 msgid ":doc:`qwen3`" -msgstr "" +msgstr "：doc:`qwen3`" #: ../../source/recipes/index.rst:76 msgid "Legend: ``✓`` validated, ``—`` not validated." -msgstr "" +msgstr "图例：``✓`` 已验证，``—`` 未验证。" #: ../../source/recipes/index.rst:79 msgid "Contributing a recipe" -msgstr "" +msgstr "贡献一个食谱" #: ../../source/recipes/index.rst:81 msgid "To add a new architecture:" -msgstr "" +msgstr "要添加一个新架构：" #: ../../source/recipes/index.rst:83 msgid "" "Copy an existing page (e.g. ``minimax_m2.rst``) to " "``recipes/.rst``." -msgstr "" +msgstr "将现有页面（例如 ``minimax_m2.rst``）复制到 ``recipes/.rst``。" #: ../../source/recipes/index.rst:85 msgid "" "Fill in **Validated models**, **Engines**, **LMCache configuration**, and" " **Caveats**. Keep each section terse -- if a field has nothing to say, " "say so in one line rather than padding it." -msgstr "" +msgstr "填写 **已验证模型**、**引擎**、**LMCache 配置** 和 **注意事项**。保持每个部分简洁 -- 如果某个字段没有内容，请用一行说明，而不是填充内容。" #: ../../source/recipes/index.rst:88 msgid "Add a row to the table above and an entry to the hidden toctree below." -msgstr "" +msgstr "在上面的表格中添加一行，并在下面的隐藏 toctree 中添加一个条目。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/minimax_m2.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/minimax_m2.po index ff066a0b428..5d7ad47c9d3 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/minimax_m2.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/minimax_m2.po @@ -21,76 +21,76 @@ msgstr "" #: ../../source/recipes/minimax_m2.rst:4 msgid "MiniMaxM2ForCausalLM" -msgstr "" +msgstr "MiniMaxM2ForCausalLM" #: ../../source/recipes/minimax_m2.rst:7 msgid "Validated models" -msgstr "" +msgstr "验证过的模型" #: ../../source/recipes/minimax_m2.rst:9 msgid "`MiniMaxAI/MiniMax-M2 `_" -msgstr "" +msgstr "`MiniMaxAI/MiniMax-M2 `_" #: ../../source/recipes/minimax_m2.rst:10 msgid "`MiniMaxAI/MiniMax-M2.5 `_" -msgstr "" +msgstr "`MiniMaxAI/MiniMax-M2.5 `_" #: ../../source/recipes/minimax_m2.rst:11 msgid "`MiniMaxAI/MiniMax-M2.7 `_" -msgstr "" +msgstr "`MiniMaxAI/MiniMax-M2.7 `_" #: ../../source/recipes/minimax_m2.rst msgid "vLLM" -msgstr "" +msgstr "vLLM" #: ../../source/recipes/minimax_m2.rst:18 msgid "" "**Engine documentation:** `MiniMax-M2 in vLLM supported models " "`_ (architecture ``MiniMaxM2ForCausalLM``)." -msgstr "" +msgstr "**引擎文档：** `vLLM 支持的模型中的 MiniMax-M2 `_ （架构 ``MiniMaxM2ForCausalLM``）。" #: ../../source/recipes/minimax_m2.rst:23 msgid "**Status:** Validated with LMCache." -msgstr "" +msgstr "**状态:** 已通过 LMCache 验证。" #: ../../source/recipes/minimax_m2.rst:25 msgid "Start the LMCache MP server:" -msgstr "" +msgstr "启动 LMCache MP 服务器：" #: ../../source/recipes/minimax_m2.rst:33 msgid "Start vLLM with the LMCache MP connector:" -msgstr "" +msgstr "启动 vLLM 与 LMCache MP 连接器：" #: ../../source/recipes/minimax_m2.rst:35 msgid "**MiniMax-M2** (8 GPUs):" -msgstr "" +msgstr "**MiniMax-M2** (8 GPUs):" #: ../../source/recipes/minimax_m2.rst:47 msgid "**MiniMax-M2.5** (4 GPUs):" -msgstr "" +msgstr "**MiniMax-M2.5** (4 GPUs):" #: ../../source/recipes/minimax_m2.rst:62 msgid "**MiniMax-M2.7** (4 GPUs):" -msgstr "" +msgstr "**MiniMax-M2.7** (4 GPUs):" #: ../../source/recipes/minimax_m2.rst:77 msgid "" "Adjust ``--tensor-parallel-size`` to match your hardware. For the generic" " LMCache + vLLM wiring (ports, remote hosts, in-process mode), see " ":doc:`../mp/quickstart`." -msgstr "" +msgstr "调整 ``--tensor-parallel-size`` 以匹配您的硬件。有关通用 LMCache + vLLM 连接（端口、远程主机、进程内模式），请参见 :doc:`../mp/quickstart`。" #: ../../source/recipes/minimax_m2.rst:81 msgid "" "If there are any issues with vLLM setup, please refer to the `vLLM " "Recipes `_ " "for more details." -msgstr "" +msgstr "如果在 vLLM 设置中遇到任何问题，请参考 `vLLM Recipes `_ 以获取更多详细信息。" #: ../../source/recipes/minimax_m2.rst msgid "SGLang" -msgstr "" +msgstr "SGLang" #: ../../source/recipes/minimax_m2.rst:87 msgid "" @@ -98,53 +98,53 @@ msgid "" "`_, " "`MiniMax M2.5/M2.1/M2 usage guide " "`_." -msgstr "" +msgstr "**引擎文档：** `MiniMax-M2 SGLang 食谱 `_，`MiniMax M2.5/M2.1/M2 使用指南 `_。" #: ../../source/recipes/minimax_m2.rst:93 msgid "**Status:** Not validated with LMCache." -msgstr "" +msgstr "**状态:** 未通过 LMCache 验证。" #: ../../source/recipes/minimax_m2.rst msgid "TRT-LLM" -msgstr "" +msgstr "TRT-LLM" #: ../../source/recipes/minimax_m2.rst:97 msgid "**Status:** Not supported. LMCache TRT-LLM integration is in progress." -msgstr "" +msgstr "**状态：** 不支持。LMCache TRT-LLM 集成正在进行中。" #: ../../source/recipes/minimax_m2.rst:100 msgid "CacheBlend support" -msgstr "" +msgstr "CacheBlend 支持" #: ../../source/recipes/minimax_m2.rst:103 msgid "Compression support" -msgstr "" +msgstr "压缩支持" #: ../../source/recipes/minimax_m2.rst:109 msgid "Method" -msgstr "" +msgstr "方法" #: ../../source/recipes/minimax_m2.rst:110 msgid "Status" -msgstr "" +msgstr "状态" #: ../../source/recipes/minimax_m2.rst:111 msgid "Notes" -msgstr "" +msgstr "笔记" #: ../../source/recipes/minimax_m2.rst:112 msgid ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" -msgstr "" +msgstr ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" #: ../../source/recipes/minimax_m2.rst:113 msgid "Not validated" -msgstr "" +msgstr "未验证" #: ../../source/recipes/minimax_m2.rst:117 msgid "Caveats" -msgstr "" +msgstr "注意事项" #: ../../source/recipes/minimax_m2.rst:119 msgid "None known." -msgstr "" +msgstr "没有已知的问题。" diff --git a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/qwen3.po b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/qwen3.po index d6badb4ff7c..80260700639 100644 --- a/docs/source/locale/zh_CN/LC_MESSAGES/recipes/qwen3.po +++ b/docs/source/locale/zh_CN/LC_MESSAGES/recipes/qwen3.po @@ -21,130 +21,130 @@ msgstr "" #: ../../source/recipes/qwen3.rst:4 msgid "Qwen3MoeForCausalLM" -msgstr "" +msgstr "Qwen3MoeForCausalLM" #: ../../source/recipes/qwen3.rst:7 msgid "Validated models" -msgstr "" +msgstr "验证过的模型" #: ../../source/recipes/qwen3.rst:9 msgid "`Qwen/Qwen3-235B-A22B `_" -msgstr "" +msgstr "`Qwen/Qwen3-235B-A22B `_" #: ../../source/recipes/qwen3.rst:10 msgid "`Qwen/Qwen3-30B-A3B `_" -msgstr "" +msgstr "`Qwen/Qwen3-30B-A3B `_" #: ../../source/recipes/qwen3.rst:11 msgid "" "`Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8 " "`_" -msgstr "" +msgstr "`Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8 `_" #: ../../source/recipes/qwen3.rst:12 msgid "" "`Qwen/Qwen3-Coder-30B-A3B-Instruct `_" -msgstr "" +msgstr "`Qwen/Qwen3-Coder-30B-A3B-Instruct `_" #: ../../source/recipes/qwen3.rst msgid "vLLM" -msgstr "" +msgstr "vLLM" #: ../../source/recipes/qwen3.rst:19 msgid "" "**Engine documentation:** `Qwen3 MoE in vLLM supported models " "`_ (architecture ``Qwen3MoeForCausalLM``)." -msgstr "" +msgstr "**引擎文档:** `vLLM 支持的 Qwen3 MoE 模型 `_ (架构 ``Qwen3MoeForCausalLM``)。" #: ../../source/recipes/qwen3.rst:24 msgid "**Status:** Validated with LMCache." -msgstr "" +msgstr "**状态：** 已通过 LMCache 验证。" #: ../../source/recipes/qwen3.rst:26 msgid "Start the LMCache MP server:" -msgstr "" +msgstr "启动 LMCache MP 服务器：" #: ../../source/recipes/qwen3.rst:34 msgid "**Qwen3-235B-A22B** (4 GPUs, expert parallel):" -msgstr "" +msgstr "**Qwen3-235B-A22B** (4 GPUs, 专家并行):" #: ../../source/recipes/qwen3.rst:49 msgid "**Qwen3-30B-A3B** (1 GPU):" -msgstr "" +msgstr "**Qwen3-30B-A3B** (1 GPU):" #: ../../source/recipes/qwen3.rst:62 msgid "**Qwen3-Coder-480B-A35B-Instruct-FP8** (8 GPUs, expert parallel):" -msgstr "" +msgstr "**Qwen3-Coder-480B-A35B-Instruct-FP8** (8 个 GPU，专家并行)：" #: ../../source/recipes/qwen3.rst:76 msgid "**Qwen3-Coder-30B-A3B-Instruct** (1 GPU):" -msgstr "" +msgstr "**Qwen3-Coder-30B-A3B-Instruct** (1 GPU):" #: ../../source/recipes/qwen3.rst:88 msgid "" "Adjust ``--tensor-parallel-size`` to match your hardware. For the generic" " LMCache + vLLM wiring (ports, remote hosts, in-process mode), see " ":doc:`../mp/quickstart`." -msgstr "" +msgstr "调整 ``--tensor-parallel-size`` 以匹配您的硬件。有关通用 LMCache + vLLM 连接（端口、远程主机、进程内模式），请参见 :doc:`../mp/quickstart`。" #: ../../source/recipes/qwen3.rst:92 msgid "" "If there are any issues with vLLM setup, please refer to the `vLLM " "Recipes `_ " "for more details." -msgstr "" +msgstr "如果在 vLLM 设置中遇到任何问题，请参考 `vLLM Recipes `_ 以获取更多详细信息。" #: ../../source/recipes/qwen3.rst msgid "SGLang" -msgstr "" +msgstr "SGLang" #: ../../source/recipes/qwen3.rst:98 msgid "**Status:** Not validated with LMCache." -msgstr "" +msgstr "**状态：** 未通过 LMCache 验证。" #: ../../source/recipes/qwen3.rst msgid "TRT-LLM" -msgstr "" +msgstr "TRT-LLM" #: ../../source/recipes/qwen3.rst:102 msgid "**Status:** Not supported. LMCache TRT-LLM integration is in progress." -msgstr "" +msgstr "**状态：** 不支持。LMCache TRT-LLM 集成正在进行中。" #: ../../source/recipes/qwen3.rst:105 msgid "CacheBlend support" -msgstr "" +msgstr "CacheBlend 支持" #: ../../source/recipes/qwen3.rst:108 msgid "Compression support" -msgstr "" +msgstr "压缩支持" #: ../../source/recipes/qwen3.rst:114 msgid "Method" -msgstr "" +msgstr "方法" #: ../../source/recipes/qwen3.rst:115 msgid "Status" -msgstr "" +msgstr "状态" #: ../../source/recipes/qwen3.rst:116 msgid "Notes" -msgstr "" +msgstr "笔记" #: ../../source/recipes/qwen3.rst:117 msgid ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" -msgstr "" +msgstr ":doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>`" #: ../../source/recipes/qwen3.rst:118 msgid "Not validated" -msgstr "" +msgstr "未验证" #: ../../source/recipes/qwen3.rst:122 msgid "Caveats" -msgstr "" +msgstr "注意事项" #: ../../source/recipes/qwen3.rst:124 msgid "None known." -msgstr "" +msgstr "没有已知的问题。" From eaa2bfee5d7bda4ce543ee724385000115625a95 Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Wed, 20 May 2026 16:15:16 -0700 Subject: [PATCH 46/69] [CI] Restore NO_CUDA_EXT=1 skip-all-extensions semantics (#3349) Signed-off-by: deng451e <838677410@qq.com> --- lmcache/cli/commands/server.py | 10 +++++++--- setup.py | 8 +++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lmcache/cli/commands/server.py b/lmcache/cli/commands/server.py index e51ba7c415d..7854da57735 100644 --- a/lmcache/cli/commands/server.py +++ b/lmcache/cli/commands/server.py @@ -6,6 +6,9 @@ # First Party from lmcache.cli.commands.base import BaseCommand +from lmcache.logging import init_logger + +logger = init_logger(__name__) class ServerCommand(BaseCommand): @@ -53,9 +56,10 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: add_http_frontend_args(parser) add_observability_args(parser) except ImportError as e: - print( - f"Failed to import server dependencies: {e}. " - "Install the full lmcache package to use 'lmcache server'." + logger.warning( + "lmcache-cli (lightweight) detected (%s); install the full " + "lmcache package to use 'server' and 'bench'.", + e, ) return diff --git a/setup.py b/setup.py index 836cf6148af..3ad43c021a4 100644 --- a/setup.py +++ b/setup.py @@ -383,20 +383,22 @@ def _collect_extensions() -> tuple[list, dict]: Notes: - `sdist` builds skip all extension compilation. - - `NO_CUDA_EXT=1` keeps pure C++ extensions and skips GPU extensions. + - `NO_CUDA_EXT=1` skips all extension compilation (pure-Python build, + used by the lmcache-cli wheel which has no torch build dependency). - Otherwise, pure C++ extensions are combined with one GPU backend extension set (CUDA, ROCm, or SYCL). """ if BUILDING_SDIST: return source_dist_extension() + if NO_CUDA_EXT: + return [], {} + common_cpp_flags = _get_common_cpp_flags() # Preserve historical SYCL compatibility: lmcache_fs was compiled without # _GLIBCXX_USE_CXX11_ABI in pre-refactor builds. fs_cpp_flags = [] if BUILD_WITH_SYCL else common_cpp_flags ext_modules, cmdclass = _common_cpp_extensions(common_cpp_flags, fs_cpp_flags) - if NO_CUDA_EXT: - return ext_modules, cmdclass if BUILD_WITH_SYCL: gpu_ext_modules, cmdclass = sycl_extension() From 50209eca9ab55e1f987ac30dbaf2894764ca58d9 Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Wed, 20 May 2026 21:28:40 -0700 Subject: [PATCH 47/69] =?UTF-8?q?[CI]=20Rename=20NO=5FCUDA=5FEXT=20?= =?UTF-8?q?=E2=86=92=20NO=5FNATIVE=5FEXT=20(keep=20legacy=20alias)=20(#335?= =?UTF-8?q?4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rename NO_CUDA_EXT Signed-off-by: deng451e <838677410@qq.com> --- .github/workflows/build_cli_artifacts.yml | 4 ++-- .github/workflows/build_main_artifacts.yml | 4 ++-- AGENTS.md | 4 ++-- setup.py | 17 ++++++++++++++--- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_cli_artifacts.yml b/.github/workflows/build_cli_artifacts.yml index 3b21cf523ae..3b37a2cfe03 100644 --- a/.github/workflows/build_cli_artifacts.yml +++ b/.github/workflows/build_cli_artifacts.yml @@ -63,10 +63,10 @@ jobs: run: | rm -rf dist/ - - name: Build lmcache-cli wheel (pure Python, no CUDA) + - name: Build lmcache-cli wheel (pure Python, no native extensions) run: | cp pyproject_cli.toml pyproject.toml - NO_CUDA_EXT=1 python -m build --wheel + NO_NATIVE_EXT=1 python -m build --wheel - name: Smoke-test CLI wheel (no torch) run: | diff --git a/.github/workflows/build_main_artifacts.yml b/.github/workflows/build_main_artifacts.yml index d8fba7ba4bc..6f5ea45fe21 100644 --- a/.github/workflows/build_main_artifacts.yml +++ b/.github/workflows/build_main_artifacts.yml @@ -55,9 +55,9 @@ jobs: run: | rm -rf dist/ - - name: Build source distribution (no CUDA) + - name: Build source distribution (no native extensions) run: | - NO_CUDA_EXT=1 python -m build --sdist + NO_NATIVE_EXT=1 python -m build --sdist - name: Build CUDA wheels with cibuildwheel # Configuration is set in pyproject.toml diff --git a/AGENTS.md b/AGENTS.md index 5cc13f9be98..4ba051573dd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,8 +30,8 @@ uv pip install -e . --no-build-isolation # Standard install with CUDA extensions (requires torch pre-installed) pip install -e . --no-build-isolation -# Source-only (no CUDA extensions) -NO_CUDA_EXT=1 pip install -e . +# Source-only (no native extensions) +NO_NATIVE_EXT=1 pip install -e . # HIP/ROCm build BUILD_WITH_HIP=1 pip install -e . diff --git a/setup.py b/setup.py index 3ad43c021a4..2d6ce1df6cf 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,18 @@ # python -m build --sdist # will run python setup.py sdist --dist-dir dist BUILDING_SDIST = "sdist" in sys.argv -NO_CUDA_EXT = os.environ.get("NO_CUDA_EXT", "0") == "1" +# `NO_NATIVE_EXT=1` skips compilation of all native extensions (pure-Python +# build). `NO_CUDA_EXT=1` is the legacy name kept for backwards compatibility; +# it has always controlled all native extensions, not just CUDA ones. +NO_NATIVE_EXT = ( + os.environ.get("NO_NATIVE_EXT", "0") == "1" + or os.environ.get("NO_CUDA_EXT", "0") == "1" +) +if os.environ.get("NO_CUDA_EXT", "0") == "1": + print( + "warning: NO_CUDA_EXT is deprecated; use NO_NATIVE_EXT=1 instead.", + file=sys.stderr, + ) # New environment variable to choose between CUDA, HIP, and SYCL BUILD_WITH_HIP = os.environ.get("BUILD_WITH_HIP", "0") == "1" @@ -383,7 +394,7 @@ def _collect_extensions() -> tuple[list, dict]: Notes: - `sdist` builds skip all extension compilation. - - `NO_CUDA_EXT=1` skips all extension compilation (pure-Python build, + - `NO_NATIVE_EXT=1` skips all extension compilation (pure-Python build, used by the lmcache-cli wheel which has no torch build dependency). - Otherwise, pure C++ extensions are combined with one GPU backend extension set (CUDA, ROCm, or SYCL). @@ -391,7 +402,7 @@ def _collect_extensions() -> tuple[list, dict]: if BUILDING_SDIST: return source_dist_extension() - if NO_CUDA_EXT: + if NO_NATIVE_EXT: return [], {} common_cpp_flags = _get_common_cpp_flags() From 639df8389a0f9fa34229b0312d5a260f2505d093 Mon Sep 17 00:00:00 2001 From: longguo <107740309+abinggo@users.noreply.github.com> Date: Thu, 21 May 2026 13:48:49 +0800 Subject: [PATCH 48/69] fix(docker): install CLI requirements in standalone image so `lmcache server` boots (fixes #3353) (#3355) fix(docker): install CLI requirements in standalone image so `lmcache server` boots `lmcache/standalone` (v0.4.5, latest, nightly) crashes on `lmcache server` with: ModuleNotFoundError: No module named 'openai' Root cause: `setup.py` only puts `requirements/common.txt` and `requirements/cuda{12,13}_core.txt` into `install_requires`. `requirements/cli.txt` -- which contains `openai` (used by the `bench` engine client) -- is not installed. `lmcache server` is registered via `lmcache.cli.main:main`, which does `from lmcache.cli.commands import ALL_COMMANDS`. That triggers `lmcache.cli.commands._discover_commands()`, which uses `discover_subclasses(...)` to **eagerly importlib.import_module** every submodule under `cli/commands/`. That includes `bench/__init__.py`, whose top-level imports chain into `engine_bench/request_sender.py:11 -> from openai import AsyncOpenAI`, and the standalone image has no `openai`. `cli/commands/__init__.py:18-30` deliberately re-raises import errors ("a broken CLI command module should fail loudly rather than silently disappear from the CLI"), so the discovery cannot be softened on the slim install path -- the install path itself has to be fixed. The other Docker images (`Dockerfile`, `Dockerfile.rocm`, `Dockerfile.lightweight`, `Dockerfile.rocm-lightweight`) all install `vllm[runai,tensorizer,...]` or are `FROM vllm/vllm-openai*`, and `openai` arrives transitively through vLLM there. Only the standalone image is vLLM-free and hits this crash. Minimal fix: install `requirements/cli.txt` alongside the existing `ray nvidia-ml-py` step in the base stage, using a `--mount=type=bind` so the file does not persist into the image (matching the `build.txt` style used later by the lmcache-build stage). Out of scope (intentional, can be a follow-up): the deeper question of whether `cli.txt` should be in `install_requires`, an `extras_require["cli"]` block in `setup.py`, or whether the fail-loudly contract in `cli/commands/__init__.py` should be loosened, is an architectural call I'd defer to maintainers. Either of those would also fix #3353 (and would fix the same crash for users who `pip install lmcache` outside of Docker), but keeping this PR scoped to the reported failure mode -- the standalone Docker image -- keeps the review surface tight and unblocks the broken images today. Fixes #3353 Signed-off-by: abinggo <107740309+abinggo@users.noreply.github.com> --- docker/Dockerfile.standalone | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile.standalone b/docker/Dockerfile.standalone index 5abbcb29963..08624dc3cab 100644 --- a/docker/Dockerfile.standalone +++ b/docker/Dockerfile.standalone @@ -37,9 +37,15 @@ WORKDIR /workspace # Install small non-torch runtime dependencies; torch is installed in lmcache-build # against the exact CUDA version so the compiled C extensions match at runtime. +# The CLI requirements (e.g. ``openai`` for the bench client) are also installed +# here so ``lmcache`` subcommand discovery in ``lmcache.cli.commands`` -- which +# eagerly imports every command module -- does not crash with +# ``ModuleNotFoundError: No module named 'openai'`` when the standalone image +# boots ``lmcache server``. See LMCache#3353. RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=requirements/cli.txt,target=/tmp/cli.txt \ . /opt/venv/bin/activate && \ - uv pip install ray nvidia-ml-py + uv pip install ray nvidia-ml-py -r /tmp/cli.txt # CUDA arch list used by torch ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' From d9f3438b067ae36fd08671fb3e058d55ffd07634 Mon Sep 17 00:00:00 2001 From: deng451e <57919305+deng451e@users.noreply.github.com> Date: Thu, 21 May 2026 05:27:39 -0700 Subject: [PATCH 49/69] [CI][Build] CPU-only build (NO_GPU_EXT=1) + CI (#3357) new cpu only ci Signed-off-by: deng451e <838677410@qq.com> --- .github/workflows/build_cpu_artifacts.yml | 122 ++++++++++++++++++++++ .github/workflows/pr_full_build.yml | 5 + AGENTS.md | 3 + setup.py | 46 ++++---- 4 files changed, 156 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/build_cpu_artifacts.yml diff --git a/.github/workflows/build_cpu_artifacts.yml b/.github/workflows/build_cpu_artifacts.yml new file mode 100644 index 00000000000..91152e7bfdf --- /dev/null +++ b/.github/workflows/build_cpu_artifacts.yml @@ -0,0 +1,122 @@ +name: Build CPU Artifacts + +# Build LMCache with NO_GPU_EXT=1 (common C++ extensions only). +# Gates the CPU-only build path against regression. + +on: + workflow_call: + pull_request: + branches: + - dev + - "release-**" + paths: + - 'setup.py' + - 'pyproject.toml' + - 'csrc/**' + - 'requirements/common.txt' + - 'requirements/build.txt' + - 'lmcache/__init__.py' + - '.github/workflows/build_cpu_artifacts.yml' + +env: + LC_ALL: en_US.UTF-8 + +defaults: + run: + shell: bash + +# Cancel stale runs per ref. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build-cpu-artifacts: + name: Build CPU artifacts + runs-on: ubuntu-latest + steps: + - name: "Harden Runner" + uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 + with: + disable-sudo-and-containers: false + egress-policy: audit + + - name: Checkout code + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 0 + + - name: Remove non-release tags + run: git tag -l | grep -vE '^v[0-9]+\.[0-9]+\.[0-9]+$' | xargs -r git tag -d + + - name: Setup Python 3.13 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: "3.13" + + - name: Install build tooling and CPU torch + run: | + python -m pip install --upgrade pip + pip install --index-url https://download.pytorch.org/whl/cpu torch + # Pre-install build deps for --no-isolation below. + pip install -r requirements/build.txt + pip install build + + - name: Clean up release artifacts + run: | + rm -rf dist/ + + - name: Build CPU-only wheel (common C++ extensions, no GPU backend) + # --skip-dependency-check: the pre-installed CPU torch may not + # match pyproject.toml's exact build-system pin, but is ABI- + # compatible for compiling CppExtensions. + run: | + NO_GPU_EXT=1 python -m build --wheel --no-isolation \ + --skip-dependency-check + + - name: Verify CPU wheel contents + run: | + set -e + # Required: common C++ extensions. + for mod in native_storage_ops lmcache_redis lmcache_fs; do + if ! unzip -l dist/lmcache-*.whl | grep -qE "lmcache/${mod}\..*\.so"; then + echo "::error::missing common C++ extension: ${mod}" + exit 1 + fi + done + # Forbidden: GPU extensions. + for mod in c_ops xpu_ops; do + if unzip -l dist/lmcache-*.whl | grep -qE "lmcache/${mod}\..*\.so"; then + echo "::error::unexpected GPU extension in CPU wheel: ${mod}" + exit 1 + fi + done + + - name: Smoke-test CPU wheel + # Run from $RUNNER_TEMP so Python resolves `lmcache` from the + # installed wheel, not the checkout on CWD. + working-directory: ${{ runner.temp }} + run: | + set -e + pip install ${{ github.workspace }}/dist/lmcache-*.whl + python -c " + import lmcache + assert lmcache.torch_device_type == 'cpu', lmcache.torch_device_type + " + python -c "import lmcache.native_storage_ops" + python -c "import lmcache.lmcache_redis" + python -c "import lmcache.lmcache_fs" + # c_ops resolves to the python_ops_fallback shim on CPU. + python -c " + import sys, lmcache, lmcache.c_ops + assert sys.modules['lmcache.c_ops'].__name__ == 'lmcache.python_ops_fallback' + " + + - name: Upload CPU release artifacts to GHA + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: release-cpu-artifacts + path: dist/ diff --git a/.github/workflows/pr_full_build.yml b/.github/workflows/pr_full_build.yml index cdba6434ba7..95cd20837f8 100644 --- a/.github/workflows/pr_full_build.yml +++ b/.github/workflows/pr_full_build.yml @@ -38,6 +38,11 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'full') uses: ./.github/workflows/build_cli_artifacts.yml + build-cpu: + name: Build CPU + if: contains(github.event.pull_request.labels.*.name, 'full') + uses: ./.github/workflows/build_cpu_artifacts.yml + build-cu129: name: Build cu129 if: contains(github.event.pull_request.labels.*.name, 'full') diff --git a/AGENTS.md b/AGENTS.md index 4ba051573dd..aa1673d0449 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,6 +33,9 @@ pip install -e . --no-build-isolation # Source-only (no native extensions) NO_NATIVE_EXT=1 pip install -e . +# CPU-only (common C++ extensions, no GPU backend) +NO_GPU_EXT=1 pip install -e . --no-build-isolation + # HIP/ROCm build BUILD_WITH_HIP=1 pip install -e . ``` diff --git a/setup.py b/setup.py index 2d6ce1df6cf..0b2dbab0b32 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,8 @@ "warning: NO_CUDA_EXT is deprecated; use NO_NATIVE_EXT=1 instead.", file=sys.stderr, ) +# Common C++ extensions only; skip CUDA / ROCm / SYCL. +NO_GPU_EXT = os.environ.get("NO_GPU_EXT", "0") == "1" # New environment variable to choose between CUDA, HIP, and SYCL BUILD_WITH_HIP = os.environ.get("BUILD_WITH_HIP", "0") == "1" @@ -393,11 +395,10 @@ def _collect_extensions() -> tuple[list, dict]: - dict: cmdclass containing BuildExtension when extensions are built. Notes: - - `sdist` builds skip all extension compilation. - - `NO_NATIVE_EXT=1` skips all extension compilation (pure-Python build, - used by the lmcache-cli wheel which has no torch build dependency). - - Otherwise, pure C++ extensions are combined with one GPU backend - extension set (CUDA, ROCm, or SYCL). + - `sdist`: no extensions. + - `NO_NATIVE_EXT=1`: no extensions (pure-Python lmcache-cli wheel). + - `NO_GPU_EXT=1`: common C++ extensions only. + - Default: common C++ extensions + one GPU backend (CUDA/ROCm/SYCL). """ if BUILDING_SDIST: return source_dist_extension() @@ -411,6 +412,9 @@ def _collect_extensions() -> tuple[list, dict]: fs_cpp_flags = [] if BUILD_WITH_SYCL else common_cpp_flags ext_modules, cmdclass = _common_cpp_extensions(common_cpp_flags, fs_cpp_flags) + if NO_GPU_EXT: + return ext_modules, cmdclass + if BUILD_WITH_SYCL: gpu_ext_modules, cmdclass = sycl_extension() elif BUILD_WITH_HIP: @@ -425,21 +429,23 @@ def _collect_extensions() -> tuple[list, dict]: ext_modules, cmdclass = _collect_extensions() install_requires = _read_requirements(ROOT_DIR / "requirements" / "common.txt") - if BUILD_WITH_HIP: - core_file = "rocm_core.txt" - elif BUILD_WITH_SYCL: - core_file = "xpu_core.txt" - else: - # CUDA major selects between cu12 and cu13 vendor pins (cupy, nixl). - # Defaults to cu13 (the PyPI build); cu12.9 wheel builds set - # LMCACHE_CUDA_MAJOR=12 to pull cu12 wheels of those deps. - cuda_major = os.environ.get("LMCACHE_CUDA_MAJOR", "13") - if cuda_major not in ("12", "13"): - raise ValueError( - f"LMCACHE_CUDA_MAJOR must be '12' or '13', got '{cuda_major}'" - ) - core_file = f"cuda{cuda_major}_core.txt" - install_requires += _read_requirements(ROOT_DIR / "requirements" / core_file) + # NO_GPU_EXT skips GPU-vendor deps (cupy / nixl). + if not NO_GPU_EXT: + if BUILD_WITH_HIP: + core_file = "rocm_core.txt" + elif BUILD_WITH_SYCL: + core_file = "xpu_core.txt" + else: + # CUDA major selects between cu12 and cu13 vendor pins (cupy, nixl). + # Defaults to cu13 (the PyPI build); cu12.9 wheel builds set + # LMCACHE_CUDA_MAJOR=12 to pull cu12 wheels of those deps. + cuda_major = os.environ.get("LMCACHE_CUDA_MAJOR", "13") + if cuda_major not in ("12", "13"): + raise ValueError( + f"LMCACHE_CUDA_MAJOR must be '12' or '13', got '{cuda_major}'" + ) + core_file = f"cuda{cuda_major}_core.txt" + install_requires += _read_requirements(ROOT_DIR / "requirements" / core_file) setup( packages=find_packages( From 3300d0d9a3411c3b79f0190ab360408d1394c20c Mon Sep 17 00:00:00 2001 From: Samuel Shen Date: Thu, 21 May 2026 21:08:26 -0700 Subject: [PATCH 50/69] [Observability]: Add L2 Usage to SM (#3320) observability: add l2_usage_bytes gauge + dashboard panel Register an observable gauge that reports each L2 adapter's current usage via adapter.get_usage(), tagged by l2_name, plus a Grafana panel to visualize it. Lets operators query how much each L2 tier currently holds. Signed-off-by: Samuel Shen --- .../provisioning/dashboards/lmcache.json | 113 ++++++++++++++++-- lmcache/v1/distributed/storage_manager.py | 48 +++++++- 2 files changed, 148 insertions(+), 13 deletions(-) diff --git a/examples/observability/grafana/provisioning/dashboards/lmcache.json b/examples/observability/grafana/provisioning/dashboards/lmcache.json index 0f7edb448f5..0178be2b349 100644 --- a/examples/observability/grafana/provisioning/dashboards/lmcache.json +++ b/examples/observability/grafana/provisioning/dashboards/lmcache.json @@ -1623,13 +1623,106 @@ "title": "L2 IOPS", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Current bytes held in each L2 adapter, sampled live from adapter.get_usage(). One line per adapter, tagged by ``l2_name``. Parallel to the L1 ``lmcache_mp_l1_memory_usage_bytes`` gauge.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "lmcache_mp_l2_usage_bytes", + "legendFormat": "{{l2_name}}", + "refId": "A" + } + ], + "title": "L2 Usage (current)", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 66 }, "id": 300, "title": "Production Insights", @@ -1698,7 +1791,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 67 }, "id": 20, "options": { @@ -1803,7 +1896,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 67 }, "id": 21, "options": { @@ -1908,7 +2001,7 @@ "h": 8, "w": 12, "x": 0, - "y": 67 + "y": 75 }, "id": 22, "options": { @@ -2034,7 +2127,7 @@ "h": 8, "w": 12, "x": 12, - "y": 67 + "y": 75 }, "id": 23, "options": { @@ -2076,7 +2169,7 @@ "h": 1, "w": 24, "x": 0, - "y": 75 + "y": 83 }, "id": 400, "title": "CacheBlend (expand if using blend_server)", @@ -2161,7 +2254,7 @@ "h": 8, "w": 12, "x": 0, - "y": 76 + "y": 84 }, "id": 30, "options": { @@ -2330,7 +2423,7 @@ "h": 8, "w": 12, "x": 12, - "y": 76 + "y": 84 }, "id": 31, "options": { @@ -2398,7 +2491,7 @@ "h": 1, "w": 24, "x": 0, - "y": 76 + "y": 84 }, "id": 500, "title": "Request Traces", @@ -2417,7 +2510,7 @@ "h": 12, "w": 24, "x": 0, - "y": 77 + "y": 85 }, "id": 50, "options": { diff --git a/lmcache/v1/distributed/storage_manager.py b/lmcache/v1/distributed/storage_manager.py index 6b4f95bd996..d5568351af3 100644 --- a/lmcache/v1/distributed/storage_manager.py +++ b/lmcache/v1/distributed/storage_manager.py @@ -40,6 +40,7 @@ from lmcache.v1.memory_management import MemoryObj from lmcache.v1.mp_observability.event import Event, EventType from lmcache.v1.mp_observability.event_bus import get_event_bus +from lmcache.v1.mp_observability.otel_init import register_gauge from lmcache.v1.mp_observability.trace.decorator import ( enable_tracing, is_tracing_enabled, @@ -119,7 +120,7 @@ def __init__(self, config: StorageManagerConfig): ) self._l2_eviction_controller.start() - adapter_descriptors = [ + self._adapter_descriptors = [ AdapterDescriptor(index=i, config=ac) for i, ac in enumerate(config.l2_adapter_config.adapters) ] @@ -127,7 +128,7 @@ def __init__(self, config: StorageManagerConfig): self._store_controller = StoreController( l1_manager=self._l1_manager, l2_adapters=self._l2_adapters, - adapter_descriptors=adapter_descriptors, + adapter_descriptors=self._adapter_descriptors, policy=create_store_policy(config.store_policy), ) self._store_controller.start() @@ -136,12 +137,24 @@ def __init__(self, config: StorageManagerConfig): self._prefetch_controller = PrefetchController( l1_manager=self._l1_manager, l2_adapters=self._l2_adapters, - adapter_descriptors=adapter_descriptors, + adapter_descriptors=self._adapter_descriptors, policy=create_prefetch_policy(config.prefetch_policy), max_in_flight=config.prefetch_max_in_flight, ) self._prefetch_controller.start() + # L2 usage gauge — one observation per adapter, tagged by + # ``l2_name``. Parallel to L1Manager's ``l1_memory_usage_bytes``. + register_gauge( + "lmcache.l2", + "lmcache_mp.l2_usage_bytes", + ( + "Bytes currently held in each L2 adapter, tagged by " + "``l2_name`` (one observation per adapter)." + ), + self.get_l2_usages, + ) + # External APIs for serving engine integration code to call @enable_tracing() def reserve_write( @@ -569,6 +582,35 @@ def quota_manager(self) -> QuotaManager: """ return self._quota_manager + def get_l2_usages( + self, + ) -> list[tuple[int | float, dict[str, object]]]: + """Per-adapter L2 usage in OTel-observation shape. + + Backing data for the ``lmcache_mp.l2_usage_bytes`` observable + gauge. One entry per configured adapter. + + Returns: + A list of ``(total_bytes_used, {"l2_name": })`` + tuples — empty when no L2 adapters are configured. Adapters + whose ``get_usage()`` raises are skipped (the gauge prefers + silence over a poison observation). + """ + out: list[tuple[int | float, dict[str, object]]] = [] + for adapter, desc in zip( + self._l2_adapters, self._adapter_descriptors, strict=True + ): + try: + usage = adapter.get_usage() + except Exception: + logger.exception( + "L2 adapter %s get_usage() failed; skipping in gauge", + desc.type_name, + ) + continue + out.append((int(usage.total_bytes_used), {"l2_name": desc.type_name})) + return out + def get_usage_bytes_by_cache_salt(self) -> dict[str, int]: """Aggregate ``cache_salt`` byte usage across every L2 adapter. From 9f06a63ffca7e6dbd5aaca268eb428b2c2f3e0b8 Mon Sep 17 00:00:00 2001 From: maobaolong Date: Fri, 22 May 2026 12:23:12 +0800 Subject: [PATCH 51/69] Add macos ci check (#3148) * ci: add macOS compatibility check Bring over the macOS CI workflow and smoke test from #3148. Signed-off-by: baoloongmao * Address comment Signed-off-by: baoloongmao * introduce c++ event_notifier abstraction Signed-off-by: baoloongmao * Address comment from liqian Signed-off-by: baoloongmao * ci(macos): use NO_GPU_EXT=1 to align with build_cpu_artifacts.yml NO_CUDA_EXT was deprecated by #3357 in favor of two distinct knobs: - NO_NATIVE_EXT=1: skip all C++ extensions (pure-Python build) - NO_GPU_EXT=1: build common C++ ext, skip CUDA/ROCm/SYCL Switch the macOS workflow to NO_GPU_EXT=1 so we actually compile native_storage_ops / lmcache_redis / lmcache_fs on macOS and exercise the new PipeNotifier fallback in csrc/storage_backends/event_notifier.h. Also extend the smoke script with an import check that mirrors the verify step in build_cpu_artifacts.yml. Signed-off-by: baoloongmao --------- Signed-off-by: baoloongmao --- .github/scripts/macos_smoke_test.sh | 124 +++++++++++++++++ .github/workflows/macos_compat.yml | 150 +++++++++++++++++++++ csrc/storage_backends/connector_base.h | 65 ++------- csrc/storage_backends/event_notifier.h | 180 +++++++++++++++++++++++++ 4 files changed, 468 insertions(+), 51 deletions(-) create mode 100644 .github/scripts/macos_smoke_test.sh create mode 100644 .github/workflows/macos_compat.yml create mode 100644 csrc/storage_backends/event_notifier.h diff --git a/.github/scripts/macos_smoke_test.sh b/.github/scripts/macos_smoke_test.sh new file mode 100644 index 00000000000..15932ff2ea4 --- /dev/null +++ b/.github/scripts/macos_smoke_test.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# +# macOS basic-compatibility smoke test for the lmcache multiprocess +# server. Verifies that: +# 1) `lmcache --help` works (CLI entry point is importable) +# 2) common C++ extensions (native_storage_ops / lmcache_redis / +# lmcache_fs) load on macOS, exercising the PipeNotifier fallback +# in csrc/storage_backends/event_notifier.h +# 3) `lmcache server` can launch the ZMQ + HTTP server on CPU +# 4) the HTTP server answers GET / and GET /healthcheck +# +# This script is intentionally minimal — it does not exercise any +# GPU / CUDA / vLLM code paths. It is meant to be a fast regression +# signal against accidental Linux-only imports or filesystem usage +# (e.g. /dev/shm, librt, eventfd, fcntl at import time). + +set -euo pipefail + +HTTP_HOST="127.0.0.1" +HTTP_PORT="${LMCACHE_HTTP_PORT:-18080}" +ZMQ_PORT="${LMCACHE_ZMQ_PORT:-15555}" +LOG_FILE="${LMCACHE_LOG_FILE:-/tmp/lmcache_server.log}" +# GitHub macOS runners are noticeably slower than local macs on cold +# `import torch / fastapi / opentelemetry` chains, so budget enough +# wall-clock for the first HTTP hit after `lmcache server` starts. +STARTUP_TIMEOUT="${LMCACHE_STARTUP_TIMEOUT:-180}" + +echo "==> Environment" +uname -a || true +sw_vers || true +python --version +python -c "import torch; print('torch', torch.__version__, 'cuda', torch.cuda.is_available())" + +echo "==> Step 1: lmcache CLI help" +lmcache --help >/dev/null + +echo "==> Step 1.5: import common C++ extensions (CPU-only build)" +# Mirrors the verify step in .github/workflows/build_cpu_artifacts.yml +# on the macOS axis: makes sure NO_GPU_EXT=1 produced loadable .so's +# and that c_ops resolves to the python_ops_fallback shim. +python -c " +import sys +import lmcache +import lmcache.native_storage_ops # noqa: F401 +import lmcache.lmcache_redis # noqa: F401 +import lmcache.lmcache_fs # noqa: F401 +import lmcache.c_ops # noqa: F401 +assert lmcache.torch_device_type == 'cpu', lmcache.torch_device_type +assert sys.modules['lmcache.c_ops'].__name__ == 'lmcache.python_ops_fallback' +" + +echo "==> Step 2: launch 'lmcache server' on ${HTTP_HOST}:${HTTP_PORT} (zmq ${ZMQ_PORT})" +rm -f "${LOG_FILE}" +# Run the server in the background. Using `setsid`-like behavior via +# a subshell so we can kill the whole process group cleanly. +( + lmcache server \ + --host "${HTTP_HOST}" \ + --port "${ZMQ_PORT}" \ + --http-host "${HTTP_HOST}" \ + --http-port "${HTTP_PORT}" \ + --l1-size-gb "${LMCACHE_L1_SIZE_GB:-1}" \ + --eviction-policy "${LMCACHE_EVICTION_POLICY:-LRU}" \ + --no-l1-use-lazy \ + >"${LOG_FILE}" 2>&1 +) & +SERVER_PID=$! + +cleanup() { + echo "==> Cleanup: stopping server (pid=${SERVER_PID})" + if kill -0 "${SERVER_PID}" 2>/dev/null; then + kill "${SERVER_PID}" 2>/dev/null || true + # Give it a moment to exit; escalate if needed. + for _ in $(seq 1 10); do + kill -0 "${SERVER_PID}" 2>/dev/null || break + sleep 1 + done + kill -9 "${SERVER_PID}" 2>/dev/null || true + fi + if [[ -f "${LOG_FILE}" ]]; then + echo "==> Last 100 lines of server log:" + tail -n 100 "${LOG_FILE}" || true + fi +} +trap cleanup EXIT + +echo "==> Step 3: wait for HTTP endpoint (timeout=${STARTUP_TIMEOUT}s)" +READY=0 +for i in $(seq 1 "${STARTUP_TIMEOUT}"); do + if ! kill -0 "${SERVER_PID}" 2>/dev/null; then + echo "!! lmcache server exited prematurely after ${i}s" + break + fi + if curl -fsS --max-time 2 "http://${HTTP_HOST}:${HTTP_PORT}/" >/dev/null 2>&1; then + READY=1 + echo "==> Server reachable after ${i}s" + break + fi + sleep 1 +done + +if [[ "${READY}" != "1" ]]; then + echo "!! lmcache server did not become ready within ${STARTUP_TIMEOUT}s" + exit 1 +fi + +echo "==> Step 4: curl GET /" +ROOT_BODY="$(curl -fsS "http://${HTTP_HOST}:${HTTP_PORT}/")" +echo " body: ${ROOT_BODY}" +echo "${ROOT_BODY}" | grep -q '"status"' || { + echo "!! GET / did not return expected status field" + exit 1 +} + +echo "==> Step 5: curl GET /healthcheck" +HEALTH_BODY="$(curl -fsS "http://${HTTP_HOST}:${HTTP_PORT}/healthcheck")" +echo " body: ${HEALTH_BODY}" +echo "${HEALTH_BODY}" | grep -q '"status"[[:space:]]*:[[:space:]]*"healthy"' || { + echo "!! GET /healthcheck did not report healthy" + exit 1 +} + +echo "==> macOS smoke test passed." diff --git a/.github/workflows/macos_compat.yml b/.github/workflows/macos_compat.yml new file mode 100644 index 00000000000..1a350673e66 --- /dev/null +++ b/.github/workflows/macos_compat.yml @@ -0,0 +1,150 @@ +name: macOS Compat + +# Smoke test that verifies the lmcache multiprocess server can be +# installed and launched on macOS (GPU backend skipped via NO_GPU_EXT=1; +# common C++ extensions — native_storage_ops / lmcache_redis / +# lmcache_fs — are still built so the cross-platform PipeNotifier +# fallback in csrc/storage_backends/event_notifier.h is exercised), +# and that both the HTTP endpoint and the `lmcache` CLI are usable. +# This is a "best-effort basic compatibility" check — it does not +# cover GPU / CUDA / vLLM code paths. + +on: + workflow_call: + workflow_dispatch: + pull_request: + branches: + - "dev" + - "release-**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +defaults: + run: + shell: bash + +# Single source of truth for the smoke-script contract. The script +# (.github/scripts/macos_smoke_test.sh) reads these from the env, and +# the failure-artifact upload step below references LMCACHE_LOG_FILE. +env: + LMCACHE_HTTP_PORT: "18080" + LMCACHE_ZMQ_PORT: "15555" + LMCACHE_LOG_FILE: "/tmp/lmcache_server.log" + LMCACHE_STARTUP_TIMEOUT: "180" + +jobs: + changes: + name: Detect changes + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + # workflow_call / workflow_dispatch bypass the paths-filter (which + # only runs for push / pull_request). Treat both as "always run" + # so callers don't get a silently-skipped job. + lmcache: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || steps.filter.outputs.lmcache == 'true' }} + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + if: github.event_name == 'push' + with: + fetch-depth: 2 + + - uses: dorny/paths-filter@v3 + if: github.event_name == 'push' || github.event_name == 'pull_request' + id: filter + with: + filters: | + lmcache: + - 'lmcache/**' + - 'pyproject.toml' + - 'setup.py' + - 'requirements/**.txt' + - '.github/workflows/macos_compat.yml' + - '.github/scripts/macos_smoke_test.sh' + - '!operator/**' + + macos-smoke: + needs: changes + if: needs.changes.outputs.lmcache == 'true' + name: "macOS smoke: py${{ matrix.python }} on ${{ matrix.platform }}" + runs-on: "${{ matrix.platform }}" + strategy: + fail-fast: false + matrix: + python: + - "3.11" + - "3.12" + platform: + # macos-latest is Apple Silicon (arm64); macos-13 is the last + # GitHub-hosted Intel runner. Keep both until macos-13 is + # retired by GitHub. + - "macos-latest" + - "macos-13" + + steps: + - name: Harden Runner + uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + # Shallow clone is enough; SETUPTOOLS_SCM_PRETEND_VERSION + # below avoids any git-tag resolution by setuptools-scm. + fetch-depth: 1 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: ${{ matrix.python }} + cache: pip + cache-dependency-path: | + pyproject.toml + requirements/*.txt + + - name: Install lmcache (CPU-only, common C++ ext, no vLLM) + env: + # NO_GPU_EXT=1: build common C++ extensions (native_storage_ops, + # lmcache_redis, lmcache_fs) but skip the CUDA / ROCm / SYCL + # backend. Mirrors build_cpu_artifacts.yml on the macOS axis. + NO_GPU_EXT: "1" + # Pin a synthetic version so setuptools-scm never inspects + # git tags. Defense-in-depth on top of pyproject.toml's + # tag_regex; harmless if tag_regex already filters tags. + SETUPTOOLS_SCM_PRETEND_VERSION: "0.0.0.dev0" + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements/build.txt + # Pure-Python deps. macOS wheels exist for everything in + # common.txt; we explicitly skip requirements/cuda13_core.txt + # (the default core file picked by setup.py) since cupy and + # nixl are CUDA/Linux-only. + python -m pip install torch + python -m pip install -r requirements/common.txt + # CLI entry points eagerly import openai / matplotlib (e.g. + # lmcache.cli.commands.bench), so cli.txt is needed for the + # `lmcache --help` step inside the smoke script. + python -m pip install -r requirements/cli.txt + # Install lmcache itself with --no-deps so setup.py's + # install_requires (which appends cuda13_core.txt by default) + # cannot drag CUDA-only wheels onto macOS. + python -m pip install -e . --no-deps --no-build-isolation + + - name: Run macOS smoke test + run: | + chmod +x .github/scripts/macos_smoke_test.sh + .github/scripts/macos_smoke_test.sh + + - name: Upload server log on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: lmcache-server-log-${{ matrix.platform }}-py${{ matrix.python }} + path: ${{ env.LMCACHE_LOG_FILE }} + if-no-files-found: ignore diff --git a/csrc/storage_backends/connector_base.h b/csrc/storage_backends/connector_base.h index 0de775714b9..c2cd35a174a 100644 --- a/csrc/storage_backends/connector_base.h +++ b/csrc/storage_backends/connector_base.h @@ -3,13 +3,14 @@ #include "connector_interface.h" #include "connector_types.h" -#include +#include "event_notifier.h" #include #include #include #include #include #include +#include #include #include #include @@ -64,13 +65,9 @@ class ConnectorBase : public IStorageConnector { } } - // create eventfd for async notification - // EFD_NONBLOCK: read() and write() are non-blocking - // EFD_CLOEXEC: close on exec - efd_ = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); - if (efd_ < 0) { - throw std::runtime_error("failed to create eventfd"); - } + // Cross-platform poll-able wakeup fd (eventfd on Linux, + // self-pipe on macOS). See event_notifier.h. + notifier_ = make_event_notifier(); } virtual ~ConnectorBase() { close(); } @@ -78,7 +75,7 @@ class ConnectorBase : public IStorageConnector { ConnectorBase(const ConnectorBase&) = delete; ConnectorBase& operator=(const ConnectorBase&) = delete; - int event_fd() const override { return efd_; } + int event_fd() const override { return notifier_->fileno(); } uint64_t submit_batch_get(const std::vector& keys, const std::vector& bufs, @@ -205,8 +202,7 @@ class ConnectorBase : public IStorageConnector { signaled_.store(false, std::memory_order_release); if (!completions_.empty() && !signaled_.exchange(true, std::memory_order_acq_rel)) { - uint64_t x = 1; - ::write(efd_, &x, sizeof(x)); + notifier_->notify(); } break; } @@ -248,10 +244,9 @@ class ConnectorBase : public IStorageConnector { // Derived cleanup that must only run after workers have stopped. on_workers_stopped(); - // Close eventfd - if (efd_ >= 0) { - ::close(efd_); - efd_ = -1; + // Close wakeup fd + if (notifier_) { + notifier_->close(); } // Clear queues (no GIL needed - python guarantees buffers stay alive) @@ -461,44 +456,12 @@ class ConnectorBase : public IStorageConnector { signal_eventfd_(); } - void drain_eventfd_() { - // loop to consume all writes that happened since last drain - for (;;) { - uint64_t x; - ssize_t r = ::read(efd_, &x, sizeof(x)); - if (r == static_cast(sizeof(x))) { - continue; // keep draining - } - if (r < 0) { - if (errno == EINTR) { - continue; // retry on EINTR - } - if (errno == EAGAIN) { - break; // drained (no more data) - } - } - break; - } - } + void drain_eventfd_() { notifier_->consume(); } void signal_eventfd_() { bool already_signaled = signaled_.exchange(true, std::memory_order_acq_rel); if (already_signaled) return; // only one signal at a time - - uint64_t x = 1; - for (;;) { - ssize_t w = ::write(efd_, &x, sizeof(x)); - if (w == static_cast(sizeof(x))) { - return; // success - } - if (w < 0) { - if (errno == EINTR) { - continue; // retry on EINTR - } - throw std::runtime_error("eventfd write failed unexpectedly"); - } - throw std::runtime_error("partial write to eventfd"); - } + notifier_->notify(); } static const std::unordered_map>& @@ -668,9 +631,9 @@ class ConnectorBase : public IStorageConnector { std::atomic next_future_id_{1}; private: - int efd_ = -1; + std::unique_ptr notifier_; - // treat eventfd as a binary wakeup flag: + // treat wakeup fd as a binary wakeup flag: // true: Python has been signaled (or will be) // false: Python is asleep, no wakeup pending std::atomic signaled_{false}; diff --git a/csrc/storage_backends/event_notifier.h b/csrc/storage_backends/event_notifier.h new file mode 100644 index 00000000000..5fc55062a5f --- /dev/null +++ b/csrc/storage_backends/event_notifier.h @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +// Cross-platform event notification abstraction (C++ mirror of +// lmcache/v1/platform/event_notifier.py). Provides a unified +// EventNotifier interface for signaling between threads using a +// poll-able file descriptor. +// +// On Linux we use eventfd(2); on macOS / other POSIX we fall back +// to a self-pipe with non-blocking I/O. Connector code stays +// platform-agnostic — only this header has #if branches. + +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) + #include +#endif + +namespace lmcache { +namespace connector { + +// Abstract base: a binary signal that a listener can poll() on. +// notify() makes fileno() readable; consume() resets it. Multiple +// notify() calls before a consume() are coalesced. +class EventNotifier { + public: + virtual ~EventNotifier() = default; + + // Poll-able file descriptor. Becomes readable after notify(). + virtual int fileno() const = 0; + + // Signal the notifier (idempotent if already signaled). + virtual void notify() = 0; + + // Consume any pending signal (non-blocking). EAGAIN/EWOULDBLOCK + // is the only swallowed error; other OS errors propagate. + virtual void consume() = 0; + + // Release underlying OS resources. Idempotent. + virtual void close() = 0; + + EventNotifier() = default; + EventNotifier(const EventNotifier&) = delete; + EventNotifier& operator=(const EventNotifier&) = delete; +}; + +#if defined(__linux__) + +// Linux eventfd-based notifier. +class EventfdNotifier final : public EventNotifier { + public: + EventfdNotifier() { + efd_ = ::eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (efd_ < 0) { + throw std::runtime_error("failed to create eventfd"); + } + } + + ~EventfdNotifier() override { close(); } + + int fileno() const override { return efd_; } + + void notify() override { + uint64_t one = 1; + for (;;) { + ssize_t w = ::write(efd_, &one, sizeof(one)); + if (w == static_cast(sizeof(one))) return; + if (w < 0) { + if (errno == EINTR) continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) return; + throw std::runtime_error("eventfd write failed unexpectedly"); + } + throw std::runtime_error("partial write to eventfd"); + } + } + + void consume() override { + for (;;) { + uint64_t x; + ssize_t r = ::read(efd_, &x, sizeof(x)); + if (r == static_cast(sizeof(x))) continue; + if (r < 0 && errno == EINTR) continue; + break; // drained or non-EINTR error (EAGAIN expected) + } + } + + void close() override { + if (efd_ >= 0) { + ::close(efd_); + efd_ = -1; + } + } + + private: + int efd_ = -1; +}; + +#endif // __linux__ + +// Pipe-based fallback notifier (macOS / other POSIX). +class PipeNotifier final : public EventNotifier { + public: + PipeNotifier() { + int fds[2]; + if (::pipe(fds) != 0) { + throw std::runtime_error("failed to create pipe"); + } + for (int i = 0; i < 2; ++i) { + int flags = ::fcntl(fds[i], F_GETFL, 0); + if (flags < 0 || ::fcntl(fds[i], F_SETFL, flags | O_NONBLOCK) < 0 || + ::fcntl(fds[i], F_SETFD, FD_CLOEXEC) < 0) { + ::close(fds[0]); + ::close(fds[1]); + throw std::runtime_error("failed to configure pipe fd flags"); + } + } + read_fd_ = fds[0]; + write_fd_ = fds[1]; + } + + ~PipeNotifier() override { close(); } + + int fileno() const override { return read_fd_; } + + void notify() override { + const uint8_t one = 1; + for (;;) { + ssize_t w = ::write(write_fd_, &one, sizeof(one)); + if (w == 1) return; + if (w < 0) { + if (errno == EINTR) continue; + // Pipe full = signal already pending; readers will see it. + if (errno == EAGAIN || errno == EWOULDBLOCK) return; + throw std::runtime_error("pipe write failed unexpectedly"); + } + } + } + + void consume() override { + uint8_t buf[256]; + for (;;) { + ssize_t r = ::read(read_fd_, buf, sizeof(buf)); + if (r > 0) continue; + if (r < 0 && errno == EINTR) continue; + break; // drained, EOF, or EAGAIN + } + } + + void close() override { + if (read_fd_ >= 0) { + ::close(read_fd_); + read_fd_ = -1; + } + if (write_fd_ >= 0) { + ::close(write_fd_); + write_fd_ = -1; + } + } + + private: + int read_fd_ = -1; + int write_fd_ = -1; +}; + +// Factory: returns the platform-appropriate notifier. +inline std::unique_ptr make_event_notifier() { +#if defined(__linux__) + return std::unique_ptr(new EventfdNotifier()); +#else + return std::unique_ptr(new PipeNotifier()); +#endif +} + +} // namespace connector +} // namespace lmcache From 4168b0991cd1c5eeef36efda6c7dc4c91f7b8989 Mon Sep 17 00:00:00 2001 From: maobaolong Date: Fri, 22 May 2026 16:36:03 +0800 Subject: [PATCH 52/69] Hotfix: remove macos-13 from ci workflow as resource insufficient (#3366) Signed-off-by: baoloongmao --- .github/workflows/macos_compat.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/macos_compat.yml b/.github/workflows/macos_compat.yml index 1a350673e66..5559896857e 100644 --- a/.github/workflows/macos_compat.yml +++ b/.github/workflows/macos_compat.yml @@ -80,11 +80,8 @@ jobs: - "3.11" - "3.12" platform: - # macos-latest is Apple Silicon (arm64); macos-13 is the last - # GitHub-hosted Intel runner. Keep both until macos-13 is - # retired by GitHub. + # macos-latest is Apple Silicon (arm64); - "macos-latest" - - "macos-13" steps: - name: Harden Runner From 9793d8b025b56ace896b1ebeb1eaed5d87021c18 Mon Sep 17 00:00:00 2001 From: maobaolong Date: Sat, 23 May 2026 08:21:16 +0800 Subject: [PATCH 53/69] [Integration] LMCache MP mode warn log while cannot reach lmcache server (#3365) Signed-off-by: baoloongmao --- .../vllm/vllm_multi_process_adapter.py | 42 +++++++++++++++---- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/lmcache/integration/vllm/vllm_multi_process_adapter.py b/lmcache/integration/vllm/vllm_multi_process_adapter.py index 5335cd259d1..48b76214bd8 100644 --- a/lmcache/integration/vllm/vllm_multi_process_adapter.py +++ b/lmcache/integration/vllm/vllm_multi_process_adapter.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Standard from dataclasses import dataclass -from typing import Any, Callable +from typing import Any, Callable, NoReturn import enum import os import threading @@ -169,6 +169,36 @@ def get_lmcache_chunk_size( return chunk_size +def _raise_server_unreachable(server_url: str, timeout: float) -> NoReturn: + """Raise a verbose ConnectionError when the LMCache MP server is + unreachable. + + The message intentionally spells out the most common cause (the + standalone ``lmcache server`` process is not running), the URL that + was being dialed, and the exact command to start it -- so that users + landing here via ``vllm serve --kv-offloading-backend lmcache`` are + not left guessing. + """ + hint = ( + "Cannot reach the LMCache MP server at " + f"'{server_url}' within {timeout}s.\n" + "This usually means the standalone LMCache server is not " + "running, or it is listening on a different host/port.\n" + "To start one locally with the default port (5555):\n" + " lmcache server --l1-size-gb 20 --eviction-policy LRU\n" + "To target a different host/port, override via " + "kv_connector_extra_config (lmcache.mp.host / lmcache.mp.port), " + "e.g.:\n" + ' --kv-transfer-config \'{"kv_connector":' + '"LMCacheMPConnector","kv_role":"kv_both",' + '"kv_connector_extra_config":{"lmcache.mp.host":' + '"tcp://localhost","lmcache.mp.port":5555}}\'\n' + "See https://docs.lmcache.ai/mp/quickstart.html for details." + ) + logger.warning(hint) + raise ConnectionError(hint) from None + + def send_ping( mq_client: MessageQueueClient, timeout: float, @@ -452,10 +482,7 @@ def __init__( ) except TimeoutError: self.mq_client.close() - raise ConnectionError( - f"LMCache server did not respond within {self._mq_timeout}s. " - "Is the server running?" - ) from None + _raise_server_unreachable(server_url, self._mq_timeout) assert self.chunk_size % vllm_block_size == 0, ( "LMCache chunk size should be a multiple of vLLM block size" ) @@ -840,10 +867,7 @@ def __init__( ) except TimeoutError: self.mq_client.close() - raise ConnectionError( - f"LMCache server did not respond within {self._mq_timeout}s. " - "Is the server running?" - ) from None + _raise_server_unreachable(server_url, self._mq_timeout) assert chunk_size % vllm_block_size == 0, ( "LMCache chunk size should be a multiple of vLLM block size" ) From 84079945c680c9820e3a7712f390c907df0221d8 Mon Sep 17 00:00:00 2001 From: Yihua Cheng Date: Mon, 25 May 2026 16:27:26 -0500 Subject: [PATCH 54/69] [CI][Hotfix] fix the pytest lazy import issue caused by vLLM torch dynamo compilation (#3390) [CI][Hotfix] Add pytest to runtime deps to fix cupy lazy import crash Signed-off-by: ApostaC --- requirements/common.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements/common.txt b/requirements/common.txt index a7bc367b51f..90170412b74 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -19,6 +19,9 @@ opentelemetry-exporter-prometheus >= 0.50b0, <= 0.61b0 prometheus_client >= 0.18.0, <= 0.24.1 psutil py-cpuinfo +# cupy.testing._random unconditionally imports pytest at module level; +# torch._dynamo triggers this via inspect.getmodule() during compilation. +pytest pyyaml pyzmq >= 25.0.0 redis From f3427260ba6254370d295294b4323413ae183821 Mon Sep 17 00:00:00 2001 From: zhengchenyu Date: Tue, 26 May 2026 06:49:32 +0800 Subject: [PATCH 55/69] docs: add docstrings to public helpers in lmcache/utils.py (#3380) * docs: add docstrings to public helpers in lmcache/utils.py Signed-off-by: zhengchenyu * fix style Signed-off-by: zhengchenyu * trigger ci Signed-off-by: zhengchenyu --------- Signed-off-by: zhengchenyu --- lmcache/utils.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lmcache/utils.py b/lmcache/utils.py index e7648b739aa..2e7193ebf28 100644 --- a/lmcache/utils.py +++ b/lmcache/utils.py @@ -285,6 +285,13 @@ def parse_mixed_slot_mapping( def get_version(): + """Return a human-readable version string. + + Returns: + ``"-"``, with ``"NA"`` substituted when the + package version or commit id is not available (e.g. when running + from a source checkout without a tag). + """ version_display = VERSION if VERSION else "NA" commit_id_display = COMMIT_ID if COMMIT_ID else "NA" return f"{version_display}-{commit_id_display}" @@ -676,6 +683,15 @@ def _lmcache_nvtx_annotate(func, domain="lmcache"): def thread_safe(func): + """Wrap a callable with the shared observability lock. + + Args: + func: Callable to execute while holding the lock. + + Returns: + A wrapper that serializes calls to ``func`` using the shared lock. + """ + def wrapper(*args, **kwargs): with _shared_observability_lock: result = func(*args, **kwargs) @@ -686,12 +702,22 @@ def wrapper(*args, **kwargs): #### Thread/asyncio-related utilities #### def handle_thread_exception(args): + """Handle an uncaught exception reported by ``threading``. + + Args: + args: Thread exception information provided by ``threading``. + """ logger.error( f"Thread {args.thread.name} crashed: {args.exc_type.__name__}: {args.exc_value}" ) def start_loop_in_thread_with_exceptions(loop: asyncio.AbstractEventLoop): + """Run an event loop forever with an exception handler. + + Args: + loop: Event loop to bind to the current thread and run. + """ # The loop must be set in the *same* thread where it runs. asyncio.set_event_loop(loop) From 59beadb002fba6fe93741a8d3301c6718c2efef6 Mon Sep 17 00:00:00 2001 From: Nachiket Torwekar Date: Mon, 25 May 2026 16:33:13 -0700 Subject: [PATCH 56/69] cleanup: remove duplicate import time in blend_server_v2 (#3379) cleanup: remove duplicate time import Signed-off-by: Nachiket Torwekar --- lmcache/v1/multiprocess/blend_server_v2.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lmcache/v1/multiprocess/blend_server_v2.py b/lmcache/v1/multiprocess/blend_server_v2.py index b5bc6efc380..1d355179a63 100644 --- a/lmcache/v1/multiprocess/blend_server_v2.py +++ b/lmcache/v1/multiprocess/blend_server_v2.py @@ -631,9 +631,6 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: if found_count is not None: break - # Standard - import time - time.sleep(0.001) # Real found count after dedup the TP From 50e454349b27a646241af1d035f75e39aa4fe0b5 Mon Sep 17 00:00:00 2001 From: elliotz-ai Date: Mon, 25 May 2026 17:01:56 -0700 Subject: [PATCH 57/69] [MP][Bugfix] Fix store/retrieve deadlock by routing completion callbacks through C++ host callback (#3336) Signed-off-by: elliotz --- csrc/completion_recorder.cpp | 59 ++++++ csrc/completion_recorder.h | 51 +++++ csrc/pybind.cpp | 14 ++ lmcache/v1/multiprocess/native_completion.py | 150 +++++++++++++++ lmcache/v1/multiprocess/server.py | 33 +++- setup.py | 2 + .../multiprocess/test_completion_recorder.py | 179 ++++++++++++++++++ 7 files changed, 484 insertions(+), 4 deletions(-) create mode 100644 csrc/completion_recorder.cpp create mode 100644 csrc/completion_recorder.h create mode 100644 lmcache/v1/multiprocess/native_completion.py create mode 100644 tests/v1/multiprocess/test_completion_recorder.py diff --git a/csrc/completion_recorder.cpp b/csrc/completion_recorder.cpp new file mode 100644 index 00000000000..9f70ecd44b3 --- /dev/null +++ b/csrc/completion_recorder.cpp @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +#include "completion_recorder.h" + +#include + +CompletionRecorder& CompletionRecorder::instance() { + static CompletionRecorder recorder; + return recorder; +} + +void CompletionRecorder::push(std::unique_ptr completion) { + std::lock_guard lock(mutex_); + buffer_.push_back(std::move(completion)); +} + +std::vector> CompletionRecorder::drain() { + std::lock_guard lock(mutex_); + std::vector> result; + result.swap(buffer_); + return result; +} + +static void +#ifndef USE_ROCM + CUDART_CB +#endif + completion_host_callback(void* data) { + // Adopt the raw pointer back into a unique_ptr. + std::unique_ptr completion( + static_cast(data)); + CompletionRecorder::instance().push(std::move(completion)); +} + +void record_completion_on_stream(int64_t cuda_stream_ptr, + const std::string& kind, std::string payload) { + auto completion = std::make_unique( + PendingCompletion{kind, std::move(payload)}); + auto stream = reinterpret_cast( + static_cast(cuda_stream_ptr)); + // Pass ownership through the driver as a raw pointer; the host callback + // re-adopts it. Reclaim if the launch itself fails. + PendingCompletion* raw = completion.release(); + auto err = LMCACHE_COMPLETION_LAUNCH_HOST_FUNC(stream, + completion_host_callback, raw); + if (err != 0) { + delete raw; + } +} + +CompletionDrainResult drain_recorded_completions() { + auto completions = CompletionRecorder::instance().drain(); + CompletionDrainResult result; + result.reserve(completions.size()); + for (auto& c : completions) { + result.emplace_back(std::move(c->kind), std::move(c->payload)); + } + return result; +} diff --git a/csrc/completion_recorder.h b/csrc/completion_recorder.h new file mode 100644 index 00000000000..b02aa55f7c7 --- /dev/null +++ b/csrc/completion_recorder.h @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// CUDA/HIP host-callback buffer for stream-completion records. +// Mirrors event_recorder.h but carries an opaque bytes payload instead +// of timestamped metadata. The callback runs without the GIL; Python +// drains the buffer and dispatches to a handler keyed by ``kind``. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#ifdef USE_ROCM + #include +using lmcache_completion_stream_t = hipStream_t; + #define LMCACHE_COMPLETION_LAUNCH_HOST_FUNC hipLaunchHostFunc +#else + #include +using lmcache_completion_stream_t = cudaStream_t; + #define LMCACHE_COMPLETION_LAUNCH_HOST_FUNC cudaLaunchHostFunc +#endif + +struct PendingCompletion { + std::string kind; // dispatch key, e.g. "finish_write" + std::string payload; // opaque encoded bytes (e.g. msgpack) +}; + +class CompletionRecorder { + public: + static CompletionRecorder& instance(); + // Takes ownership of the heap-allocated PendingCompletion. + void push(std::unique_ptr completion); + std::vector> drain(); + + private: + CompletionRecorder() = default; + std::mutex mutex_; + std::vector> buffer_; +}; + +// Schedule a completion record. Called WITHOUT the GIL. +void record_completion_on_stream(int64_t cuda_stream_ptr, + const std::string& kind, std::string payload); + +using CompletionDrainResult = std::vector>; + +CompletionDrainResult drain_recorded_completions(); diff --git a/csrc/pybind.cpp b/csrc/pybind.cpp index 57e4a02e56e..2228795d59a 100644 --- a/csrc/pybind.cpp +++ b/csrc/pybind.cpp @@ -9,6 +9,7 @@ #include "mem_alloc.h" #include "utils.h" #include "event_recorder.h" +#include "completion_recorder.h" #include #include #include @@ -96,4 +97,17 @@ PYBIND11_MODULE(c_ops, m) { py::arg("session_id"), py::arg("str_metadata"), py::arg("int_metadata"), py::call_guard()); m.def("drain_recorded_events", &drain_recorded_events); + m.def("record_completion_on_stream", &record_completion_on_stream, + py::arg("cuda_stream_ptr"), py::arg("kind"), py::arg("payload"), + py::call_guard()); + // Return each payload as py::bytes; pybind11 utf-8-decodes std::string + // by default, corrupting binary payloads (e.g. msgpack). + m.def("drain_recorded_completions", []() { + auto items = drain_recorded_completions(); + py::list out; + for (auto& kv : items) { + out.append(py::make_tuple(py::str(kv.first), py::bytes(kv.second))); + } + return out; + }); } diff --git a/lmcache/v1/multiprocess/native_completion.py b/lmcache/v1/multiprocess/native_completion.py new file mode 100644 index 00000000000..31c2e7e1923 --- /dev/null +++ b/lmcache/v1/multiprocess/native_completion.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Dispatch device-stream-ordered host callbacks without GIL contention. + +Call graph:: + + Server.store / Server.retrieve (Python, holds GIL) + | + v + submit_callback_to_stream(stream, kind, payload) + | + v Python: msgspec.msgpack.encode(payload) + v C++: record_completion_on_stream (gil_scoped_release) + v C++: cudaLaunchHostFunc / hipLaunchHostFunc + | + v -------- on the CUDA/HIP driver thread (no GIL) -------- + v completion_host_callback: append PendingCompletion to buffer + | + v -------- on DeviceHostFuncDispatcher drain thread (Python) -- + v drain_recorded_completions (returns list of (kind, bytes)) + v msgspec.msgpack.decode(bytes) with the kind's registered type + v handler(decoded_payload) ← finish_write / finish_read_prefetched + +Payload round-trips as a single opaque blob. For multi-arg handlers, wrap +the args in a tuple/struct matching the registered ``payload_type``. + +Extends PR #2952's AB-BA fix (GIL x driver lock) to Server.store/retrieve. +""" + +# Future +from __future__ import annotations + +# Standard +from typing import Any, Callable +import threading + +# Third Party +import msgspec +import torch # noqa: F401 — must be imported before lmcache.c_ops + +# First Party +from lmcache.logging import init_logger +import lmcache.c_ops as _lmc_ops + +logger = init_logger(__name__) + +# Runs on dispatcher thread with the decoded payload for one submission. +DeviceHostFunc = Callable[[Any], None] + + +class _Registration(msgspec.Struct): + handler: DeviceHostFunc + decoder: msgspec.msgpack.Decoder + + +class DeviceHostFuncDispatcher: + """Drain buffered C++ completions and dispatch each payload to the handler + registered for its ``kind``. One instance per process; owned by + ``MPCacheEngine``.""" + + def __init__(self, drain_interval_seconds: float = 0.005) -> None: + self._registry: dict[str, _Registration] = {} + self._lock = threading.Lock() + self._stop_flag = threading.Event() + self._wake = threading.Event() + self._thread: threading.Thread | None = None + self._drain_interval = drain_interval_seconds + self._dispatched_count = 0 + self._exception_counts: dict[str, int] = {} + + def register(self, kind: str, handler: DeviceHostFunc, payload_type: Any) -> None: + """Register *handler* for *kind*. ``payload_type`` is the msgspec + decode type for the whole payload (e.g. ``list[ObjectKey]``).""" + with self._lock: + self._registry[kind] = _Registration( + handler=handler, + decoder=msgspec.msgpack.Decoder(type=payload_type), + ) + + def start(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return + self._stop_flag.clear() + self._thread = threading.Thread( + target=self._run, + daemon=True, + name="DeviceHostFuncDispatcher", + ) + self._thread.start() + + def stop(self) -> None: + self._stop_flag.set() + self._wake.set() + if self._thread is not None and self._thread.is_alive(): + self._thread.join() + self._drain_once() + + def dispatched_count(self) -> int: + return self._dispatched_count # single-writer; read is GIL-atomic + + def handler_exception_counts(self) -> dict[str, int]: + with self._lock: + return dict(self._exception_counts) + + def _run(self) -> None: + while not self._stop_flag.is_set(): + self._wake.wait(timeout=self._drain_interval) + self._wake.clear() + self._drain_once() + + def _drain_once(self) -> None: + # Broad except keeps the drain thread alive across native/handler errors. + try: + completions = _lmc_ops.drain_recorded_completions() + except Exception: + logger.exception("DeviceHostFuncDispatcher: drain failed") + return + if not completions: + return + with self._lock: + registry = dict(self._registry) + for kind, encoded_payload in completions: + reg = registry.get(kind) + if reg is None: + logger.warning( + "DeviceHostFuncDispatcher: no handler for kind=%r (dropped)", + kind, + ) + continue + try: + decoded = reg.decoder.decode(encoded_payload) + reg.handler(decoded) + self._dispatched_count += 1 + except Exception: + with self._lock: + self._exception_counts[kind] = ( + self._exception_counts.get(kind, 0) + 1 + ) + logger.exception( + "DeviceHostFuncDispatcher: handler for %r raised", kind + ) + + +def submit_callback_to_stream(stream: Any, kind: str, payload: Any) -> None: + """Schedule a stream-ordered host callback for *kind* with *payload*. + Runs on the dispatcher worker registered for *kind*; never acquires the + GIL on the driver thread. ``payload`` is delivered to the handler as a + single argument.""" + encoded = msgspec.msgpack.encode(payload) + _lmc_ops.record_completion_on_stream(stream.ptr, kind, encoded) diff --git a/lmcache/v1/multiprocess/server.py b/lmcache/v1/multiprocess/server.py index bb67d07bf54..b8cedd73256 100644 --- a/lmcache/v1/multiprocess/server.py +++ b/lmcache/v1/multiprocess/server.py @@ -63,6 +63,10 @@ GPUCacheContext, ) from lmcache.v1.multiprocess.mq import MessageQueueServer +from lmcache.v1.multiprocess.native_completion import ( + DeviceHostFuncDispatcher, + submit_callback_to_stream, +) from lmcache.v1.multiprocess.non_gpu_context import NonGpuContextMetadata from lmcache.v1.multiprocess.protocol import ( RequestType, @@ -249,6 +253,21 @@ def __init__( # EventBus for observability self._event_bus = get_event_bus() + # Route finish_write / finish_read_prefetched through a C++ host + # callback so the driver thread doesn't acquire the GIL. + self._device_host_func_dispatcher = DeviceHostFuncDispatcher() + self._device_host_func_dispatcher.register( + "finish_write", + self.storage_manager.finish_write, + payload_type=list[ObjectKey], + ) + self._device_host_func_dispatcher.register( + "finish_read_prefetched", + self.storage_manager.finish_read_prefetched, + payload_type=list[ObjectKey], + ) + self._device_host_func_dispatcher.start() + # Prefetch job tracking for two-phase lookup, keyed by request_id. # TODO: implement periodic cleanup of stale _prefetch_jobs entries # for crash resilience (e.g., client calls lookup but never queries) @@ -679,8 +698,9 @@ def store( finally: event.record() if reserved_dict: - gpu_context.cupy_stream.launch_host_func( - self.storage_manager.finish_write, + submit_callback_to_stream( + gpu_context.cupy_stream, + "finish_write", list(reserved_dict.keys()), ) # All reserved MemoryObjs share one layout_desc, so per-object @@ -889,8 +909,9 @@ def _retrieve_loop(keys: list[ObjectKey], memory_objs: list[MemoryObj]) -> None: finally: event.record() if retrieve_succeeded: - gpu_context.cupy_stream.launch_host_func( - self.storage_manager.finish_read_prefetched, + submit_callback_to_stream( + gpu_context.cupy_stream, + "finish_read_prefetched", prefetched_keys, ) self._event_bus.publish_on_stream( @@ -1343,6 +1364,10 @@ def close(self) -> None: """ Closes the MPCacheEngine and releases all resources. """ + # Stop the drain thread before storage_manager.close() so any + # in-flight completions reach a live storage manager. + self._device_host_func_dispatcher.stop() + # Close storage manager self.storage_manager.close() logger.info("MPCacheEngine closed") diff --git a/setup.py b/setup.py index 0b2dbab0b32..3dde30b3f79 100644 --- a/setup.py +++ b/setup.py @@ -236,6 +236,7 @@ def cuda_extension() -> tuple[list, dict]: "csrc/mem_alloc.cpp", "csrc/utils.cpp", "csrc/event_recorder.cpp", + "csrc/completion_recorder.cpp", ] ext_modules = [ cpp_extension.CUDAExtension( @@ -268,6 +269,7 @@ def rocm_extension() -> tuple[list, dict]: "csrc/mem_alloc_hip.cpp", "csrc/utils_hip.cpp", "csrc/event_recorder.cpp", + "csrc/completion_recorder.cpp", ] # For HIP, we generally use CppExtension and let hipcc handle things. # Ensure CXX environment variable is set to hipcc when running this build. diff --git a/tests/v1/multiprocess/test_completion_recorder.py b/tests/v1/multiprocess/test_completion_recorder.py new file mode 100644 index 00000000000..9a29740d0e9 --- /dev/null +++ b/tests/v1/multiprocess/test_completion_recorder.py @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the C++ CompletionRecorder and DeviceHostFuncDispatcher.""" + +# Standard +import threading +import time + +# Third Party +import msgspec +import pytest + +# First Party +from lmcache.v1.multiprocess.native_completion import ( + DeviceHostFuncDispatcher, + submit_callback_to_stream, +) + +try: + # Third Party + import torch # noqa: F401 + + _has_cuda = torch.cuda.is_available() +except ImportError: + _has_cuda = False + +try: + # First Party + import lmcache.c_ops as lmc_ops + + _has_native_op = hasattr(lmc_ops, "record_completion_on_stream") +except ImportError: + lmc_ops = None + _has_native_op = False + +native_only = pytest.mark.skipif( + not (_has_cuda and _has_native_op), + reason="requires CUDA and native record_completion_on_stream", +) + +if _has_cuda and _has_native_op: + # Third Party + import cupy # noqa: E402 + + +@pytest.fixture() +def stream(): + s = cupy.cuda.Stream() + yield s + s.synchronize() + + +@pytest.fixture() +def dispatcher(): + d = DeviceHostFuncDispatcher(drain_interval_seconds=0.001) + d.start() + yield d + d.stop() + + +@native_only +class TestRecordAndDrain: + """Low-level tests on lmc_ops.record_completion_on_stream / drain.""" + + def test_drain_empty(self): + completions = lmc_ops.drain_recorded_completions() + assert completions == [] + + def test_single_completion_payload_is_bytes(self, stream): + # drain must hand back py::bytes, not utf-8-decoded str — msgpack + # output is arbitrary bytes including invalid utf-8. + encoded = msgspec.msgpack.encode([b"key-1", b"key-2"]) + lmc_ops.record_completion_on_stream(stream.ptr, "finish_write", encoded) + stream.synchronize() + + completions = lmc_ops.drain_recorded_completions() + assert len(completions) == 1 + kind, payload = completions[0] + assert kind == "finish_write" + assert isinstance(payload, bytes) + assert msgspec.msgpack.decode(payload, type=list[bytes]) == [ + b"key-1", + b"key-2", + ] + + def test_many_completions_in_order(self, stream): + for i in range(50): + lmc_ops.record_completion_on_stream( + stream.ptr, "finish_write", msgspec.msgpack.encode(i) + ) + stream.synchronize() + + completions = lmc_ops.drain_recorded_completions() + assert len(completions) == 50 + for idx, (kind, payload) in enumerate(completions): + assert kind == "finish_write" + assert msgspec.msgpack.decode(payload, type=int) == idx + + +@native_only +class TestDispatcher: + """Integration tests for DeviceHostFuncDispatcher (drain + dispatch).""" + + def test_dispatch_to_registered_handler(self, dispatcher, stream): + seen: list[list[bytes]] = [] + dispatcher.register("finish_write", seen.append, payload_type=list[bytes]) + + submit_callback_to_stream(stream, "finish_write", [b"k0", b"k1", b"k2"]) + stream.synchronize() + + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and not seen: + time.sleep(0.01) + assert seen == [[b"k0", b"k1", b"k2"]] + + def test_unknown_kind_drops_payload(self, dispatcher, stream): + submit_callback_to_stream(stream, "finish_unknown", [b"x"]) + stream.synchronize() + time.sleep(0.1) + assert dispatcher.dispatched_count() == 0 + + def test_handler_exception_does_not_kill_thread(self, dispatcher, stream): + calls: list[list[bytes]] = [] + + def handler(payload): + calls.append(payload) + if len(calls) == 1: + raise RuntimeError("boom") + + dispatcher.register("finish_write", handler, payload_type=list[bytes]) + submit_callback_to_stream(stream, "finish_write", [b"a"]) + submit_callback_to_stream(stream, "finish_write", [b"b"]) + stream.synchronize() + + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and len(calls) < 2: + time.sleep(0.01) + assert calls == [[b"a"], [b"b"]] + assert dispatcher.handler_exception_counts().get("finish_write", 0) == 1 + + def test_tuple_payload_for_multiple_arguments(self, dispatcher, stream): + # Multi-arg handlers: wrap args in a tuple, register tuple decode type. + seen: list[tuple[int, str]] = [] + dispatcher.register("finish_multi", seen.append, payload_type=tuple[int, str]) + + submit_callback_to_stream(stream, "finish_multi", (42, "hello")) + stream.synchronize() + + deadline = time.monotonic() + 5.0 + while time.monotonic() < deadline and not seen: + time.sleep(0.01) + assert seen == [(42, "hello")] + + +@native_only +class TestDeadlockRegression: + """Regression for the GIL deadlock: the legacy + ``stream.launch_host_func(python_fn, ...)`` path stalls; the C++ + callback runs without the GIL and must finish within the timeout.""" + + def test_many_concurrent_records_no_deadlock(self, dispatcher, stream): + received: list[int] = [] + ready = threading.Event() + + def handler(payload): + received.append(payload) + if len(received) >= 200: + ready.set() + + dispatcher.register("finish_write", handler, payload_type=int) + + for i in range(200): + submit_callback_to_stream(stream, "finish_write", i) + stream.synchronize() + + assert ready.wait(timeout=10.0), ( + f"deadlock or drop: only {len(received)} of 200 dispatched" + ) + assert len(received) == 200 From a0574afc3b76cf2fa214c60172435e5ed9cdac78 Mon Sep 17 00:00:00 2001 From: GentleCold Date: Tue, 26 May 2026 11:08:15 +0800 Subject: [PATCH 58/69] [BugFix] fix sha256_cbor + async_loading type mismatch in _hash_tokens (#2999) * [BugFix] fix sha256_cbor + async_loading type mismatch in _hash_tokens When LMCACHE_PRE_CACHING_HASH_ALGORITHM=sha256_cbor is used together with LMCACHE_ENABLE_ASYNC_LOADING=True, the async lookup path fails because sha256_cbor (vLLM PR#23673) returns a 32-byte bytes digest. This digest is placed into LookupRequestMsg.hashes which msgpack cannot encode as int, raising OverflowError. Fix: after calling hash_func in _hash_tokens, if the result is bytes, fold the first 8 bytes into a uint64 int. This keeps all downstream code (CacheEngineKey, msgpack serialisation) working with plain ints regardless of the configured hash algorithm. Both the store path and the async lookup path go through _hash_tokens, so the keys produced are always consistent. Fixes #2997 Signed-off-by: GentleCold * [BugFix] address review: fix NONE_HASH type and use public test interface - token_database.py: convert NONE_HASH from bytes to int immediately after it is set from vLLM. This ensures the entire prefix-hash chain uses a consistent int type as canon_prefix input to hash_func, not bytes on the first chunk and int on subsequent ones. - test_token_database.py: replace direct call to private _hash_tokens with the public process_tokens interface (make_key=False). Drop cbor2 import in favour of a simpler hashlib stub. Signed-off-by: GentleCold * [BugFix] normalize byte hash outputs in token database - token_database.py: add a module-level helper to normalize hash outputs from vLLM hash functions into LMCache's int chunk-hash representation. - Reuse the helper for both vLLM NONE_HASH initialization and token chunk hashing so sha256_cbor byte digests are folded consistently before entering the prefix hash chain. Signed-off-by: GentleCold * [CI] Fix isort in token database test Signed-off-by: GentleCold --------- Signed-off-by: GentleCold --- lmcache/v1/token_database.py | 32 ++++++++++++++++++++++++++++++-- tests/v1/test_token_database.py | 23 +++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/lmcache/v1/token_database.py b/lmcache/v1/token_database.py index 652ca1deabe..67d65636fb1 100644 --- a/lmcache/v1/token_database.py +++ b/lmcache/v1/token_database.py @@ -30,6 +30,32 @@ NONE_HASH = 0 + +def _normalize_hash_to_int(hash_value: Union[int, bytes]) -> int: + """Normalize hash outputs to LMCache's int chunk-hash representation. + + This function is triggered when vLLM's ``sha256_cbor`` hash function is + used because it returns a 32-byte digest. vLLM's + ``kv_cache_utils.init_none_hash`` can therefore initialize ``NONE_HASH`` as + bytes, and direct hash calls for token chunks can also return bytes. + + LMCache stores chunk hashes in ``CacheEngineKey`` and serializes them with + msgpack, so byte digests must be folded into uint64-compatible ints before + they enter the prefix hash chain. This also keeps ``NONE_HASH`` and later + prefix hashes using the same structural type for CBOR hashing. + + Args: + hash_value: Hash output from vLLM or Python's builtin hash. + + Returns: + The original int hash value, or the first eight bytes of a digest as a + big-endian int. + """ + if isinstance(hash_value, bytes): + return int.from_bytes(hash_value[:8], "big") + return hash_value + + # Type alias for process_tokens return value # (start_index, end_index, cache_engine_key｜hash) ProcessTokensResult = Tuple[int, int, Union[CacheEngineKey, int]] @@ -70,7 +96,7 @@ def __init__( if hasattr(kv_cache_utils, "init_none_hash"): kv_cache_utils.init_none_hash(self.hash_func) - NONE_HASH = kv_cache_utils.NONE_HASH + NONE_HASH = _normalize_hash_to_int(kv_cache_utils.NONE_HASH) logger.info( f"Initialized NONE_HASH={NONE_HASH} from vLLM (>= PR#20511)" ) @@ -263,7 +289,9 @@ def _hash_tokens( prefix_hash, tokens_tuple, extra_keys ) - return self.hash_func((canon_prefix, canon_tokens, canon_extra)) + return _normalize_hash_to_int( + self.hash_func((canon_prefix, canon_tokens, canon_extra)) + ) class ChunkedTokenDatabase(TokenDatabase): diff --git a/tests/v1/test_token_database.py b/tests/v1/test_token_database.py index eb9181df7b8..33abfb38924 100644 --- a/tests/v1/test_token_database.py +++ b/tests/v1/test_token_database.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # Standard +import hashlib import os # Third Party @@ -132,3 +133,25 @@ def test_segment_token_database(prefix_length, chunk_lengths): assert key.chunk_hash == hashes[i] # print(st, starts[i]) # print(ed, ends[i]) + + +def test_process_tokens_returns_int_keys_for_bytes_hash_func() -> None: + """process_tokens must produce int chunk_hash keys even when the underlying + hash function returns bytes (e.g. sha256_cbor). This ensures downstream + code such as msgpack serialisation always receives plain ints. + """ + cfg = LMCacheEngineConfig.from_legacy(chunk_size=16, backend="cpu") + metadata = dumb_metadata() + db = ChunkedTokenDatabase(cfg, metadata) + + # Stub that returns bytes, mimicking sha256_cbor without requiring cbor2. + db.hash_func = lambda x: hashlib.sha256(str(x).encode()).digest() + + tokens = generate_tokens(32, "cpu") + results = list(db.process_tokens(tokens=tokens, make_key=False)) + + assert len(results) > 0 + for _, _, hash_val in results: + assert isinstance(hash_val, int), f"Expected int, got {type(hash_val)}" + # Must fit in uint64 (msgpack range: 0 to 2**64 - 1) + assert 0 <= hash_val <= 2**64 - 1 From a534b92750c6f015c38047651f3953635622f2df Mon Sep 17 00:00:00 2001 From: Yihua Cheng Date: Mon, 25 May 2026 22:15:27 -0500 Subject: [PATCH 59/69] [MP] Update the L2 adapter interface to force measuring the real transferred bytes (#3363) Signed-off-by: ApostaC --- docs/design/v1/mp_observability/EVENTS.md | 2 +- .../src/lmc_external_l2_adapter/adapter.py | 7 ++-- .../tests/test_plugin.py | 15 ++++--- .../commands/bench/l2_adapter_bench/runner.py | 9 +++-- .../v1/check/check_mode_test_l2_adapter.py | 4 +- lmcache/v1/distributed/internal_api.py | 30 ++++++++++++++ lmcache/v1/distributed/l2_adapters/base.py | 37 ++++------------- .../distributed/l2_adapters/dax_l2_adapter.py | 12 ++++-- .../distributed/l2_adapters/fs_l2_adapter.py | 26 +++++------- .../l2_adapters/mock_l2_adapter.py | 23 ++++------- .../native_connector_l2_adapter.py | 9 +++-- .../nixl_store_dynamic_l2_adapter.py | 11 +++-- .../l2_adapters/nixl_store_l2_adapter.py | 36 +++++------------ .../l2_adapters/raw_block_l2_adapter.py | 11 +++-- .../distributed/l2_adapters/s3_l2_adapter.py | 12 ++++-- .../distributed/l2_adapters/serde_wrapper.py | 36 +++++------------ .../storage_controllers/store_controller.py | 25 +++--------- .../subscribers/metrics/l2_throughput.py | 40 +++++++++---------- tests/v1/distributed/test_dax_l2_adapter.py | 8 ++-- tests/v1/distributed/test_l2_adapter_base.py | 5 ++- tests/v1/distributed/test_mock_l2_adapter.py | 10 ++--- .../test_mooncake_store_l2_adapter.py | 14 +++---- .../test_native_connector_l2_adapter.py | 10 ++--- .../test_nixl_store_dynamic_l2_adapter.py | 4 +- .../distributed/test_nixl_store_l2_adapter.py | 12 +++--- .../distributed/test_raw_block_l2_adapter.py | 4 +- .../test_resp_l2_adapter_integration.py | 6 +-- tests/v1/distributed/test_s3_l2_adapter.py | 4 +- 28 files changed, 202 insertions(+), 220 deletions(-) diff --git a/docs/design/v1/mp_observability/EVENTS.md b/docs/design/v1/mp_observability/EVENTS.md index 90bbae58b87..04f592d1f55 100644 --- a/docs/design/v1/mp_observability/EVENTS.md +++ b/docs/design/v1/mp_observability/EVENTS.md @@ -71,7 +71,7 @@ Producers: | EventType | Metadata keys | Types | |---|---|---| | `L2_STORE_SUBMITTED` | `adapter_index`, `task_id`, `l2_name`, `key_count`, `total_bytes` | `int`, `int`, `str`, `int`, `int` | -| `L2_STORE_COMPLETED` | `adapter_index`, `task_id`, `l2_name`, `succeeded_count`, `failed_count` | `int`, `int`, `str`, `int`, `int` | +| `L2_STORE_COMPLETED` | `adapter_index`, `task_id`, `l2_name`, `bytes_transferred`, `succeeded_count`, `failed_count` | `int`, `int`, `str`, `int`, `int`, `int` | --- diff --git a/examples/lmc_external_l2_adapter/src/lmc_external_l2_adapter/adapter.py b/examples/lmc_external_l2_adapter/src/lmc_external_l2_adapter/adapter.py index a1d8a5e4738..766d8db81f5 100644 --- a/examples/lmc_external_l2_adapter/src/lmc_external_l2_adapter/adapter.py +++ b/examples/lmc_external_l2_adapter/src/lmc_external_l2_adapter/adapter.py @@ -22,6 +22,7 @@ from lmcache.logging import init_logger from lmcache.native_storage_ops import Bitmap from lmcache.v1.distributed.api import ObjectKey +from lmcache.v1.distributed.internal_api import L2StoreResult from lmcache.v1.distributed.l2_adapters.base import ( L2AdapterInterface, L2TaskId, @@ -128,7 +129,7 @@ def __init__( self._locked: dict[ObjectKey, int] = defaultdict(int) self._next_id: L2TaskId = 0 - self._done_store: dict[L2TaskId, bool] = {} + self._done_store: dict[L2TaskId, L2StoreResult] = {} self._done_lookup: dict[L2TaskId, Bitmap] = {} self._done_load: dict[L2TaskId, Bitmap] = {} self._lock = threading.Lock() @@ -171,7 +172,7 @@ def submit_store_task( def pop_completed_store_tasks( self, - ) -> dict[L2TaskId, bool]: + ) -> dict[L2TaskId, L2StoreResult]: with self._lock: done = self._done_store self._done_store = {} @@ -300,7 +301,7 @@ async def _do_store( await asyncio.sleep(delay) with self._lock: - self._done_store[tid] = ok + self._done_store[tid] = L2StoreResult(ok, total) self._store_efd.notify() def _do_lookup( diff --git a/examples/lmc_external_l2_adapter/tests/test_plugin.py b/examples/lmc_external_l2_adapter/tests/test_plugin.py index d1f376df2d5..7164af86eca 100644 --- a/examples/lmc_external_l2_adapter/tests/test_plugin.py +++ b/examples/lmc_external_l2_adapter/tests/test_plugin.py @@ -289,7 +289,8 @@ def test_store_and_lookup(self, adapter): tid = adapter.submit_store_task([key], [obj]) assert _wait_event_fd(store_fd) done = adapter.pop_completed_store_tasks() - assert done.get(tid) is True + assert done.get(tid) is not None + assert done[tid].is_successful() ltid = adapter.submit_lookup_and_lock_task([key]) assert _wait_event_fd(lookup_fd) @@ -311,7 +312,8 @@ def test_store_and_load(self, adapter): tid = adapter.submit_store_task([key], [obj]) assert _wait_event_fd(store_fd) done = adapter.pop_completed_store_tasks() - assert done.get(tid) is True + assert done.get(tid) is not None + assert done[tid].is_successful() dst = torch.zeros_like(src) load_obj = _make_tensor_obj(dst) @@ -334,7 +336,8 @@ def test_batch_store_and_lookup(self, adapter): tid = adapter.submit_store_task(keys, objs) assert _wait_event_fd(store_fd) done = adapter.pop_completed_store_tasks() - assert done.get(tid) is True + assert done.get(tid) is not None + assert done[tid].is_successful() ltid = adapter.submit_lookup_and_lock_task(keys) assert _wait_event_fd(lookup_fd) @@ -387,13 +390,15 @@ def test_fifo_eviction(self, adapter): tid = adapter.submit_store_task([k], [_create_obj(OBJ_SIZE)]) assert _wait_event_fd(store_fd) done = adapter.pop_completed_store_tasks() - assert done.get(tid) is True + assert done.get(tid) is not None + assert done[tid].is_successful() # Store k3 -- should evict k1 tid = adapter.submit_store_task([k3], [_create_obj(OBJ_SIZE)]) assert _wait_event_fd(store_fd) done = adapter.pop_completed_store_tasks() - assert done.get(tid) is True + assert done.get(tid) is not None + assert done[tid].is_successful() # Lookup all three ltid = adapter.submit_lookup_and_lock_task([k1, k2, k3]) diff --git a/lmcache/cli/commands/bench/l2_adapter_bench/runner.py b/lmcache/cli/commands/bench/l2_adapter_bench/runner.py index 378724182c4..b79e4359405 100644 --- a/lmcache/cli/commands/bench/l2_adapter_bench/runner.py +++ b/lmcache/cli/commands/bench/l2_adapter_bench/runner.py @@ -21,6 +21,7 @@ # First Party from lmcache.native_storage_ops import Bitmap from lmcache.v1.distributed.api import ObjectKey +from lmcache.v1.distributed.internal_api import L2StoreResult from lmcache.v1.memory_management import MemoryObj # Local @@ -46,10 +47,10 @@ def _bitmap_count(bitmap: Bitmap | None) -> int: def _wait_store_finished( adapter, task_ids: list[int], timeout: float -) -> dict[int, bool]: +) -> dict[int, L2StoreResult]: """Wait for all store tasks to finish. - Returns the accumulated ``{task_id: success}`` dict. + Returns the accumulated ``{task_id: L2StoreResult}`` dict. ``pop_completed_store_tasks`` consumes the adapter's completion dict, so we must accumulate the results here for the caller to use. On timeout, returns whatever was harvested so far (possibly @@ -58,7 +59,7 @@ def _wait_store_finished( """ unfinished = len(task_ids) efd = adapter.get_store_event_fd() - completed: dict[int, bool] = {} + completed: dict[int, L2StoreResult] = {} while unfinished > 0: if not wait_eventfd(efd, timeout=timeout): return completed @@ -168,7 +169,7 @@ def bench_store( success_keys = sum( len(keys_batches[i]) for i, tid in enumerate(task_ids) - if completed.get(tid, False) + if completed.get(tid, L2StoreResult(False, 0)).is_successful() ) if timed_out: diff --git a/lmcache/v1/check/check_mode_test_l2_adapter.py b/lmcache/v1/check/check_mode_test_l2_adapter.py index 379fcb57d91..f632e2111db 100644 --- a/lmcache/v1/check/check_mode_test_l2_adapter.py +++ b/lmcache/v1/check/check_mode_test_l2_adapter.py @@ -18,7 +18,7 @@ print_performance_results, ) from lmcache.v1.distributed.api import ObjectKey -from lmcache.v1.distributed.internal_api import L1MemoryDesc +from lmcache.v1.distributed.internal_api import L1MemoryDesc, L2StoreResult from lmcache.v1.distributed.l2_adapters import create_l2_adapter from lmcache.v1.distributed.l2_adapters.config import ( parse_args_to_l2_adapters_config, @@ -108,7 +108,7 @@ def _run_store_phase(adapter, keys, objects): return None, False completed = adapter.pop_completed_store_tasks() elapsed_ms = (time.perf_counter() - start) * 1000 - ok = completed.get(task_id, False) + ok = completed.get(task_id, L2StoreResult(False, 0)).is_successful() return elapsed_ms, ok diff --git a/lmcache/v1/distributed/internal_api.py b/lmcache/v1/distributed/internal_api.py index dcc660bfd98..ad76ab206d2 100644 --- a/lmcache/v1/distributed/internal_api.py +++ b/lmcache/v1/distributed/internal_api.py @@ -170,6 +170,36 @@ class EvictionAction: """The key of the object to be evicted""" +class L2StoreResult(int): + """Immutable result of a completed L2 store task. + + Encodes both the success flag and bytes transferred in the int + value: ``>= 0`` means success (value = bytes transferred); + ``-1`` means failure. + + Args: + success: Whether the store task succeeded. + bytes_transferred: Bytes actually written to L2. Must be >= 0. + + Raises: + ValueError: If ``bytes_transferred`` is negative. + """ + + def __new__(cls, success: bool, bytes_transferred: int) -> "L2StoreResult": + if bytes_transferred < 0: + raise ValueError(f"bytes_transferred must be >= 0, got {bytes_transferred}") + return super().__new__(cls, bytes_transferred if success else -1) + + def is_successful(self) -> bool: + """Return ``True`` when the store task succeeded.""" + return int(self) >= 0 + + def bytes_transferred(self) -> int: + """Return the number of bytes actually written, or 0 on failure.""" + value = int(self) + return value if value >= 0 else 0 + + @dataclass(frozen=True) class QuotaEntry: """Snapshot of a single quota registration.""" diff --git a/lmcache/v1/distributed/l2_adapters/base.py b/lmcache/v1/distributed/l2_adapters/base.py index a8963a6da7b..a972efe0438 100644 --- a/lmcache/v1/distributed/l2_adapters/base.py +++ b/lmcache/v1/distributed/l2_adapters/base.py @@ -20,7 +20,7 @@ # First Party from lmcache.logging import init_logger from lmcache.v1.distributed.api import ObjectKey -from lmcache.v1.distributed.internal_api import L2AdapterListener +from lmcache.v1.distributed.internal_api import L2AdapterListener, L2StoreResult from lmcache.v1.memory_management import MemoryObj logger = init_logger(__name__) @@ -219,39 +219,18 @@ def submit_store_task( pass @abstractmethod - def pop_completed_store_tasks(self) -> dict[L2TaskId, bool]: - """ - Pop all the completed store tasks with a flag indicating - whether the task is successful or not. + def pop_completed_store_tasks(self) -> dict[L2TaskId, L2StoreResult]: + """Pop all completed store tasks. Returns: - dict[L2TaskId, bool]: a dictionary mapping the task id to a boolean flag - indicating whether the task is successful or not. True means - successful, and False means failed. + dict[L2TaskId, L2StoreResult]: a dictionary mapping the task + id to an ``L2StoreResult`` that encodes both the success flag + and the bytes actually transferred. Use + ``result.is_successful()`` and ``result.bytes_transferred()`` + to inspect the outcome. """ pass - def pop_completed_store_task_bytes(self) -> dict[L2TaskId, int]: - """Report bytes actually transferred per completed store task. - - Optional. Default returns ``{}``, which leaves the L2 throughput - subscriber to fall back on submitted-bytes accounting. - - Adapters that fast-path duplicate keys (e.g. skip the write when - the key already exists in the backend) SHOULD override this so - the throughput histogram reflects real work, not skipped no-ops. - A task with all keys fast-pathed reports ``0`` here, which the - subscriber treats as "no useful sample" and skips. - - Returns: - dict[L2TaskId, int]: task id -> bytes actually written. Keys - present here MUST also appear in the next - ``pop_completed_store_tasks`` call (they share the same - per-task completion record), and the returned dict should be - cleared on read, mirroring ``pop_completed_store_tasks``. - """ - return {} - ##################### # Lookup and Lock Interface ##################### diff --git a/lmcache/v1/distributed/l2_adapters/dax_l2_adapter.py b/lmcache/v1/distributed/l2_adapters/dax_l2_adapter.py index 35e1a2edf70..81c0f43fece 100644 --- a/lmcache/v1/distributed/l2_adapters/dax_l2_adapter.py +++ b/lmcache/v1/distributed/l2_adapters/dax_l2_adapter.py @@ -17,6 +17,7 @@ from lmcache.logging import init_logger from lmcache.native_storage_ops import Bitmap from lmcache.v1.distributed.api import ObjectKey +from lmcache.v1.distributed.internal_api import L2StoreResult from lmcache.v1.distributed.l2_adapters.base import ( AdapterUsage, L2AdapterInterface, @@ -175,7 +176,7 @@ def __init__(self, config: DaxL2AdapterConfig) -> None: self._load_executor: Optional[ThreadPoolExecutor] = None self._next_task_id: L2TaskId = 0 - self._completed_store_tasks: dict[L2TaskId, bool] = {} + self._completed_store_tasks: dict[L2TaskId, L2StoreResult] = {} self._completed_lookup_tasks: dict[L2TaskId, Bitmap] = {} self._completed_load_tasks: dict[L2TaskId, Bitmap] = {} @@ -265,11 +266,11 @@ def submit_store_task( ) return task_id - def pop_completed_store_tasks(self) -> dict[L2TaskId, bool]: + def pop_completed_store_tasks(self) -> dict[L2TaskId, L2StoreResult]: """Drain completed store task results. Returns: - Mapping from task id to task-level success. Each task appears at + Mapping from task id to store result. Each task appears at most once; subsequent calls do not return already drained tasks. """ with self._lock: @@ -512,8 +513,11 @@ def _execute_store_task( if ok: stored_keys.append(key) finally: + bytes_transferred = self._config.slot_bytes * len(stored_keys) with self._lock: - self._completed_store_tasks[task_id] = all(per_key_results) + self._completed_store_tasks[task_id] = L2StoreResult( + all(per_key_results), bytes_transferred + ) self._inflight_store_tasks -= 1 if stored_keys: diff --git a/lmcache/v1/distributed/l2_adapters/fs_l2_adapter.py b/lmcache/v1/distributed/l2_adapters/fs_l2_adapter.py index 346ded986e9..6ba7a1e8402 100644 --- a/lmcache/v1/distributed/l2_adapters/fs_l2_adapter.py +++ b/lmcache/v1/distributed/l2_adapters/fs_l2_adapter.py @@ -30,6 +30,7 @@ from lmcache.logging import init_logger from lmcache.native_storage_ops import Bitmap from lmcache.v1.distributed.api import ObjectKey +from lmcache.v1.distributed.internal_api import L2StoreResult from lmcache.v1.distributed.l2_adapters.base import ( L2AdapterInterface, L2TaskId, @@ -286,12 +287,7 @@ def __init__(self, config: FSL2AdapterConfig): # Task bookkeeping self._next_task_id: L2TaskId = 0 - self._completed_store_tasks: dict[L2TaskId, bool] = {} - # Bytes actually written per completed store task. Excludes - # duplicate-key fast-paths (see ``_execute_store``). Reported to - # the L2 throughput subscriber via ``pop_completed_store_task_bytes`` - # so the histogram reflects real disk I/O, not skipped no-ops. - self._completed_store_task_bytes: dict[L2TaskId, int] = {} + self._completed_store_tasks: dict[L2TaskId, L2StoreResult] = {} self._completed_lookup_tasks: dict[L2TaskId, Bitmap] = {} self._completed_load_tasks: dict[L2TaskId, Bitmap] = {} self._lock = threading.Lock() @@ -344,18 +340,19 @@ def submit_store_task( def pop_completed_store_tasks( self, - ) -> dict[L2TaskId, bool]: + ) -> dict[L2TaskId, L2StoreResult]: + """Pop all completed store tasks. + + Returns: + dict[L2TaskId, L2StoreResult]: a dictionary mapping the task + id to an ``L2StoreResult`` that encodes both the success flag + and the bytes actually transferred. + """ with self._lock: completed = self._completed_store_tasks self._completed_store_tasks = {} return completed - def pop_completed_store_task_bytes(self) -> dict[L2TaskId, int]: - with self._lock: - completed_bytes = self._completed_store_task_bytes - self._completed_store_task_bytes = {} - return completed_bytes - # ------------------------------------------------------------------ # Lookup and Lock Interface # ------------------------------------------------------------------ @@ -640,8 +637,7 @@ async def _execute_store( success = False with self._lock: - self._completed_store_tasks[task_id] = success - self._completed_store_task_bytes[task_id] = bytes_written + self._completed_store_tasks[task_id] = L2StoreResult(success, bytes_written) self._store_efd.notify() # ---- lookup --------------------------------------------------------- diff --git a/lmcache/v1/distributed/l2_adapters/mock_l2_adapter.py b/lmcache/v1/distributed/l2_adapters/mock_l2_adapter.py index bde0f3c41f9..afc611dee94 100644 --- a/lmcache/v1/distributed/l2_adapters/mock_l2_adapter.py +++ b/lmcache/v1/distributed/l2_adapters/mock_l2_adapter.py @@ -20,6 +20,7 @@ from lmcache.logging import init_logger from lmcache.native_storage_ops import Bitmap from lmcache.v1.distributed.api import ObjectKey +from lmcache.v1.distributed.internal_api import L2StoreResult from lmcache.v1.distributed.l2_adapters.base import L2AdapterInterface, L2TaskId from lmcache.v1.distributed.l2_adapters.config import ( L2AdapterConfigBase, @@ -124,8 +125,7 @@ def __init__(self, config: MockL2AdapterConfig): # Task ID management self._next_task_id: L2TaskId = 0 - self._completed_store_tasks: dict[L2TaskId, bool] = {} - self._completed_store_task_bytes: dict[L2TaskId, int] = {} + self._completed_store_tasks: dict[L2TaskId, L2StoreResult] = {} self._completed_lookup_tasks: dict[L2TaskId, Bitmap] = {} self._completed_load_tasks: dict[L2TaskId, Bitmap] = {} self._lock = threading.Lock() # lock for all shared state @@ -182,27 +182,19 @@ def submit_store_task( return task_id - def pop_completed_store_tasks(self) -> dict[L2TaskId, bool]: + def pop_completed_store_tasks(self) -> dict[L2TaskId, L2StoreResult]: """ - Pop all the completed store tasks with a flag indicating - whether the task is successful or not. + Pop all the completed store tasks with their results. Returns: - dict[L2TaskId, bool]: a dictionary mapping the task id to a boolean flag - indicating whether the task is successful or not. True means - successful, and False means failed. + dict[L2TaskId, L2StoreResult]: a dictionary mapping the task id to + an L2StoreResult encoding success/failure and bytes transferred. """ with self._lock: completed = self._completed_store_tasks self._completed_store_tasks = {} return completed - def pop_completed_store_task_bytes(self) -> dict[L2TaskId, int]: - with self._lock: - completed_bytes = self._completed_store_task_bytes - self._completed_store_task_bytes = {} - return completed_bytes - def submit_lookup_and_lock_task(self, keys: list[ObjectKey]) -> L2TaskId: with self._lock: task_id = self._get_next_task_id() @@ -440,12 +432,11 @@ async def _execute_store_in_the_loop( # Schedule completion coroutine on the event loop await asyncio.sleep(delay_seconds) with self._lock: - self._completed_store_tasks[task_id] = success # ``total_bytes`` counts only objects actually written into # ``self._memory_objects`` — duplicates and capacity-skipped # keys are excluded. Reporting this lets the L2 throughput # subscriber distinguish real I/O from fast-pathed no-ops. - self._completed_store_task_bytes[task_id] = total_bytes + self._completed_store_tasks[task_id] = L2StoreResult(success, total_bytes) if stored_keys: self._notify_keys_stored(stored_keys, stored_sizes) diff --git a/lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py b/lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py index c6c4ad75d5b..8464e485a31 100644 --- a/lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py +++ b/lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py @@ -30,6 +30,7 @@ from lmcache.logging import init_logger from lmcache.native_storage_ops import Bitmap from lmcache.v1.distributed.api import ObjectKey +from lmcache.v1.distributed.internal_api import L2StoreResult from lmcache.v1.distributed.l2_adapters.base import ( L2AdapterInterface, L2TaskId, @@ -133,7 +134,7 @@ def __init__( ] = {} # Completed results (same pattern as MockL2Adapter) - self._completed_stores: dict[L2TaskId, bool] = {} + self._completed_stores: dict[L2TaskId, L2StoreResult] = {} self._completed_lookups: dict[L2TaskId, Bitmap] = {} self._completed_loads: dict[L2TaskId, Bitmap] = {} @@ -215,7 +216,7 @@ def submit_store_task( def pop_completed_store_tasks( self, - ) -> dict[L2TaskId, bool]: + ) -> dict[L2TaskId, L2StoreResult]: with self._lock: completed = self._completed_stores self._completed_stores = {} @@ -451,8 +452,8 @@ def _demux_loop(self) -> None: ) = entry if op_type == self._OP_STORE: - self._completed_stores[task_id] = ok store_info = self._pending_store_sizes.pop(fid, None) + task_bytes = 0 if ok and store_info is not None: store_keys, sizes = store_info for key, size in zip(store_keys, sizes, strict=True): @@ -467,9 +468,11 @@ def _demux_loop(self) -> None: self._key_sizes[key] = size keys_stored.append(key) sizes_stored.append(size) + task_bytes += size else: keys_stored.append(key) sizes_stored.append(0) + self._completed_stores[task_id] = L2StoreResult(ok, task_bytes) self._store_efd.notify() elif op_type == self._OP_LOOKUP: diff --git a/lmcache/v1/distributed/l2_adapters/nixl_store_dynamic_l2_adapter.py b/lmcache/v1/distributed/l2_adapters/nixl_store_dynamic_l2_adapter.py index a26d85081eb..9193918b766 100644 --- a/lmcache/v1/distributed/l2_adapters/nixl_store_dynamic_l2_adapter.py +++ b/lmcache/v1/distributed/l2_adapters/nixl_store_dynamic_l2_adapter.py @@ -41,7 +41,7 @@ from lmcache.logging import init_logger from lmcache.native_storage_ops import Bitmap from lmcache.v1.distributed.api import MemoryLayoutDesc, ObjectKey -from lmcache.v1.distributed.internal_api import L1MemoryDesc +from lmcache.v1.distributed.internal_api import L1MemoryDesc, L2StoreResult from lmcache.v1.distributed.l2_adapters.base import L2AdapterInterface, L2TaskId from lmcache.v1.distributed.l2_adapters.config import ( L2AdapterConfigBase, @@ -358,7 +358,7 @@ def __init__( # Task ID management self._next_task_id: L2TaskId = 0 - self._completed_store_tasks: dict[L2TaskId, bool] = {} + self._completed_store_tasks: dict[L2TaskId, L2StoreResult] = {} self._completed_lookup_tasks: dict[L2TaskId, Bitmap] = {} self._completed_load_tasks: dict[L2TaskId, Bitmap] = {} self._lock = threading.Lock() @@ -408,7 +408,7 @@ def submit_store_task( ) return task_id - def pop_completed_store_tasks(self) -> dict[L2TaskId, bool]: + def pop_completed_store_tasks(self) -> dict[L2TaskId, L2StoreResult]: with self._lock: completed = self._completed_store_tasks self._completed_store_tasks = {} @@ -649,8 +649,11 @@ async def _execute_store_in_the_loop( if stored_keys: self._notify_keys_stored(stored_keys, stored_sizes) + bytes_transferred = sum(stored_sizes) with self._lock: - self._completed_store_tasks[task_id] = success + self._completed_store_tasks[task_id] = L2StoreResult( + success, bytes_transferred + ) self._signal_store_event() def _execute_lookup_in_the_loop( diff --git a/lmcache/v1/distributed/l2_adapters/nixl_store_l2_adapter.py b/lmcache/v1/distributed/l2_adapters/nixl_store_l2_adapter.py index 00c702b0f7d..17c783a82e7 100644 --- a/lmcache/v1/distributed/l2_adapters/nixl_store_l2_adapter.py +++ b/lmcache/v1/distributed/l2_adapters/nixl_store_l2_adapter.py @@ -24,7 +24,7 @@ from lmcache.logging import init_logger from lmcache.native_storage_ops import Bitmap from lmcache.v1.distributed.api import MemoryLayoutDesc, ObjectKey -from lmcache.v1.distributed.internal_api import L1MemoryDesc +from lmcache.v1.distributed.internal_api import L1MemoryDesc, L2StoreResult from lmcache.v1.distributed.l2_adapters.base import L2AdapterInterface, L2TaskId from lmcache.v1.distributed.l2_adapters.config import ( L2AdapterConfigBase, @@ -457,13 +457,7 @@ def __init__(self, config: NixlStoreL2AdapterConfig, l1_memory_desc: L1MemoryDes # Task ID management self._next_task_id: L2TaskId = 0 - self._completed_store_tasks: dict[L2TaskId, bool] = {} - # Bytes actually transferred per completed store task. Excludes - # keys that were fast-pathed (already present in - # ``_memory_objects``) and any pool-exhaustion fallouts. Surfaces - # to the L2 throughput subscriber so its histogram reflects real - # NIXL transfers, not skipped no-ops. - self._completed_store_task_bytes: dict[L2TaskId, int] = {} + self._completed_store_tasks: dict[L2TaskId, L2StoreResult] = {} self._completed_lookup_tasks: dict[L2TaskId, Bitmap] = {} self._completed_load_tasks: dict[L2TaskId, Bitmap] = {} self._lock = threading.Lock() # lock for all shared state @@ -517,27 +511,19 @@ def submit_store_task( return task_id - def pop_completed_store_tasks(self) -> dict[L2TaskId, bool]: - """ - Pop all the completed store tasks with a flag indicating - whether the task is successful or not. + def pop_completed_store_tasks(self) -> dict[L2TaskId, L2StoreResult]: + """Pop all completed store tasks. Returns: - dict[L2TaskId, bool]: a dictionary mapping the task id to a boolean flag - indicating whether the task is successful or not. True means - successful, and False means failed. + dict[L2TaskId, L2StoreResult]: a dictionary mapping the task + id to an ``L2StoreResult`` that encodes both the success flag + and the bytes actually transferred. """ with self._lock: completed = self._completed_store_tasks self._completed_store_tasks = {} return completed - def pop_completed_store_task_bytes(self) -> dict[L2TaskId, int]: - with self._lock: - completed_bytes = self._completed_store_task_bytes - self._completed_store_task_bytes = {} - return completed_bytes - ##################### # Lookup and Lock Interface ##################### @@ -777,8 +763,7 @@ async def _execute_store_in_the_loop( if not mem_indices_flat: # Nothing to store (all keys already existed or pool empty) with self._lock: - self._completed_store_tasks[task_id] = True - self._completed_store_task_bytes[task_id] = 0 + self._completed_store_tasks[task_id] = L2StoreResult(True, 0) self._signal_store_event() return @@ -812,8 +797,9 @@ async def _execute_store_in_the_loop( self.nixl_agent.pool.batched_free(storage_indices_flat) with self._lock: - self._completed_store_tasks[task_id] = success - self._completed_store_task_bytes[task_id] = bytes_transferred + self._completed_store_tasks[task_id] = L2StoreResult( + success, bytes_transferred + ) self._signal_store_event() diff --git a/lmcache/v1/distributed/l2_adapters/raw_block_l2_adapter.py b/lmcache/v1/distributed/l2_adapters/raw_block_l2_adapter.py index 5c7e57328a9..558df6907c4 100644 --- a/lmcache/v1/distributed/l2_adapters/raw_block_l2_adapter.py +++ b/lmcache/v1/distributed/l2_adapters/raw_block_l2_adapter.py @@ -23,6 +23,7 @@ # First Party from lmcache.logging import init_logger from lmcache.v1.distributed.api import ObjectKey +from lmcache.v1.distributed.internal_api import L2StoreResult from lmcache.v1.distributed.l2_adapters.base import ( L2AdapterInterface, L2TaskId, @@ -351,7 +352,7 @@ def __init__( self._lock = threading.Lock() self._next_task_id: L2TaskId = 0 - self._completed_store_tasks: dict[L2TaskId, bool] = {} + self._completed_store_tasks: dict[L2TaskId, L2StoreResult] = {} self._completed_lookup_tasks: dict[L2TaskId, Bitmap] = {} self._completed_load_tasks: dict[L2TaskId, Bitmap] = {} @@ -414,7 +415,7 @@ def submit_store_task( future.add_done_callback(partial(self._finish_store_task, task_id)) return task_id - def pop_completed_store_tasks(self) -> dict[L2TaskId, bool]: + def pop_completed_store_tasks(self) -> dict[L2TaskId, L2StoreResult]: """Drain and return completed store task results.""" with self._lock: completed = self._completed_store_tasks @@ -657,13 +658,17 @@ def _finish_store_task( success = False stored_keys: list[ObjectKey] = [] stored_sizes: list[int] = [] + bytes_transferred = 0 try: success, stored_keys, stored_sizes = future.result() + bytes_transferred = sum(stored_sizes) except Exception as e: logger.error("RawBlockL2Adapter store task %d failed: %s", task_id, e) with self._lock: self._store_inflight_tasks -= 1 - self._completed_store_tasks[task_id] = success + self._completed_store_tasks[task_id] = L2StoreResult( + success, bytes_transferred + ) event_fd = self._store_efd if stored_keys: try: diff --git a/lmcache/v1/distributed/l2_adapters/s3_l2_adapter.py b/lmcache/v1/distributed/l2_adapters/s3_l2_adapter.py index 7c5bdcc4a7b..749c570cc2f 100644 --- a/lmcache/v1/distributed/l2_adapters/s3_l2_adapter.py +++ b/lmcache/v1/distributed/l2_adapters/s3_l2_adapter.py @@ -34,6 +34,7 @@ from lmcache.logging import init_logger from lmcache.native_storage_ops import Bitmap from lmcache.v1.distributed.api import ObjectKey +from lmcache.v1.distributed.internal_api import L2StoreResult from lmcache.v1.distributed.l2_adapters.base import ( L2AdapterInterface, L2TaskId, @@ -391,7 +392,7 @@ def __init__(self, config: S3L2AdapterConfig): self._load_efd = create_event_notifier() self._next_task_id: L2TaskId = 0 - self._completed_store_tasks: dict[L2TaskId, bool] = {} + self._completed_store_tasks: dict[L2TaskId, L2StoreResult] = {} self._completed_lookup_tasks: dict[L2TaskId, Bitmap] = {} self._completed_load_tasks: dict[L2TaskId, Bitmap] = {} @@ -462,7 +463,7 @@ def submit_store_task( task_id = self._next_task_id self._next_task_id += 1 if self._connection_disabled: - self._completed_store_tasks[task_id] = False + self._completed_store_tasks[task_id] = L2StoreResult(False, 0) disabled = True else: disabled = False @@ -477,7 +478,7 @@ def submit_store_task( ) return task_id - def pop_completed_store_tasks(self) -> dict[L2TaskId, bool]: + def pop_completed_store_tasks(self) -> dict[L2TaskId, L2StoreResult]: with self._lock: completed = self._completed_store_tasks self._completed_store_tasks = {} @@ -876,8 +877,11 @@ async def _execute_store( self._record_connection_outcome(last_error if not success else None) + bytes_transferred = sum(newly_stored_sizes) with self._lock: - self._completed_store_tasks[task_id] = success + self._completed_store_tasks[task_id] = L2StoreResult( + success, bytes_transferred + ) if newly_stored_keys: self._notify_keys_stored(newly_stored_keys, newly_stored_sizes) diff --git a/lmcache/v1/distributed/l2_adapters/serde_wrapper.py b/lmcache/v1/distributed/l2_adapters/serde_wrapper.py index c372fa313d2..977705c7d37 100644 --- a/lmcache/v1/distributed/l2_adapters/serde_wrapper.py +++ b/lmcache/v1/distributed/l2_adapters/serde_wrapper.py @@ -40,7 +40,7 @@ from lmcache.native_storage_ops import Bitmap from lmcache.v1.distributed.api import MemoryLayoutDesc, ObjectKey from lmcache.v1.distributed.error import L1Error -from lmcache.v1.distributed.internal_api import L2AdapterListener +from lmcache.v1.distributed.internal_api import L2AdapterListener, L2StoreResult from lmcache.v1.distributed.l1_manager import L1Manager from lmcache.v1.distributed.l2_adapters.base import ( AdapterUsage, @@ -131,11 +131,7 @@ def __init__( self._serde_to_load: dict[SerdeTaskId, L2TaskId] = {} # User-visible completion queues (drained by controller polls). - self._completed_store: dict[L2TaskId, bool] = {} - # Bytes actually transferred per completed store task, forwarded - # from the inner adapter so wrapped fast-path adapters still - # expose accurate throughput data. - self._completed_store_bytes: dict[L2TaskId, int] = {} + self._completed_store: dict[L2TaskId, L2StoreResult] = {} self._completed_load: dict[L2TaskId, Bitmap] = {} self._stop_flag = threading.Event() @@ -217,18 +213,12 @@ def submit_store_task( return wrapped_id return wrapped_id - def pop_completed_store_tasks(self) -> dict[L2TaskId, bool]: + def pop_completed_store_tasks(self) -> dict[L2TaskId, L2StoreResult]: with self._lock: result = self._completed_store self._completed_store = {} return result - def pop_completed_store_task_bytes(self) -> dict[L2TaskId, int]: - with self._lock: - result = self._completed_store_bytes - self._completed_store_bytes = {} - return result - # ------------------------------------------------------------------ # Lookup / unlock (pure delegation) # ------------------------------------------------------------------ @@ -467,8 +457,7 @@ def _drain_inner_store(self) -> None: """Drain inner store completions; release temp read locks (auto- delete) and finalize the wrapped tasks.""" completed = self._inner.pop_completed_store_tasks() - inner_bytes = self._inner.pop_completed_store_task_bytes() - for inner_id, success in completed.items(): + for inner_id, result in completed.items(): with self._lock: wrapped_id = self._inner_to_store.pop(inner_id, None) state = ( @@ -484,7 +473,9 @@ def _drain_inner_store(self) -> None: continue if state is not None: self._l1_manager.finish_read(state.temp_keys) - self._finalize_store(wrapped_id, success, inner_bytes.get(inner_id)) + self._finalize_store( + wrapped_id, result.is_successful(), result.bytes_transferred() + ) def _drain_inner_load(self) -> None: """Drain inner load completions; on per-key success submit @@ -622,18 +613,13 @@ def _finalize_store( self, wrapped_id: L2TaskId, success: bool, - bytes_transferred: int | None = None, + bytes_transferred: int = 0, ) -> None: with self._lock: self._store_tasks.pop(wrapped_id, None) - self._completed_store[wrapped_id] = success - # Only record bytes when the inner adapter reported them. - # Absence in the dict signals "unknown" to the controller, - # which then falls back to submitted-bytes accounting. A 0 - # here means "transferred nothing" -- the subscriber drops - # those samples. - if bytes_transferred is not None: - self._completed_store_bytes[wrapped_id] = bytes_transferred + self._completed_store[wrapped_id] = L2StoreResult( + success, bytes_transferred + ) try: self._store_efd.notify() except OSError: diff --git a/lmcache/v1/distributed/storage_controllers/store_controller.py b/lmcache/v1/distributed/storage_controllers/store_controller.py index c24982ba786..96adb1fa084 100644 --- a/lmcache/v1/distributed/storage_controllers/store_controller.py +++ b/lmcache/v1/distributed/storage_controllers/store_controller.py @@ -182,11 +182,8 @@ class InFlightStoreTask: l2_store_result: bool | None = None """L2 outcome (True=success, False=failure, None=still in flight).""" - l2_bytes_transferred: int | None = None - """Bytes actually transferred by the adapter for this task. ``None`` - means the adapter does not track per-task transfer bytes (default - behavior for non fast-path adapters); consumers fall back to the - submitted-bytes count from the SUBMITTED event.""" + l2_bytes_transferred: int = 0 + """Bytes actually transferred by the adapter for this task.""" # Main class @@ -528,8 +525,7 @@ def _drain_l2_store_completions(self, signaled_adapters: set[int]) -> None: for adapter_index in signaled_adapters: adapter = self._l2_adapters[adapter_index] completed = adapter.pop_completed_store_tasks() - bytes_map = adapter.pop_completed_store_task_bytes() - for task_id, success in completed.items(): + for task_id, result in completed.items(): task = self._in_flight_tasks.get((adapter_index, task_id)) if task is None: logger.warning( @@ -538,9 +534,8 @@ def _drain_l2_store_completions(self, signaled_adapters: set[int]) -> None: adapter_index, ) continue - task.l2_store_result = success - if task_id in bytes_map: - task.l2_bytes_transferred = bytes_map[task_id] + task.l2_store_result = result.is_successful() + task.l2_bytes_transferred = result.bytes_transferred() def _advance_request( self, @@ -570,20 +565,12 @@ def _finalize_store( self._status_in_flight_count -= 1 l2_name = self._adapter_descriptors[adapter_index].type_name - # Adapters that fast-path duplicate keys report ``bytes_transferred`` - # so the L2 throughput subscriber can use real-work bytes for the - # histogram instead of submitted bytes (which inflates dt-based - # throughput when the adapter skipped the write). Adapters that - # don't report bytes leave this at ``None`` -- the field is then - # omitted from the event so consumers fall back to the submitted - # bytes carried on the SUBMITTED event. completion_meta: dict[str, object] = { "adapter_index": adapter_index, "task_id": task_id, "l2_name": l2_name, + "bytes_transferred": task.l2_bytes_transferred, } - if task.l2_bytes_transferred is not None: - completion_meta["bytes_transferred"] = task.l2_bytes_transferred if success: self._event_bus.publish( Event( diff --git a/lmcache/v1/mp_observability/subscribers/metrics/l2_throughput.py b/lmcache/v1/mp_observability/subscribers/metrics/l2_throughput.py index ed415936e5f..1d4119cddf8 100644 --- a/lmcache/v1/mp_observability/subscribers/metrics/l2_throughput.py +++ b/lmcache/v1/mp_observability/subscribers/metrics/l2_throughput.py @@ -21,17 +21,6 @@ - ``(end_ts - start_ts)`` spans submit -> complete and therefore includes adapter queue, network, and disk time — not just transfer. The histogram is "bytes / end-to-end latency", not raw transfer rate. - -Fast-path adapters: - Some adapters (``mock``, ``fs``, ``nixl_store``) skip writes for keys - that are already present, which makes ``dt`` collapse to near-zero and - inflates store throughput (the controller asked to store N bytes but - the adapter actually transferred 0). When ``L2_STORE_COMPLETED`` - carries ``bytes_transferred``, the store path uses it instead of the - submitted bytes; if ``bytes_transferred == 0`` the sample is dropped - (no work, no useful throughput data). When the field is absent - (adapter doesn't track it), behavior matches the load path -- submitted - bytes / dt. """ # Future @@ -101,16 +90,16 @@ def _on_store_completed(self, event: Event) -> None: key = self._store_key(event) if key is None: return - # If the adapter reports per-task transfer bytes, prefer that - # over submitted bytes. ``None`` means "adapter doesn't track" -> - # fall back to submitted-bytes accounting (the load-path code). + + # ``bytes_transferred`` is always present in L2_STORE_COMPLETED + # events (populated from L2StoreResult). bytes_transferred = event.metadata.get("bytes_transferred") self._record( event=event, correlation_key=key, pending=self._pending_store, hist=self._store_hist, - override_bytes=bytes_transferred, + real_bytes_transferred=bytes_transferred, ) # -- Load path (L2->L1) ------------------------------------------------ @@ -166,18 +155,27 @@ def _record( correlation_key: tuple[int, int], pending: dict[tuple[int, int], tuple[float, int]], hist: Any, - override_bytes: int | None = None, + real_bytes_transferred: int | None = None, ) -> None: + """ + Record a throughput sample. + + Args: + event: The COMPLETED event, used for timestamp and metadata. + correlation_key: The key to look up the matching SUBMITTED event. + pending: The dict mapping correlation keys to (t_start, total_bytes). + hist: The OTel histogram to record into. + real_bytes_transferred: If set, use this instead of the submitted + totay_bytes, When it's 0, drop the sample. + """ pending_entry = pending.pop(correlation_key, None) if pending_entry is None: return # no matching SUBMITTED event; t_start, total_bytes = pending_entry - # ``override_bytes`` carries the adapter-reported transfer bytes - # for fast-path-aware adapters. ``None`` -> use submitted bytes - # (current behavior). ``0`` -> adapter fast-pathed everything, - # there is no real throughput to record -- drop the sample. - effective_bytes = total_bytes if override_bytes is None else override_bytes + effective_bytes = ( + total_bytes if real_bytes_transferred is None else real_bytes_transferred + ) if effective_bytes <= 0: return diff --git a/tests/v1/distributed/test_dax_l2_adapter.py b/tests/v1/distributed/test_dax_l2_adapter.py index e42a753c467..9a0005ca3c0 100644 --- a/tests/v1/distributed/test_dax_l2_adapter.py +++ b/tests/v1/distributed/test_dax_l2_adapter.py @@ -134,7 +134,7 @@ def store_and_wait(adapter: DaxL2Adapter, key: ObjectKey, obj: MemoryObj) -> Non task_id = adapter.submit_store_task([key], [obj]) assert wait_for_event_fd(adapter.get_store_event_fd()) completed = adapter.pop_completed_store_tasks() - assert completed == {task_id: True} + assert completed[task_id].is_successful() def bitmap_to_bools(bitmap: Bitmap, size: int) -> list[bool]: @@ -170,7 +170,8 @@ def test_dax_adapter_store_lookup_load_and_one_shot_results(tmp_path): store_task = adapter.submit_store_task([key0, key2], [obj0, obj2]) assert wait_for_event_fd(adapter.get_store_event_fd()) - assert adapter.pop_completed_store_tasks() == {store_task: True} + completed = adapter.pop_completed_store_tasks() + assert completed[store_task].is_successful() assert adapter.pop_completed_store_tasks() == {} assert listener.stored == [[key0, key2]] @@ -294,7 +295,8 @@ def test_dax_adapter_full_arena_does_not_evict_internally(tmp_path): task_id = adapter.submit_store_task([key2], [obj2]) assert wait_for_event_fd(adapter.get_store_event_fd()) - assert adapter.pop_completed_store_tasks() == {task_id: False} + completed = adapter.pop_completed_store_tasks() + assert not completed[task_id].is_successful() assert listener.stored == [[key0], [key1]] assert adapter.get_usage().usage_fraction == pytest.approx(1.0) diff --git a/tests/v1/distributed/test_l2_adapter_base.py b/tests/v1/distributed/test_l2_adapter_base.py index 5b17288289c..dbaf10410de 100644 --- a/tests/v1/distributed/test_l2_adapter_base.py +++ b/tests/v1/distributed/test_l2_adapter_base.py @@ -15,10 +15,11 @@ # First Party from lmcache.v1.distributed.api import ObjectKey -from lmcache.v1.distributed.internal_api import L2AdapterListener +from lmcache.v1.distributed.internal_api import L2AdapterListener, L2StoreResult from lmcache.v1.distributed.l2_adapters.base import ( AdapterUsage, L2AdapterInterface, + L2TaskId, ) @@ -47,7 +48,7 @@ def get_load_event_fd(self) -> int: def submit_store_task(self, keys, objects): return 0 - def pop_completed_store_tasks(self): + def pop_completed_store_tasks(self) -> dict[L2TaskId, L2StoreResult]: return {} def submit_lookup_and_lock_task(self, keys): diff --git a/tests/v1/distributed/test_mock_l2_adapter.py b/tests/v1/distributed/test_mock_l2_adapter.py index eb0ddd522df..b3180c1a29a 100644 --- a/tests/v1/distributed/test_mock_l2_adapter.py +++ b/tests/v1/distributed/test_mock_l2_adapter.py @@ -202,7 +202,7 @@ def test_pop_completed_store_tasks_returns_completed(self, adapter): completed = adapter.pop_completed_store_tasks() assert task_id in completed - assert completed[task_id] is True # Should be successful + assert completed[task_id].is_successful() # Should be successful def test_pop_completed_store_tasks_clears_completed(self, adapter): """pop_completed_store_tasks should clear the completed tasks.""" @@ -244,7 +244,7 @@ def test_submit_multiple_store_tasks(self, adapter): # All tasks should be completed for task_id in task_ids: assert task_id in completed - assert completed[task_id] is True + assert completed[task_id].is_successful() def test_store_batch_of_objects(self, adapter): """A single store task can store multiple key-object pairs.""" @@ -258,7 +258,7 @@ def test_store_batch_of_objects(self, adapter): completed = adapter.pop_completed_store_tasks() assert task_id in completed - assert completed[task_id] is True + assert completed[task_id].is_successful() # ============================================================================= @@ -516,7 +516,7 @@ def test_store_lookup_load_workflow(self, adapter): store_task_id = adapter.submit_store_task([key], [store_obj]) assert wait_for_event_fd(store_fd, timeout=5.0) completed = adapter.pop_completed_store_tasks() - assert completed[store_task_id] is True + assert completed[store_task_id].is_successful() # Step 2: Lookup and lock lookup_task_id = adapter.submit_lookup_and_lock_task([key]) @@ -556,7 +556,7 @@ def test_multiple_objects_workflow(self, adapter): store_task_id = adapter.submit_store_task(keys, store_objs) assert wait_for_event_fd(store_fd, timeout=5.0) completed = adapter.pop_completed_store_tasks() - assert completed[store_task_id] is True + assert completed[store_task_id].is_successful() # Lookup all lookup_task_id = adapter.submit_lookup_and_lock_task(keys) diff --git a/tests/v1/distributed/test_mooncake_store_l2_adapter.py b/tests/v1/distributed/test_mooncake_store_l2_adapter.py index 67806bec4f7..1f6a74d56ea 100644 --- a/tests/v1/distributed/test_mooncake_store_l2_adapter.py +++ b/tests/v1/distributed/test_mooncake_store_l2_adapter.py @@ -657,7 +657,7 @@ def test_store_and_lookup(self): store_tid = self.adapter.submit_store_task(keys, objs) assert wait_for_event_fd(store_fd) completed = self.adapter.pop_completed_store_tasks() - assert completed[store_tid] is True + assert completed[store_tid].is_successful() # Lookup all — should find everything lookup_tid = self.adapter.submit_lookup_and_lock_task(keys) @@ -695,7 +695,7 @@ def test_full_store_lookup_load_workflow(self): # Store store_tid = self.adapter.submit_store_task([key], [store_obj]) assert wait_for_event_fd(store_fd) - assert self.adapter.pop_completed_store_tasks()[store_tid] is True + assert self.adapter.pop_completed_store_tasks()[store_tid].is_successful() # Lookup lookup_tid = self.adapter.submit_lookup_and_lock_task([key]) @@ -733,7 +733,7 @@ def test_batch_store_lookup_load(self): # Store all store_tid = self.adapter.submit_store_task(keys, store_objs) assert wait_for_event_fd(store_fd) - assert self.adapter.pop_completed_store_tasks()[store_tid] is True + assert self.adapter.pop_completed_store_tasks()[store_tid].is_successful() # Lookup all lookup_tid = self.adapter.submit_lookup_and_lock_task(keys) @@ -798,7 +798,7 @@ def test_delete_stored_key(self): # Store store_tid = self.adapter.submit_store_task([key], [store_obj]) assert wait_for_event_fd(store_fd) - assert self.adapter.pop_completed_store_tasks()[store_tid] is True + assert self.adapter.pop_completed_store_tasks()[store_tid].is_successful() # Confirm stored lookup_tid = self.adapter.submit_lookup_and_lock_task([key]) @@ -840,7 +840,7 @@ def test_batch_delete_mixed_existing_and_missing(self): # Store the first 3 store_tid = self.adapter.submit_store_task(stored_keys, stored_objs) assert wait_for_event_fd(store_fd) - assert self.adapter.pop_completed_store_tasks()[store_tid] is True + assert self.adapter.pop_completed_store_tasks()[store_tid].is_successful() # Confirm they exist lookup_tid = self.adapter.submit_lookup_and_lock_task(stored_keys) @@ -874,7 +874,7 @@ def test_delete_updates_usage_tracking(self): # Store store_tid = self.adapter.submit_store_task([key], [store_obj]) assert wait_for_event_fd(store_fd) - assert self.adapter.pop_completed_store_tasks()[store_tid] is True + assert self.adapter.pop_completed_store_tasks()[store_tid].is_successful() usage_after_store = self.adapter.get_usage().total_bytes_used assert usage_after_store > usage_before, "Usage should increase after store" @@ -967,7 +967,7 @@ def test_buffer_backed_store_lookup_load(self): store_tid = adapter.submit_store_task([key], [store_obj]) assert wait_for_event_fd(adapter.get_store_event_fd()) - assert adapter.pop_completed_store_tasks()[store_tid] is True + assert adapter.pop_completed_store_tasks()[store_tid].is_successful() lookup_tid = adapter.submit_lookup_and_lock_task([key]) assert wait_for_event_fd(adapter.get_lookup_and_lock_event_fd()) diff --git a/tests/v1/distributed/test_native_connector_l2_adapter.py b/tests/v1/distributed/test_native_connector_l2_adapter.py index 58136a963fe..4f242481632 100644 --- a/tests/v1/distributed/test_native_connector_l2_adapter.py +++ b/tests/v1/distributed/test_native_connector_l2_adapter.py @@ -432,7 +432,7 @@ def test_store_signals_event_fd_and_completes(self, adapter): completed = adapter.pop_completed_store_tasks() assert task_id in completed - assert completed[task_id] is True + assert completed[task_id].is_successful() def test_pop_clears_completed_tasks(self, adapter): key = create_object_key(1) @@ -465,7 +465,7 @@ def test_multiple_store_tasks_get_unique_ids(self, adapter): completed.update(adapter.pop_completed_store_tasks()) for tid in task_ids: - assert completed[tid] is True + assert completed[tid].is_successful() def test_batch_store(self, adapter): keys = [create_object_key(i) for i in range(3)] @@ -476,7 +476,7 @@ def test_batch_store(self, adapter): assert wait_for_event_fd(store_fd, timeout=5.0) completed = adapter.pop_completed_store_tasks() - assert completed[task_id] is True + assert completed[task_id].is_successful() # ============================================================================= @@ -670,7 +670,7 @@ def test_store_lookup_load_workflow(self, adapter): # Store store_tid = adapter.submit_store_task([key], [store_obj]) assert wait_for_event_fd(store_fd, timeout=5.0) - assert adapter.pop_completed_store_tasks()[store_tid] is True + assert adapter.pop_completed_store_tasks()[store_tid].is_successful() # Lookup lookup_tid = adapter.submit_lookup_and_lock_task([key]) @@ -703,7 +703,7 @@ def test_multiple_objects_workflow(self, adapter): # Store all store_tid = adapter.submit_store_task(keys, store_objs) assert wait_for_event_fd(store_fd, timeout=5.0) - assert adapter.pop_completed_store_tasks()[store_tid] is True + assert adapter.pop_completed_store_tasks()[store_tid].is_successful() # Lookup all lookup_tid = adapter.submit_lookup_and_lock_task(keys) diff --git a/tests/v1/distributed/test_nixl_store_dynamic_l2_adapter.py b/tests/v1/distributed/test_nixl_store_dynamic_l2_adapter.py index 91fb18dd9e2..59a4c924349 100644 --- a/tests/v1/distributed/test_nixl_store_dynamic_l2_adapter.py +++ b/tests/v1/distributed/test_nixl_store_dynamic_l2_adapter.py @@ -255,7 +255,7 @@ def test_pop_completed_store_tasks_returns_completed(self, adapter): completed = adpt.pop_completed_store_tasks() assert task_id in completed - assert completed[task_id] is True + assert completed[task_id].is_successful() def test_store_creates_file_on_disk(self, adapter): adpt, buf, tmp_dir = adapter @@ -407,7 +407,7 @@ def test_store_lookup_load_workflow(self, adapter): store_task = adpt.submit_store_task([key], [store_obj]) wait_for_event_fd(adpt.get_store_event_fd()) completed = adpt.pop_completed_store_tasks() - assert completed[store_task] is True + assert completed[store_task].is_successful() # Lookup lookup_task = adpt.submit_lookup_and_lock_task([key]) diff --git a/tests/v1/distributed/test_nixl_store_l2_adapter.py b/tests/v1/distributed/test_nixl_store_l2_adapter.py index 1155e1ba938..fed9b40d2fd 100644 --- a/tests/v1/distributed/test_nixl_store_l2_adapter.py +++ b/tests/v1/distributed/test_nixl_store_l2_adapter.py @@ -253,7 +253,7 @@ def test_pop_completed_store_tasks_returns_completed(self, adapter): completed = adpt.pop_completed_store_tasks() assert task_id in completed - assert completed[task_id] is True + assert completed[task_id].is_successful() def test_pop_completed_store_tasks_clears_completed(self, adapter): """pop_completed_store_tasks should clear the completed tasks.""" @@ -298,7 +298,7 @@ def test_submit_multiple_store_tasks(self, adapter): for task_id in task_ids: assert task_id in completed - assert completed[task_id] is True + assert completed[task_id].is_successful() def test_store_batch_of_objects(self, adapter): """A single store task can store multiple key-object pairs.""" @@ -315,7 +315,7 @@ def test_store_batch_of_objects(self, adapter): completed = adpt.pop_completed_store_tasks() assert task_id in completed - assert completed[task_id] is True + assert completed[task_id].is_successful() # ============================================================================= @@ -589,7 +589,7 @@ def test_store_lookup_load_workflow(self, adapter): store_task_id = adpt.submit_store_task([key], [store_obj]) assert wait_for_event_fd(store_fd, timeout=5.0) completed = adpt.pop_completed_store_tasks() - assert completed[store_task_id] is True + assert completed[store_task_id].is_successful() # Step 2: Lookup and lock lookup_task_id = adpt.submit_lookup_and_lock_task([key]) @@ -627,7 +627,7 @@ def test_multi_page_object_workflow(self, adapter): store_task_id = adpt.submit_store_task([key], [store_obj]) assert wait_for_event_fd(store_fd, timeout=5.0) completed = adpt.pop_completed_store_tasks() - assert completed[store_task_id] is True + assert completed[store_task_id].is_successful() # Lookup lookup_task_id = adpt.submit_lookup_and_lock_task([key]) @@ -668,7 +668,7 @@ def test_multiple_objects_workflow(self, adapter): store_task_id = adpt.submit_store_task(keys, store_objs) assert wait_for_event_fd(store_fd, timeout=5.0) completed = adpt.pop_completed_store_tasks() - assert completed[store_task_id] is True + assert completed[store_task_id].is_successful() # Lookup all lookup_task_id = adpt.submit_lookup_and_lock_task(keys) diff --git a/tests/v1/distributed/test_raw_block_l2_adapter.py b/tests/v1/distributed/test_raw_block_l2_adapter.py index 3f39adc6a88..697dad81ee0 100644 --- a/tests/v1/distributed/test_raw_block_l2_adapter.py +++ b/tests/v1/distributed/test_raw_block_l2_adapter.py @@ -203,7 +203,7 @@ def _run_store(adapter: RawBlockL2Adapter, keys, objects) -> bool: assert _wait_event_fd(adapter.get_store_event_fd()) completed = adapter.pop_completed_store_tasks() assert task_id in completed - return completed[task_id] + return completed[task_id].is_successful() def _run_lookup(adapter: RawBlockL2Adapter, keys): @@ -400,7 +400,7 @@ def test_raw_block_l2_adapter_listener_errors_do_not_block_eventfds(): store_task_id = adapter.submit_store_task([key], [obj]) assert _wait_event_fd(adapter.get_store_event_fd()) - assert adapter.pop_completed_store_tasks()[store_task_id] is True + assert adapter.pop_completed_store_tasks()[store_task_id].is_successful() load_buffer = _create_memory_obj(fill_value=0.0) load_task_id = adapter.submit_load_task([key], [load_buffer]) diff --git a/tests/v1/distributed/test_resp_l2_adapter_integration.py b/tests/v1/distributed/test_resp_l2_adapter_integration.py index 486b3a76ca3..df6414d1502 100644 --- a/tests/v1/distributed/test_resp_l2_adapter_integration.py +++ b/tests/v1/distributed/test_resp_l2_adapter_integration.py @@ -148,7 +148,7 @@ def test_store_and_lookup(self): store_tid = self.adapter.submit_store_task(keys, objs) assert wait_for_event_fd(store_fd) completed = self.adapter.pop_completed_store_tasks() - assert completed[store_tid] is True + assert completed[store_tid].is_successful() # Lookup all — should find everything lookup_tid = self.adapter.submit_lookup_and_lock_task(keys) @@ -186,7 +186,7 @@ def test_full_store_lookup_load_workflow(self): # Store store_tid = self.adapter.submit_store_task([key], [store_obj]) assert wait_for_event_fd(store_fd) - assert self.adapter.pop_completed_store_tasks()[store_tid] is True + assert self.adapter.pop_completed_store_tasks()[store_tid].is_successful() # Lookup lookup_tid = self.adapter.submit_lookup_and_lock_task([key]) @@ -224,7 +224,7 @@ def test_batch_store_lookup_load(self): # Store all store_tid = self.adapter.submit_store_task(keys, store_objs) assert wait_for_event_fd(store_fd) - assert self.adapter.pop_completed_store_tasks()[store_tid] is True + assert self.adapter.pop_completed_store_tasks()[store_tid].is_successful() # Lookup all lookup_tid = self.adapter.submit_lookup_and_lock_task(keys) diff --git a/tests/v1/distributed/test_s3_l2_adapter.py b/tests/v1/distributed/test_s3_l2_adapter.py index ab6ea5d29b7..afcb23c96ae 100644 --- a/tests/v1/distributed/test_s3_l2_adapter.py +++ b/tests/v1/distributed/test_s3_l2_adapter.py @@ -380,7 +380,7 @@ def test_roundtrip_single_key(self, adapter): tid = adapter.submit_store_task([key], [obj]) assert wait_for_event_fd(adapter.get_store_event_fd()) completed = adapter.pop_completed_store_tasks() - assert completed == {tid: True} + assert completed[tid].is_successful() # Lookup tid = adapter.submit_lookup_and_lock_task([key]) @@ -564,7 +564,7 @@ def test_trips_after_three_connection_errors(self, adapter): ) wait_for_event_fd(adapter.get_store_event_fd(), timeout=2.0) completed = adapter.pop_completed_store_tasks() - assert completed[disabled_tid] is False + assert not completed[disabled_tid].is_successful() assert _BACKEND.counts()["put"] == put_before # never reached the backend # Lookup and load also short-circuit to all-zero bitmaps. From aa56b7db3dc4e0a752888fe10bdc1e384e3d2b17 Mon Sep 17 00:00:00 2001 From: chunxiaozheng Date: Tue, 26 May 2026 12:16:19 +0800 Subject: [PATCH 60/69] [refactor] refactor bench engine cli (#3341) Signed-off-by: idellzheng --- .../bench/engine_bench/bench-engine.md | 4 +- lmcache/cli/commands/bench/__init__.py | 547 +--------------- .../commands/bench/engine_bench/command.py | 601 ++++++++++++++++++ .../bench/engine_bench/interactive/state.py | 4 +- .../cli/commands/bench/test_bench_command.py | 39 +- 5 files changed, 630 insertions(+), 565 deletions(-) create mode 100644 lmcache/cli/commands/bench/engine_bench/command.py diff --git a/docs/design/cli/commands/bench/engine_bench/bench-engine.md b/docs/design/cli/commands/bench/engine_bench/bench-engine.md index 5eadcf1b1be..8b1f5d2bc16 100644 --- a/docs/design/cli/commands/bench/engine_bench/bench-engine.md +++ b/docs/design/cli/commands/bench/engine_bench/bench-engine.md @@ -269,7 +269,7 @@ instance, and returns it ready to `run()`. ## 3. End-to-End Flow -The orchestrator in `BenchCommand._bench_engine()` wires everything together: +The orchestrator in `engine_bench.command.run_engine_bench()` wires everything together: ``` 0. _resolve_args(args) → argparse.Namespace @@ -628,7 +628,7 @@ def create_workload(...): ### Step 3: Add CLI args -In `bench/__init__.py`, inside `_register_engine()`: +In `engine_bench/command.py`, inside `register_engine_parser()`: 1. Add `"my-workload"` to the `--workload` choices list. 2. Add a new argument group with **prefixed** arg names: diff --git a/lmcache/cli/commands/bench/__init__.py b/lmcache/cli/commands/bench/__init__.py index fbb7c2f47cd..b256f37ed71 100644 --- a/lmcache/cli/commands/bench/__init__.py +++ b/lmcache/cli/commands/bench/__init__.py @@ -3,30 +3,15 @@ # Standard import argparse -import asyncio -import os import sys # First Party from lmcache.cli.commands.base import BaseCommand from lmcache.cli.commands.bench import test_cache as _test_cache_mod -from lmcache.cli.commands.bench.engine_bench.config import ( - EngineBenchConfig, - parse_args_to_config, +from lmcache.cli.commands.bench.engine_bench.command import ( + register_engine_parser, + run_engine_bench, ) -from lmcache.cli.commands.bench.engine_bench.interactive import run_interactive -from lmcache.cli.commands.bench.engine_bench.interactive.state import ( - InteractiveState, -) -from lmcache.cli.commands.bench.engine_bench.progress import ProgressMonitor -from lmcache.cli.commands.bench.engine_bench.request_sender import ( - RequestSender, -) -from lmcache.cli.commands.bench.engine_bench.stats import ( - FinalStats, - StatsCollector, -) -from lmcache.cli.commands.bench.engine_bench.workloads import create_workload from lmcache.cli.commands.bench.l2_adapter_bench.command import ( register_l2_parser, run_l2_adapter_bench, @@ -67,267 +52,11 @@ def register(self, subparsers: argparse._SubParsersAction) -> None: required=True, metavar="{engine,kvcache,l2}", ) - # TODO(chunxiaozheng): move engine and kvcache to sub module too - self._register_engine(inner) + # TODO(chunxiaozheng): move kvcache to its own sub module too + register_engine_parser(inner, self.execute) self._register_kvcache(inner) register_l2_parser(inner, self.execute) - def _register_engine( - self, - subparsers: argparse._SubParsersAction, - ) -> None: - parser = subparsers.add_parser( - "engine", - help="Benchmark an inference engine.", - ) - - # --- Config file --- - parser.add_argument( - "--config", - default=None, - metavar="FILE", - help="Load configuration from a JSON file (skips interactive mode).", - ) - - # --- General args --- - parser.add_argument( - "--engine-url", - default=None, - help=( - "Inference engine URL (e.g., http://localhost:8000). " - "Set OPENAI_API_KEY env var if authentication is needed." - ), - ) - parser.add_argument( - "--lmcache-url", - default=None, - help="LMCache MP server URL for auto-detecting tokens per GB.", - ) - parser.add_argument( - "--model", - default=None, - help="Model name (auto-detected from engine if omitted).", - ) - parser.add_argument( - "--workload", - default=None, - choices=[ - "long-doc-permutator", - "long-doc-qa", - "multi-round-chat", - "prefix-suffix-tuner", - "random-prefill", - ], - help="Workload type.", - ) - parser.add_argument( - "--kv-cache-volume", - type=float, - default=100.0, - help="Target active KV cache in GB (default: 100).", - ) - parser.add_argument( - "--tokens-per-gb-kvcache", - type=int, - default=None, - help=("Tokens per GB of KV cache (required if --lmcache-url is not set)."), - ) - parser.add_argument( - "--seed", - type=int, - default=42, - help="Random seed (default: 42).", - ) - parser.add_argument( - "--output-dir", - default=".", - help="Directory for output files (default: current).", - ) - parser.add_argument( - "--no-csv", - action="store_true", - help="Skip CSV export.", - ) - parser.add_argument( - "--json", - action="store_true", - help="Export JSON summary.", - ) - parser.add_argument( - "-q", - "--quiet", - action="store_true", - help="Suppress real-time progress display.", - ) - parser.add_argument( - "--no-interactive", - action="store_true", - help=( - "Disable interactive mode. Errors if required arguments are missing." - ), - ) - parser.add_argument( - "--export-config", - default=None, - metavar="FILE", - help=( - "Export resolved configuration to a JSON file and exit. " - "Does not run the benchmark or enter interactive mode." - ), - ) - - # --- Long-doc-permutator workload args --- - ldp_group = parser.add_argument_group("long-doc-permutator workload options") - ldp_group.add_argument( - "--ldp-num-contexts", - type=int, - default=5, - help="Number of unique context documents (default: 5).", - ) - ldp_group.add_argument( - "--ldp-context-length", - type=int, - default=5000, - help="Token length of each context (default: 5000).", - ) - ldp_group.add_argument( - "--ldp-system-prompt-length", - type=int, - default=1000, - help="Token length of the shared system prompt (default: 1000). " - "Use 0 for no system prompt.", - ) - ldp_group.add_argument( - "--ldp-num-permutations", - type=int, - default=10, - help="Number of distinct permutations to send (default: 10). " - "Capped at N! where N = --ldp-num-contexts.", - ) - ldp_group.add_argument( - "--ldp-num-inflight-requests", - type=int, - default=1, - help="Max concurrent in-flight requests (default: 1).", - ) - - # --- Long-doc-qa workload args --- - group = parser.add_argument_group("long-doc-qa workload options") - group.add_argument( - "--ldqa-document-length", - type=int, - default=10000, - help="Token length per document (default: 10000).", - ) - group.add_argument( - "--ldqa-query-per-document", - type=int, - default=2, - help="Questions per document (default: 2).", - ) - group.add_argument( - "--ldqa-shuffle-policy", - default="random", - choices=["random", "tile"], - help="Request ordering (default: random).", - ) - group.add_argument( - "--ldqa-num-inflight-requests", - type=int, - default=3, - help="Max concurrent in-flight requests (default: 3).", - ) - - # --- Multi-round-chat workload args --- - mrc_group = parser.add_argument_group( - "multi-round-chat workload options", - ) - mrc_group.add_argument( - "--mrc-shared-prompt-length", - type=int, - default=2000, - help="System prompt token length (default: 2000).", - ) - mrc_group.add_argument( - "--mrc-chat-history-length", - type=int, - default=10000, - help="Pre-filled chat history token length (default: 10000).", - ) - mrc_group.add_argument( - "--mrc-user-input-length", - type=int, - default=50, - help="Tokens per user query (default: 50).", - ) - mrc_group.add_argument( - "--mrc-output-length", - type=int, - default=200, - help="Max tokens to generate per response (default: 200).", - ) - mrc_group.add_argument( - "--mrc-qps", - type=float, - default=1.0, - help="Queries per second (default: 1.0).", - ) - mrc_group.add_argument( - "--mrc-duration", - type=float, - default=60.0, - help="Benchmark duration in seconds (default: 60).", - ) - - # --- Prefix-suffix-tuner workload args --- - psf_group = parser.add_argument_group( - "prefix-suffix-tuner workload options", - ) - psf_group.add_argument( - "--psf-context-length", - type=int, - default=8000, - help="Total tokens per request (prefix + breaker + suffix) " - "(default: 8000).", - ) - psf_group.add_argument( - "--psf-prefix-ratio", - type=float, - default=0.8, - help="Fraction of context-length used by the prefix (default: 0.8). " - "Must be in (0.0, 1.0). The remainder (minus a 32-token breaker) is " - "the shared suffix.", - ) - psf_group.add_argument( - "--psf-thrash", - type=float, - default=20.0, - help="Size in GB of the KV-cache tier to overflow (default: 20.0). " - "The workload sizes its prefix pool to slightly more than this, " - "so every pass-2 request misses that tier and falls through to " - "the next one. Use the L0 (HBM) size for vanilla vLLM baselines, " - "or the L1 (LMCache DRAM) size for tiered baselines.", - ) - - # --- Random-prefill workload args --- - rp_group = parser.add_argument_group( - "random-prefill workload options", - ) - rp_group.add_argument( - "--rp-request-length", - type=int, - default=10000, - help="Token length per request (default: 10000).", - ) - rp_group.add_argument( - "--rp-num-requests", - type=int, - default=50, - help="Number of requests to send (default: 50).", - ) - - parser.set_defaults(func=self.execute) - # ------------------------------------------------------------------ # kvcache bench target — end-to-end MP cache sanity test # ------------------------------------------------------------------ @@ -382,7 +111,7 @@ def _bench_kvcache(self, args: argparse.Namespace) -> None: def execute(self, args: argparse.Namespace) -> None: handlers = { - "engine": self._bench_engine, + "engine": lambda a: run_engine_bench(self, a), "kvcache": self._bench_kvcache, "l2": lambda a: run_l2_adapter_bench(self, a), } @@ -394,267 +123,3 @@ def execute(self, args: argparse.Namespace) -> None: ) sys.exit(1) handler(args) - - # ------------------------------------------------------------------ - # Engine benchmark orchestrator - # ------------------------------------------------------------------ - - def _get_missing_args(self, args: argparse.Namespace) -> list[str]: - """Return list of missing required CLI flags.""" - missing: list[str] = [] - if args.engine_url is None: - missing.append("--engine-url") - if args.workload is None: - missing.append("--workload") - if ( - args.tokens_per_gb_kvcache is None - and getattr(args, "lmcache_url", None) is None - ): - missing.append("--tokens-per-gb-kvcache or --lmcache-url") - return missing - - def _needs_interactive(self, args: argparse.Namespace) -> bool: - """Check whether interactive mode should be triggered.""" - if getattr(args, "config", None): - return False - return len(self._get_missing_args(args)) > 0 - - def _resolve_args(self, args: argparse.Namespace) -> argparse.Namespace: - """Resolve args via config file, interactive mode, or pass through.""" - # Case 1: --config file - config_path = getattr(args, "config", None) - if config_path: - state = InteractiveState.load_json(config_path) - state.merge_cli_args(args) - resolved = state.to_namespace() - # Carry over output flags from CLI - for attr in ( - "output_dir", - "seed", - "no_csv", - "json", - "quiet", - "format", - "output", - ): - cli_val = getattr(args, attr, None) - if cli_val is not None: - setattr(resolved, attr, cli_val) - return resolved - - # Case 2: --no-interactive or --export-config — error if missing - no_interactive = getattr(args, "no_interactive", False) - export_config = getattr(args, "export_config", None) - if no_interactive or export_config: - missing = self._get_missing_args(args) - if missing: - flag = "--export-config" if export_config else "--no-interactive" - raise SystemExit( - "Missing required arguments: " - + ", ".join(missing) - + f". Provide them or remove {flag} " - "for guided setup." - ) - return args - - # Case 3: Interactive mode - if self._needs_interactive(args): - return run_interactive(args) - - # Case 4: All required args present — run directly - return args - - def _export_config( - self, - config: EngineBenchConfig, - args: argparse.Namespace, - path: str, - ) -> None: - """Export resolved config to JSON and exit. - - Builds a standalone config dict from the resolved - ``EngineBenchConfig`` and workload-specific CLI args. - Environment-specific keys (``engine_url``, ``lmcache_url``) - are excluded by ``InteractiveState.to_json()`` so the exported - config is portable. - """ - # Standard - import json as json_mod - - state = InteractiveState() - state.set("engine_url", config.engine_url) - state.set("model", config.model) - state.set("workload", config.workload) - state.set("kv_cache_volume", config.kv_cache_volume_gb) - state.set("tokens_per_gb_kvcache", config.tokens_per_gb_kvcache) - - # Workload-specific args from namespace - for item in state.get_workload_items(): - value = getattr(args, item.key, item.default) - if value is not None: - state.set(item.key, value) - - # to_json() handles filtering out engine_url, lmcache_url, etc. - data = state.to_json() - - with open(path, "w") as f: - json_mod.dump(data, f, indent=2) - f.write("\n") - - print(f"Configuration exported to {path}") - print( - f"\033[1mReplay with:\033[0m \033[96mlmcache bench engine " - f"--engine-url --config {path}\033[0m" - ) - - def _bench_engine(self, args: argparse.Namespace) -> None: - """Centralized orchestrator: create all modules and run benchmark.""" - # 0. Resolve args (config file / interactive / pass-through) - args = self._resolve_args(args) - - # 1. Parse config - config = parse_args_to_config(args) - - # 1b. --export-config: save resolved config and exit - export_path = getattr(args, "export_config", None) - if export_path: - self._export_config(config, args, export_path) - return - - logger.info( - "Benchmark config: workload=%s, model=%s, " - "kv_cache=%.1f GB, tokens_per_gb=%d", - config.workload, - config.model, - config.kv_cache_volume_gb, - config.tokens_per_gb_kvcache, - ) - - # 2. Create shared modules - stats_collector = StatsCollector() - progress_monitor = ProgressMonitor( - stats_collector, - quiet=config.quiet, - ) - - # 3. Create request sender (callbacks wired after workload creation) - request_sender = RequestSender(config.engine_url, config.model) - - # 4. Create workload - workload = create_workload( - config, - args, - request_sender, - stats_collector, - progress_monitor, - ) - - # 5. Wire callbacks on sender - request_sender.add_on_finished_callback( - lambda result, _text: stats_collector.on_request_finished(result), - ) - request_sender.add_on_finished_callback( - lambda result, _text: progress_monitor.on_request_finished( - result.request_id, - result.successful, - ), - ) - request_sender.add_on_finished_callback(workload.request_finished) - - # 6. Log config and run benchmark - workload.log_config() - progress_monitor.start() - try: - workload.run() - finally: - progress_monitor.stop() - asyncio.run(request_sender.close()) - - # 7. Final metrics - final = stats_collector.get_final_stats() - self._emit_final_metrics(config, final, args) - - # 8. Export - if config.export_csv: - csv_path = os.path.join(config.output_dir, "bench_results.csv") - stats_collector.export_csv(csv_path) - logger.info("CSV results written to %s", csv_path) - if config.export_json: - json_path = os.path.join( - config.output_dir, - "bench_summary.json", - ) - stats_collector.export_json(json_path, config) - logger.info("JSON summary written to %s", json_path) - - # 9. Exit code - if final.failed_requests > 0: - sys.exit(1) - - def _emit_final_metrics( - self, - config: EngineBenchConfig, - final: FinalStats, - args: argparse.Namespace, - ) -> None: - """Emit final benchmark summary using the CLI metrics system.""" - title = f"Engine Benchmark Result ({config.workload})" - metrics = self.create_metrics(title, args, width=56) - - cfg_section = metrics.add_section("config", "Configuration") - cfg_section.add("engine_url", "Engine URL", config.engine_url) - cfg_section.add("model", "Model", config.model) - cfg_section.add("workload", "Workload", config.workload) - - results = metrics.add_section("results", "Results") - results.add( - "successful", - "Successful requests", - final.successful_requests, - ) - results.add("failed", "Failed requests", final.failed_requests) - results.add( - "duration", - "Benchmark duration (s)", - round(final.elapsed_time, 2), - ) - results.add( - "input_tokens", - "Total input tokens", - final.total_input_tokens, - ) - results.add( - "output_tokens", - "Total output tokens", - final.total_output_tokens, - ) - results.add( - "input_tput", - "Input throughput (tok/s)", - round(final.input_throughput, 2), - ) - results.add( - "output_tput", - "Output throughput (tok/s)", - round(final.output_throughput, 2), - ) - - ttft = metrics.add_section("ttft", "Time to First Token") - ttft.add("mean", "Mean TTFT (ms)", round(final.mean_ttft_ms, 2)) - ttft.add("p50", "P50 TTFT (ms)", round(final.p50_ttft_ms, 2)) - ttft.add("p90", "P90 TTFT (ms)", round(final.p90_ttft_ms, 2)) - ttft.add("p99", "P99 TTFT (ms)", round(final.p99_ttft_ms, 2)) - - decode = metrics.add_section("decode", "Decoding Speed") - decode.add( - "mean", - "Mean decode (tok/s)", - round(final.mean_decode_speed, 2), - ) - decode.add( - "p99", - "P99 decode (tok/s)", - round(final.p99_decode_speed, 2), - ) - - metrics.emit() diff --git a/lmcache/cli/commands/bench/engine_bench/command.py b/lmcache/cli/commands/bench/engine_bench/command.py new file mode 100644 index 00000000000..6f01f8b34b2 --- /dev/null +++ b/lmcache/cli/commands/bench/engine_bench/command.py @@ -0,0 +1,601 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``lmcache bench engine`` subcommand implementation. + +This module owns the full registration + execution flow for the +inference engine benchmark. ``BenchCommand`` only forwards CLI dispatch +to :func:`run_engine_bench` and parser registration to +:func:`register_engine_parser`. +""" + +# Future +from __future__ import annotations + +# Standard +from typing import TYPE_CHECKING +import argparse +import asyncio +import os +import sys + +# First Party +from lmcache.cli.commands.bench.engine_bench.config import ( + EngineBenchConfig, + parse_args_to_config, +) +from lmcache.cli.commands.bench.engine_bench.interactive import run_interactive +from lmcache.cli.commands.bench.engine_bench.interactive.state import ( + InteractiveState, +) +from lmcache.cli.commands.bench.engine_bench.progress import ProgressMonitor +from lmcache.cli.commands.bench.engine_bench.request_sender import ( + RequestSender, +) +from lmcache.cli.commands.bench.engine_bench.stats import ( + FinalStats, + StatsCollector, +) +from lmcache.cli.commands.bench.engine_bench.workloads import create_workload +from lmcache.logging import init_logger + +if TYPE_CHECKING: + # First Party + from lmcache.cli.commands.base import BaseCommand + +logger = init_logger(__name__) + + +# --------------------------------------------------------------------------- +# Parser registration +# --------------------------------------------------------------------------- + + +def register_engine_parser( + subparsers: argparse._SubParsersAction, + dispatch_func, +) -> argparse.ArgumentParser: + """Register the ``lmcache bench engine`` subcommand parser. + + Args: + subparsers: The ``bench`` subparsers action. + dispatch_func: Function to bind via ``set_defaults(func=...)``. + Typically ``BenchCommand.execute`` so that the outer + dispatcher can route the call back into + :func:`run_engine_bench`. + + Returns: + The created ``ArgumentParser`` (mostly for testing). + """ + parser = subparsers.add_parser( + "engine", + help="Benchmark an inference engine.", + ) + + # --- Config file --- + parser.add_argument( + "--config", + default=None, + metavar="FILE", + help="Load configuration from a JSON file (skips interactive mode).", + ) + + # --- General args --- + parser.add_argument( + "--engine-url", + default=None, + help=( + "Inference engine URL (e.g., http://localhost:8000). " + "Set OPENAI_API_KEY env var if authentication is needed." + ), + ) + parser.add_argument( + "--lmcache-url", + default=None, + help="LMCache MP server URL for auto-detecting tokens per GB.", + ) + parser.add_argument( + "--model", + default=None, + help="Model name (auto-detected from engine if omitted).", + ) + parser.add_argument( + "--workload", + default=None, + choices=[ + "long-doc-permutator", + "long-doc-qa", + "multi-round-chat", + "prefix-suffix-tuner", + "random-prefill", + ], + help="Workload type.", + ) + parser.add_argument( + "--kv-cache-volume", + type=float, + default=100.0, + help="Target active KV cache in GB (default: 100).", + ) + parser.add_argument( + "--tokens-per-gb-kvcache", + type=int, + default=None, + help=("Tokens per GB of KV cache (required if --lmcache-url is not set)."), + ) + parser.add_argument( + "--seed", + type=int, + default=42, + help="Random seed (default: 42).", + ) + parser.add_argument( + "--output-dir", + default=".", + help="Directory for output files (default: current).", + ) + parser.add_argument( + "--no-csv", + action="store_true", + help="Skip CSV export.", + ) + parser.add_argument( + "--json", + action="store_true", + help="Export JSON summary.", + ) + parser.add_argument( + "-q", + "--quiet", + action="store_true", + help="Suppress real-time progress display.", + ) + parser.add_argument( + "--no-interactive", + action="store_true", + help=("Disable interactive mode. Errors if required arguments are missing."), + ) + parser.add_argument( + "--export-config", + default=None, + metavar="FILE", + help=( + "Export resolved configuration to a JSON file and exit. " + "Does not run the benchmark or enter interactive mode." + ), + ) + + # --- Long-doc-permutator workload args --- + ldp_group = parser.add_argument_group("long-doc-permutator workload options") + ldp_group.add_argument( + "--ldp-num-contexts", + type=int, + default=5, + help="Number of unique context documents (default: 5).", + ) + ldp_group.add_argument( + "--ldp-context-length", + type=int, + default=5000, + help="Token length of each context (default: 5000).", + ) + ldp_group.add_argument( + "--ldp-system-prompt-length", + type=int, + default=1000, + help="Token length of the shared system prompt (default: 1000). " + "Use 0 for no system prompt.", + ) + ldp_group.add_argument( + "--ldp-num-permutations", + type=int, + default=10, + help="Number of distinct permutations to send (default: 10). " + "Capped at N! where N = --ldp-num-contexts.", + ) + ldp_group.add_argument( + "--ldp-num-inflight-requests", + type=int, + default=1, + help="Max concurrent in-flight requests (default: 1).", + ) + + # --- Long-doc-qa workload args --- + group = parser.add_argument_group("long-doc-qa workload options") + group.add_argument( + "--ldqa-document-length", + type=int, + default=10000, + help="Token length per document (default: 10000).", + ) + group.add_argument( + "--ldqa-query-per-document", + type=int, + default=2, + help="Questions per document (default: 2).", + ) + group.add_argument( + "--ldqa-shuffle-policy", + default="random", + choices=["random", "tile"], + help="Request ordering (default: random).", + ) + group.add_argument( + "--ldqa-num-inflight-requests", + type=int, + default=3, + help="Max concurrent in-flight requests (default: 3).", + ) + + # --- Multi-round-chat workload args --- + mrc_group = parser.add_argument_group( + "multi-round-chat workload options", + ) + mrc_group.add_argument( + "--mrc-shared-prompt-length", + type=int, + default=2000, + help="System prompt token length (default: 2000).", + ) + mrc_group.add_argument( + "--mrc-chat-history-length", + type=int, + default=10000, + help="Pre-filled chat history token length (default: 10000).", + ) + mrc_group.add_argument( + "--mrc-user-input-length", + type=int, + default=50, + help="Tokens per user query (default: 50).", + ) + mrc_group.add_argument( + "--mrc-output-length", + type=int, + default=200, + help="Max tokens to generate per response (default: 200).", + ) + mrc_group.add_argument( + "--mrc-qps", + type=float, + default=1.0, + help="Queries per second (default: 1.0).", + ) + mrc_group.add_argument( + "--mrc-duration", + type=float, + default=60.0, + help="Benchmark duration in seconds (default: 60).", + ) + + # --- Prefix-suffix-tuner workload args --- + psf_group = parser.add_argument_group( + "prefix-suffix-tuner workload options", + ) + psf_group.add_argument( + "--psf-context-length", + type=int, + default=8000, + help="Total tokens per request (prefix + breaker + suffix) (default: 8000).", + ) + psf_group.add_argument( + "--psf-prefix-ratio", + type=float, + default=0.8, + help="Fraction of context-length used by the prefix (default: 0.8). " + "Must be in (0.0, 1.0). The remainder (minus a 32-token breaker) is " + "the shared suffix.", + ) + psf_group.add_argument( + "--psf-thrash", + type=float, + default=20.0, + help="Size in GB of the KV-cache tier to overflow (default: 20.0). " + "The workload sizes its prefix pool to slightly more than this, " + "so every pass-2 request misses that tier and falls through to " + "the next one. Use the L0 (HBM) size for vanilla vLLM baselines, " + "or the L1 (LMCache DRAM) size for tiered baselines.", + ) + + # --- Random-prefill workload args --- + rp_group = parser.add_argument_group( + "random-prefill workload options", + ) + rp_group.add_argument( + "--rp-request-length", + type=int, + default=10000, + help="Token length per request (default: 10000).", + ) + rp_group.add_argument( + "--rp-num-requests", + type=int, + default=50, + help="Number of requests to send (default: 50).", + ) + + parser.set_defaults(func=dispatch_func) + return parser + + +# --------------------------------------------------------------------------- +# Argument resolution helpers +# --------------------------------------------------------------------------- + + +def _get_missing_args(args: argparse.Namespace) -> list[str]: + """Return list of missing required CLI flags.""" + missing: list[str] = [] + if args.engine_url is None: + missing.append("--engine-url") + if args.workload is None: + missing.append("--workload") + if ( + args.tokens_per_gb_kvcache is None + and getattr(args, "lmcache_url", None) is None + ): + missing.append("--tokens-per-gb-kvcache or --lmcache-url") + return missing + + +def _needs_interactive(args: argparse.Namespace) -> bool: + """Check whether interactive mode should be triggered.""" + if getattr(args, "config", None): + return False + return len(_get_missing_args(args)) > 0 + + +def _resolve_args(args: argparse.Namespace) -> argparse.Namespace: + """Resolve args via config file, interactive mode, or pass through.""" + # Case 1: --config file + config_path = getattr(args, "config", None) + if config_path: + state = InteractiveState.load_json(config_path) + state.merge_cli_args(args) + resolved = state.to_namespace() + # Carry over output flags from CLI + for attr in ( + "output_dir", + "seed", + "no_csv", + "json", + "quiet", + "format", + "output", + ): + cli_val = getattr(args, attr, None) + if cli_val is not None: + setattr(resolved, attr, cli_val) + return resolved + + # Case 2: --no-interactive or --export-config — error if missing + no_interactive = getattr(args, "no_interactive", False) + export_config = getattr(args, "export_config", None) + if no_interactive or export_config: + missing = _get_missing_args(args) + if missing: + flag = "--export-config" if export_config else "--no-interactive" + raise SystemExit( + "Missing required arguments: " + + ", ".join(missing) + + f". Provide them or remove {flag} " + "for guided setup." + ) + return args + + # Case 3: Interactive mode + if _needs_interactive(args): + return run_interactive(args) + + # Case 4: All required args present — run directly + return args + + +def _export_config( + config: EngineBenchConfig, + args: argparse.Namespace, + path: str, +) -> None: + """Export resolved config to JSON and exit. + + Builds a standalone config dict from the resolved + ``EngineBenchConfig`` and workload-specific CLI args. + Environment-specific keys (``engine_url``, ``lmcache_url``) + are excluded by ``InteractiveState.to_json()`` so the exported + config is portable. + """ + # Standard + import json as json_mod + + state = InteractiveState() + state.set("engine_url", config.engine_url) + state.set("model", config.model) + state.set("workload", config.workload) + state.set("kv_cache_volume", config.kv_cache_volume_gb) + state.set("tokens_per_gb_kvcache", config.tokens_per_gb_kvcache) + + # Workload-specific args from namespace + for item in state.get_workload_items(): + value = getattr(args, item.key, item.default) + if value is not None: + state.set(item.key, value) + + # to_json() handles filtering out engine_url, lmcache_url, etc. + data = state.to_json() + + with open(path, "w") as f: + json_mod.dump(data, f, indent=2) + f.write("\n") + + print(f"Configuration exported to {path}") + print( + f"\033[1mReplay with:\033[0m \033[96mlmcache bench engine " + f"--engine-url --config {path}\033[0m" + ) + + +# --------------------------------------------------------------------------- +# Final metrics emission +# --------------------------------------------------------------------------- + + +def _emit_final_metrics( + command: "BaseCommand", + config: EngineBenchConfig, + final: FinalStats, + args: argparse.Namespace, +) -> None: + """Emit final benchmark summary using the CLI metrics system.""" + title = f"Engine Benchmark Result ({config.workload})" + metrics = command.create_metrics(title, args, width=56) + + cfg_section = metrics.add_section("config", "Configuration") + cfg_section.add("engine_url", "Engine URL", config.engine_url) + cfg_section.add("model", "Model", config.model) + cfg_section.add("workload", "Workload", config.workload) + + results = metrics.add_section("results", "Results") + results.add( + "successful", + "Successful requests", + final.successful_requests, + ) + results.add("failed", "Failed requests", final.failed_requests) + results.add( + "duration", + "Benchmark duration (s)", + round(final.elapsed_time, 2), + ) + results.add( + "input_tokens", + "Total input tokens", + final.total_input_tokens, + ) + results.add( + "output_tokens", + "Total output tokens", + final.total_output_tokens, + ) + results.add( + "input_tput", + "Input throughput (tok/s)", + round(final.input_throughput, 2), + ) + results.add( + "output_tput", + "Output throughput (tok/s)", + round(final.output_throughput, 2), + ) + + ttft = metrics.add_section("ttft", "Time to First Token") + ttft.add("mean", "Mean TTFT (ms)", round(final.mean_ttft_ms, 2)) + ttft.add("p50", "P50 TTFT (ms)", round(final.p50_ttft_ms, 2)) + ttft.add("p90", "P90 TTFT (ms)", round(final.p90_ttft_ms, 2)) + ttft.add("p99", "P99 TTFT (ms)", round(final.p99_ttft_ms, 2)) + + decode = metrics.add_section("decode", "Decoding Speed") + decode.add( + "mean", + "Mean decode (tok/s)", + round(final.mean_decode_speed, 2), + ) + decode.add( + "p99", + "P99 decode (tok/s)", + round(final.p99_decode_speed, 2), + ) + + metrics.emit() + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def run_engine_bench(command: "BaseCommand", args: argparse.Namespace) -> None: + """Centralized orchestrator: create all modules and run the engine bench. + + Args: + command: The outer ``BenchCommand`` instance, used for + ``create_metrics`` (inherited from ``BaseCommand``). + args: Parsed CLI arguments for ``lmcache bench engine``. + """ + # 0. Resolve args (config file / interactive / pass-through) + args = _resolve_args(args) + + # 1. Parse config + config = parse_args_to_config(args) + + # 1b. --export-config: save resolved config and exit + export_path = getattr(args, "export_config", None) + if export_path: + _export_config(config, args, export_path) + return + + logger.info( + "Benchmark config: workload=%s, model=%s, kv_cache=%.1f GB, tokens_per_gb=%d", + config.workload, + config.model, + config.kv_cache_volume_gb, + config.tokens_per_gb_kvcache, + ) + + # 2. Create shared modules + stats_collector = StatsCollector() + progress_monitor = ProgressMonitor( + stats_collector, + quiet=config.quiet, + ) + + # 3. Create request sender (callbacks wired after workload creation) + request_sender = RequestSender(config.engine_url, config.model) + + # 4. Create workload + workload = create_workload( + config, + args, + request_sender, + stats_collector, + progress_monitor, + ) + + # 5. Wire callbacks on sender + request_sender.add_on_finished_callback( + lambda result, _text: stats_collector.on_request_finished(result), + ) + request_sender.add_on_finished_callback( + lambda result, _text: progress_monitor.on_request_finished( + result.request_id, + result.successful, + ), + ) + request_sender.add_on_finished_callback(workload.request_finished) + + # 6. Log config and run benchmark + workload.log_config() + progress_monitor.start() + try: + workload.run() + finally: + progress_monitor.stop() + asyncio.run(request_sender.close()) + + # 7. Final metrics + final = stats_collector.get_final_stats() + _emit_final_metrics(command, config, final, args) + + # 8. Export + if config.export_csv: + csv_path = os.path.join(config.output_dir, "bench_results.csv") + stats_collector.export_csv(csv_path) + logger.info("CSV results written to %s", csv_path) + if config.export_json: + json_path = os.path.join( + config.output_dir, + "bench_summary.json", + ) + stats_collector.export_json(json_path, config) + logger.info("JSON summary written to %s", json_path) + + # 9. Exit code + if final.failed_requests > 0: + sys.exit(1) diff --git a/lmcache/cli/commands/bench/engine_bench/interactive/state.py b/lmcache/cli/commands/bench/engine_bench/interactive/state.py index 9ba159375cc..785af7ab631 100644 --- a/lmcache/cli/commands/bench/engine_bench/interactive/state.py +++ b/lmcache/cli/commands/bench/engine_bench/interactive/state.py @@ -3,7 +3,7 @@ ``InteractiveState`` holds the partially-configured benchmark parameters, can be initialized from CLI args or a saved JSON file, and can be -converted to the ``argparse.Namespace`` that ``_bench_engine()`` expects. +converted to the ``argparse.Namespace`` that ``run_engine_bench()`` expects. """ # Standard @@ -210,7 +210,7 @@ def from_cli_args(cls, args: argparse.Namespace) -> "InteractiveState": return state def to_namespace(self) -> argparse.Namespace: - """Convert to an ``argparse.Namespace`` compatible with ``_bench_engine``. + """Convert to ``argparse.Namespace`` compatible with ``run_engine_bench``. Fills defaults for any unset items, then builds the namespace with the attribute names that ``parse_args_to_config()`` and diff --git a/tests/cli/commands/bench/test_bench_command.py b/tests/cli/commands/bench/test_bench_command.py index 5b667a3a65d..ad1fabc484d 100644 --- a/tests/cli/commands/bench/test_bench_command.py +++ b/tests/cli/commands/bench/test_bench_command.py @@ -14,6 +14,11 @@ # First Party from lmcache.cli.commands.bench import BenchCommand +from lmcache.cli.commands.bench.engine_bench.command import ( + _emit_final_metrics, + _resolve_args, + run_engine_bench, +) from lmcache.cli.commands.bench.engine_bench.config import EngineBenchConfig from lmcache.cli.commands.bench.engine_bench.stats import ( FinalStats, @@ -228,9 +233,8 @@ def test_no_interactive_missing_engine_url(self) -> None: tokens_per_gb_kvcache=6553, no_interactive=True, ) - cmd = BenchCommand() with pytest.raises(SystemExit, match="--engine-url"): - cmd._resolve_args(args) + _resolve_args(args) def test_no_interactive_missing_workload(self) -> None: args = _make_args( @@ -239,9 +243,8 @@ def test_no_interactive_missing_workload(self) -> None: tokens_per_gb_kvcache=6553, no_interactive=True, ) - cmd = BenchCommand() with pytest.raises(SystemExit, match="--workload"): - cmd._resolve_args(args) + _resolve_args(args) def test_no_interactive_missing_tokens_and_lmcache(self) -> None: args = _make_args( @@ -251,16 +254,14 @@ def test_no_interactive_missing_tokens_and_lmcache(self) -> None: lmcache_url=None, no_interactive=True, ) - cmd = BenchCommand() with pytest.raises( SystemExit, match="--tokens-per-gb-kvcache or --lmcache-url" ): - cmd._resolve_args(args) + _resolve_args(args) def test_no_interactive_passes_with_all_args(self) -> None: args = _make_args(no_interactive=True) - cmd = BenchCommand() - result = cmd._resolve_args(args) + result = _resolve_args(args) assert result is args def test_no_interactive_passes_with_lmcache_url(self) -> None: @@ -269,8 +270,7 @@ def test_no_interactive_passes_with_lmcache_url(self) -> None: lmcache_url="http://localhost:8080", no_interactive=True, ) - cmd = BenchCommand() - result = cmd._resolve_args(args) + result = _resolve_args(args) assert result is args @@ -307,9 +307,8 @@ def test_export_config_errors_when_missing_args(self) -> None: engine_url=None, export_config="out.json", ) - cmd = BenchCommand() with pytest.raises(SystemExit, match="--engine-url"): - cmd._resolve_args(args) + _resolve_args(args) def test_export_config_writes_json(self, tmp_path) -> None: # type: ignore[no-untyped-def] export_path = str(tmp_path / "exported.json") @@ -322,7 +321,7 @@ def test_export_config_writes_json(self, tmp_path) -> None: # type: ignore[no-u old_stdout = sys.stdout sys.stdout = io.StringIO() try: - cmd._bench_engine(args) + run_engine_bench(cmd, args) finally: sys.stdout = old_stdout @@ -349,7 +348,7 @@ def test_export_config_excludes_lmcache_url( old_stdout = sys.stdout sys.stdout = io.StringIO() try: - cmd._bench_engine(args) + run_engine_bench(cmd, args) finally: sys.stdout = old_stdout @@ -376,7 +375,7 @@ def test_export_config_includes_workload_args( old_stdout = sys.stdout sys.stdout = io.StringIO() try: - cmd._bench_engine(args) + run_engine_bench(cmd, args) finally: sys.stdout = old_stdout @@ -402,7 +401,7 @@ def test_emit_final_metrics_terminal(self) -> None: old_stdout = sys.stdout sys.stdout = buf = io.StringIO() try: - cmd._emit_final_metrics(config, final, args) + _emit_final_metrics(cmd, config, final, args) finally: sys.stdout = old_stdout @@ -422,7 +421,7 @@ def test_emit_final_metrics_json(self) -> None: old_stdout = sys.stdout sys.stdout = buf = io.StringIO() try: - cmd._emit_final_metrics(config, final, args) + _emit_final_metrics(cmd, config, final, args) finally: sys.stdout = old_stdout @@ -512,7 +511,7 @@ async def _fake_stream(*_args, **_kwargs): old_stdout = sys.stdout sys.stdout = io.StringIO() try: - cmd._bench_engine(args) + run_engine_bench(cmd, args) finally: sys.stdout = old_stdout @@ -592,7 +591,7 @@ async def _fake_stream(*_args, **_kwargs): old_stdout = sys.stdout sys.stdout = io.StringIO() try: - cmd._bench_engine(args) + run_engine_bench(cmd, args) finally: sys.stdout = old_stdout @@ -691,7 +690,7 @@ async def _fake_stream(*_args, **_kwargs): old_stdout = sys.stdout sys.stdout = io.StringIO() try: - cmd._bench_engine(args) + run_engine_bench(cmd, args) finally: sys.stdout = old_stdout From c276b334e5c0d5631ed29781856f105b4cae6917 Mon Sep 17 00:00:00 2001 From: sh <100367948+yeoshuheng@users.noreply.github.com> Date: Tue, 26 May 2026 21:48:38 +0800 Subject: [PATCH 61/69] [Doc] Add recipes for Mistral, Phi & Llama (#3270) * docs: add recipes for phi, mistral, llama Signed-off-by: yeoshuheng <100367948+yeoshuheng@users.noreply.github.com> * docs: update tool calling link Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Signed-off-by: sh <100367948+yeoshuheng@users.noreply.github.com> * docs: update jinja template fp Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Signed-off-by: sh <100367948+yeoshuheng@users.noreply.github.com> * docs: update jinja template fp Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Signed-off-by: sh <100367948+yeoshuheng@users.noreply.github.com> --------- Signed-off-by: yeoshuheng <100367948+yeoshuheng@users.noreply.github.com> Signed-off-by: sh <100367948+yeoshuheng@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- docs/source/recipes/index.rst | 29 +++++++ docs/source/recipes/llama.rst | 132 ++++++++++++++++++++++++++++++++ docs/source/recipes/mixtral.rst | 92 ++++++++++++++++++++++ docs/source/recipes/phi3.rst | 90 ++++++++++++++++++++++ 4 files changed, 343 insertions(+) create mode 100644 docs/source/recipes/llama.rst create mode 100644 docs/source/recipes/mixtral.rst create mode 100644 docs/source/recipes/phi3.rst diff --git a/docs/source/recipes/index.rst b/docs/source/recipes/index.rst index 7d3a05eaa62..2491f188fee 100644 --- a/docs/source/recipes/index.rst +++ b/docs/source/recipes/index.rst @@ -42,30 +42,35 @@ Supported architectures - SGLang - TRT-LLM - Recipe + * - ``MiniMaxM2ForCausalLM`` - ``MiniMaxAI/MiniMax-M2`` - ✓ - — - — - :doc:`minimax_m2` + * - ``Gemma4ForConditionalGeneration`` - ``google/gemma-4-31B-it`` - ✓ - — - — - :doc:`gemma4` + * - ``MistralForCausalLM`` - ``mistralai/Devstral-2-123B-Instruct-2512`` - ✓ - — - — - :doc:`devstral` + * - ``GptOssForCausalLM`` - ``openai/gpt-oss-120b`` - ✓ - — - — - :doc:`gpt_oss` + * - ``Qwen3MoeForCausalLM`` - ``Qwen/Qwen3-235B-A22B`` - ✓ @@ -73,6 +78,27 @@ Supported architectures - — - :doc:`qwen3` + * - ``LlamaForCausalLM`` + - ``meta-llama/Meta-Llama-3.1-70B-Instruct`` + - ✓ + - — + - — + - :doc:`llama` + + * - ``Phi3ForCausalLM`` + - ``microsoft/Phi-4-mini-instruct`` + - ✓ + - — + - — + - :doc:`phi3` + + * - ``MixtralForCausalLM`` + - ``mistralai/Mixtral-8x7B-Instruct-v0.1`` + - ✓ + - — + - — + - :doc:`mixtral` + Legend: ``✓`` validated, ``—`` not validated. Contributing a recipe @@ -96,3 +122,6 @@ To add a new architecture: devstral gpt_oss qwen3 + llama + phi3 + mixtral \ No newline at end of file diff --git a/docs/source/recipes/llama.rst b/docs/source/recipes/llama.rst new file mode 100644 index 00000000000..d7fafb5575b --- /dev/null +++ b/docs/source/recipes/llama.rst @@ -0,0 +1,132 @@ +.. _recipe_llama: + +LlamaForCausalLM +==================== + +Validated models +---------------- + +- `meta-llama/Meta-Llama-3.1-8B `_ +- `meta-llama/Meta-Llama-3.1-8B-Instruct `_ +- `meta-llama/Meta-Llama-3.1-70B `_ +- `meta-llama/Meta-Llama-3.1-70B-Instruct `_ + +.. tab-set:: + :sync-group: engine + + .. tab-item:: vLLM + + **Engine documentation:** + `LlamaForCausalLM in vLLM supported models + `_ + (architecture `LlamaForCausalLM`). + + **Status:** Validated with LMCache. + + Apply for access on the model card page and add your + `huggingface token `_ + as an environment variable: + + .. code-block:: bash + + export HUGGING_FACE_HUB_TOKEN=hf_xxxxxxxxxxxxxxxxx + + | + + Start the LMCache MP server: + + .. code-block:: bash + + lmcache server --l1-size-gb 100 --eviction-policy LRU + + | + + Get the chat templates for tool calling by following the `Llama tool calling guide `_ from vLLM. + + Start vLLM with the LMCache MP connector: + + **Meta-Llama-3.1-8B** (1 GPU): + + .. code-block:: bash + + vllm serve meta-llama/Meta-Llama-3.1-8B \ + --trust-remote-code \ + --kv-transfer-config \ + '{"kv_connector":"LMCacheMPConnector", "kv_role":"kv_both"}' + + | + + **Meta-Llama-3.1-8B-Instruct** (1 GPU): + + .. code-block:: bash + + vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct \ + --trust-remote-code \ + --enable-auto-tool-choice \ + --tool-call-parser llama3_json \ + --chat-template \ + --kv-transfer-config \ + '{"kv_connector":"LMCacheMPConnector", "kv_role":"kv_both"}' + + | + + **Meta-Llama-3.1-70B** (4 GPUs): + + .. code-block:: bash + + vllm serve meta-llama/Meta-Llama-3.1-70B \ + --tensor-parallel-size 4 \ + --trust-remote-code \ + --kv-transfer-config \ + '{"kv_connector":"LMCacheMPConnector", "kv_role":"kv_both"}' + + | + + **Meta-Llama-3.1-70B-Instruct** (4 GPUs): + + .. code-block:: bash + + vllm serve meta-llama/Meta-Llama-3.1-70B-Instruct \ + --tensor-parallel-size 4 \ + --trust-remote-code \ + --enable-auto-tool-choice \ + --tool-call-parser llama3_json \ + --chat-template \ + --kv-transfer-config \ + '{"kv_connector":"LMCacheMPConnector", "kv_role":"kv_both"}' + + | + + Adjust ``--tensor-parallel-size`` to match your hardware. For the + generic LMCache + vLLM wiring (ports, remote hosts, in-process mode), + see :doc:`../mp/quickstart`. + + .. tab-item:: SGLang + + **Status:** Not validated with LMCache. + + .. tab-item:: TRT-LLM + + **Status:** Not supported. LMCache TRT-LLM integration is in progress. + +CacheBlend support +------------------ + +Compression support +------------------- + +.. list-table:: + :header-rows: 1 + :widths: 25 20 55 + + * - Method + - Status + - Notes + * - :doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>` + - Not validated + - + +Caveats +------- + +None known. \ No newline at end of file diff --git a/docs/source/recipes/mixtral.rst b/docs/source/recipes/mixtral.rst new file mode 100644 index 00000000000..7e94c6feebe --- /dev/null +++ b/docs/source/recipes/mixtral.rst @@ -0,0 +1,92 @@ +.. _recipe_mixtral: + +MixtralForCausalLM +==================== + +Validated models +---------------- + +- `mistralai/Mixtral-8x7B-v0.1 `_ +- `mistralai/Mixtral-8x7B-Instruct-v0.1 `_ + +.. tab-set:: + :sync-group: engine + + .. tab-item:: vLLM + + **Engine documentation:** + `MixtralForCausalLM in vLLM supported models + `_ + (architecture `MixtralForCausalLM`). + + **Status:** Validated with LMCache. + + Start the LMCache MP server: + + .. code-block:: bash + + lmcache server --l1-size-gb 100 --eviction-policy LRU + + | + + Start vLLM with the LMCache MP connector: + + **Mixtral-8x7B-v0.1** (4 GPUs): + + .. code-block:: bash + + vllm serve mistralai/Mixtral-8x7B-v0.1 \ + --tensor-parallel-size 4 \ + --trust-remote-code \ + --kv-transfer-config \ + '{"kv_connector":"LMCacheMPConnector", "kv_role":"kv_both"}' + + | + + **Mixtral-8x7B-Instruct-v0.1** (4 GPUs): + + .. code-block:: bash + + vllm serve mistralai/Mixtral-8x7B-Instruct-v0.1 \ + --tensor-parallel-size 4 \ + --trust-remote-code \ + --enable-auto-tool-choice \ + --tool-call-parser mistral \ + --kv-transfer-config \ + '{"kv_connector":"LMCacheMPConnector", "kv_role":"kv_both"}' + + | + + Adjust ``--tensor-parallel-size`` to match your hardware. For the + generic LMCache + vLLM wiring (ports, remote hosts, in-process mode), + see :doc:`../mp/quickstart`. + + .. tab-item:: SGLang + + **Status:** Not validated with LMCache. + + .. tab-item:: TRT-LLM + + **Status:** Not supported. LMCache TRT-LLM integration is in progress. + +CacheBlend support +------------------ + +Compression support +------------------- + +.. list-table:: + :header-rows: 1 + :widths: 25 20 55 + + * - Method + - Status + - Notes + * - :doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>` + - Not validated + - + +Caveats +------- + +None known. \ No newline at end of file diff --git a/docs/source/recipes/phi3.rst b/docs/source/recipes/phi3.rst new file mode 100644 index 00000000000..743bf359060 --- /dev/null +++ b/docs/source/recipes/phi3.rst @@ -0,0 +1,90 @@ +.. _recipe_phi3: + +Phi3ForCausalLM +==================== + +Validated models +---------------- + +- `microsoft/Phi-4-mini-instruct `_ +- `microsoft/Phi-3-medium-128k-instruct `_ + +.. tab-set:: + :sync-group: engine + + .. tab-item:: vLLM + + **Engine documentation:** + `Phi3ForCausalLM in vLLM supported models + `_ + (architecture `Phi3ForCausalLM`). + + **Status:** Validated with LMCache. + + Start the LMCache MP server: + + .. code-block:: bash + + lmcache server --l1-size-gb 100 --eviction-policy LRU + + | + + Start vLLM with the LMCache MP connector: + + **Phi-4-mini-instruct** (1 GPU): + + .. code-block:: bash + + vllm serve microsoft/Phi-4-mini-instruct \ + --trust-remote-code \ + --enable-auto-tool-choice \ + --tool-call-parser phi4_mini_json \ + --kv-transfer-config \ + '{"kv_connector":"LMCacheMPConnector", "kv_role":"kv_both"}' + + | + + **Phi-3-medium-128k-instruct** (1 GPU): + + .. code-block:: bash + + vllm serve microsoft/Phi-3-medium-128k-instruct \ + --trust-remote-code \ + --kv-transfer-config \ + '{"kv_connector":"LMCacheMPConnector", "kv_role":"kv_both"}' + + | + + Adjust ``--tensor-parallel-size`` to match your hardware. For the + generic LMCache + vLLM wiring (ports, remote hosts, in-process mode), + see :doc:`../mp/quickstart`. + + .. tab-item:: SGLang + + **Status:** Not validated with LMCache. + + .. tab-item:: TRT-LLM + + **Status:** Not supported. LMCache TRT-LLM integration is in progress. + +CacheBlend support +------------------ + +Compression support +------------------- + +.. list-table:: + :header-rows: 1 + :widths: 25 20 55 + + * - Method + - Status + - Notes + * - :doc:`CacheGen <../kv_cache_optimizations/compression/cachegen>` + - Not validated + - + +Caveats +------- + +None known. \ No newline at end of file From 4fb03710be6453e5050419172e8ec2ef8d1b2187 Mon Sep 17 00:00:00 2001 From: Zhengfei He <157287166+zhengfeihe@users.noreply.github.com> Date: Wed, 27 May 2026 00:59:21 +0900 Subject: [PATCH 62/69] [Perf] Replace Condvar polling with eventfd + epoll in iouring worker thread (#3271) * Replace Condvar polling with eventfd+epoll for io_uring worker Signed-off-by: zhengfeihe * Fix rust format check Signed-off-by: zhengfeihe * Add RAII guard for file descriptor in rust raw block Signed-off-by: zhengfeihe --------- Signed-off-by: zhengfeihe --- rust/raw_block/src/lib.rs | 985 ++++++++++++++++++++++++-------------- 1 file changed, 617 insertions(+), 368 deletions(-) diff --git a/rust/raw_block/src/lib.rs b/rust/raw_block/src/lib.rs index 62551e2efab..f170de532aa 100644 --- a/rust/raw_block/src/lib.rs +++ b/rust/raw_block/src/lib.rs @@ -20,6 +20,7 @@ use pyo3::prelude::*; use pyo3::types::PyAny; use std::collections::HashMap; use std::ffi::CString; +use std::io; use std::os::unix::io::RawFd; use std::slice; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -297,6 +298,174 @@ impl IoCompletion { } } +/// Manages io_uring worker thread notification, using one `epoll` instance +/// over two eventfds to wait on two event sources at once: a producer-side +/// eventfd signalled when Python pushes a new submission, and a CQ-side +/// eventfd signalled by the kernel when a completion is posted. +struct UringNotify { + /// Epoll instance watching both `producer_efd` and `cq_efd`. A single + /// `epoll_wait` on this fd blocks until either eventfd becomes readable, + /// so the worker can react to user-space queue pushes and kernel CQE + /// posts from the same call site. + epoll_fd: RawFd, + /// Eventfd written by Python producer threads after pushing into the + /// submission queue. Replaces the `Condvar::notify_one` used in the + /// pre-eventfd design. Also written by `do_close` so the worker can + /// break out of `epoll_wait` and observe the shutdown flag. + producer_efd: RawFd, + /// Eventfd registered with the io_uring instance via + /// `Submitter::register_eventfd`. The kernel writes to it whenever a + /// CQE is posted, so the worker is woken without having to drain the + /// completion queue speculatively. + cq_efd: RawFd, +} + +impl UringNotify { + /// Builds the three fds (two eventfds + one epoll) and wires the + /// eventfds into the epoll instance. Cleans up partially-built state + /// on any error path so no fd leaks if construction fails midway. + fn new() -> io::Result { + // Producer-side eventfd. Counter starts at 0 (no pending events). + // EFD_CLOEXEC prevents leaking the fd into a child process via + // execve. EFD_NONBLOCK makes read() return EAGAIN (instead of + // blocking) when the counter is already 0; wait() relies on this + // non-blocking behaviour to drain safely without hanging. + let producer_efd = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC | libc::EFD_NONBLOCK) }; + if producer_efd < 0 { + // Nothing else has been allocated yet -- just bubble the error. + return Err(io::Error::last_os_error()); + } + + // CQ-side eventfd. The kernel writes to this one after + // register_eventfd() is called on the io_uring instance. + let cq_efd = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC | libc::EFD_NONBLOCK) }; + if cq_efd < 0 { + // producer_efd is live -- close it before bubbling the error. + let e = io::Error::last_os_error(); + unsafe { libc::close(producer_efd) }; + return Err(e); + } + + // Epoll instance fd. EPOLL_CLOEXEC for the same hygiene reason as + // EFD_CLOEXEC above. + let epoll_fd = unsafe { libc::epoll_create1(libc::EPOLL_CLOEXEC) }; + if epoll_fd < 0 { + // Both eventfds are live; close them before returning. + let e = io::Error::last_os_error(); + unsafe { + libc::close(producer_efd); + libc::close(cq_efd); + } + return Err(e); + } + + // Register both eventfds with the epoll instance so epoll_wait + // can block on either source. + // + // We use EPOLLIN: wake when the counter becomes non-zero + // (someone signalled). Alternatives we deliberately don't use: + // - EPOLLOUT (writable): pointless -- eventfd is effectively + // always writable, so it would just busy-loop epoll_wait. + // - EPOLLET (edge-triggered): would require fully draining + // every wake-up in one go to avoid missing edges. Default + // level-triggered fits our drain pattern in wait(). + // - EPOLLONESHOT: would auto-disarm after each fire and force + // us to re-register; not worth the complexity. + // u64 stores the fd value itself so wait() can identify which + // fd fired without keeping a side map. + for fd in [producer_efd, cq_efd] { + let mut ev = libc::epoll_event { + events: libc::EPOLLIN as u32, + u64: fd as u64, + }; + let rc = unsafe { libc::epoll_ctl(epoll_fd, libc::EPOLL_CTL_ADD, fd, &mut ev) }; + if rc < 0 { + // All three fds are live; clean up all of them. + let e = io::Error::last_os_error(); + unsafe { + libc::close(epoll_fd); + libc::close(producer_efd); + libc::close(cq_efd); + } + return Err(e); + } + } + + // Ownership of all three fds moves into Self; Drop handles the + // happy-path close. + Ok(Self { + epoll_fd, + producer_efd, + cq_efd, + }) + } + + /// Wakes the worker thread by writing 1 to `producer_efd`. Called by + /// producers after pushing a submission into the queue and by `do_close` + /// to break the worker out of `epoll_wait` for shutdown. + fn signal_producer(&self) { + let v: u64 = 1; + // Given EFD_NONBLOCK and counter < u64::MAX - 1, the 8-byte write + // always succeeds, so the return value is intentionally ignored. + unsafe { + libc::write( + self.producer_efd, + &v as *const u64 as *const libc::c_void, + 8, + ); + } + } + + /// Blocks the worker until either eventfd is readable, then drains + /// each fired fd. Drain is required because epoll is level-triggered: + /// without consuming the counter, the next epoll_wait would return + /// immediately on the same already-handled signal. + fn wait(&self) { + // A capacity of 2 is enough: only two fds are registered with this + // epoll instance, so at most two events can come back per call. + let mut events = [libc::epoll_event { events: 0, u64: 0 }; 2]; + + // Timeout = -1 means "block indefinitely". Shutdown wakes us by + // writing producer_efd from do_close, so we never need a timeout. + let n = unsafe { libc::epoll_wait(self.epoll_fd, events.as_mut_ptr(), 2, -1) }; + + // n < 0 is usually EINTR (signal interruption); we just return and + // the worker's outer loop will call wait() again. n == 0 should + // not happen with timeout=-1 but is handled defensively. + if n <= 0 { + return; + } + + // For each event reported, read 8 bytes from the corresponding + // eventfd to reset its counter to 0. The fd value was stashed in + // ev.u64 during epoll_ctl registration. We discard the read value + // (we only care that a signal arrived, not how many). + let mut buf = [0u8; 8]; + for ev in &events[..n as usize] { + let fd = ev.u64 as RawFd; + // Discard the result. The wake-up was already delivered by + // epoll_wait; this read only exists to reset the eventfd + // counter so epoll stops reporting the fd as readable. If it + // fails, the worst case is one spurious wake-up next + // iteration. No work is lost, because the real submissions + // live in the queue and CQ, not in the bytes we just read. + unsafe { + libc::read(fd, buf.as_mut_ptr() as *mut libc::c_void, 8); + } + } + } +} + +impl Drop for UringNotify { + fn drop(&mut self) { + unsafe { + libc::close(self.epoll_fd); + libc::close(self.producer_efd); + libc::close(self.cq_efd); + } + } +} + /// Represents a single I/O submission to io_uring. /// /// This struct is sent from Python threads to the worker thread via a queue. @@ -381,8 +550,10 @@ struct RawBlockDevice { // Per-batch in-flight count tracking // Maps batch_id -> (in_flight_count, condition_variable) batch_in_flight: Arc>>, - // Signal to wake up worker when new requests are available - batch_ready: Option>, + // Worker wake-up: producer eventfd + ring CQ eventfd, both polled via + // a single epoll_fd. Replaces the previous `Arc` which couldn't + // be signaled from the kernel-side completion queue. + batch_ready: Option>, // Store Python buffer objects for writes, reads to keep them alive until they complete // This prevents premature garbage collection while io_uring is using the buffers // Keyed by batch_id to isolate concurrent batches @@ -394,6 +565,42 @@ struct RawBlockDevice { next_batch_id: Arc, } +/// RAII guard for a raw file descriptor +struct FdGuard { + fd: RawFd, +} + +impl FdGuard { + /// Takes ownership of an open file descriptor. + /// + /// Args: + /// - `fd`: a descriptor returned by a successful `open()`. + fn new(fd: RawFd) -> Self { + FdGuard { fd } + } + + /// Releases ownership of the fd to the caller, disarming the guard so the + /// descriptor is not closed on drop. + /// + /// Returns the raw descriptor, now owned by the caller. + fn disarm(self) -> RawFd { + let fd = self.fd; + std::mem::forget(self); + fd + } +} + +impl Drop for FdGuard { + fn drop(&mut self) { + // SAFETY: `fd` was returned by a successful `open()` and ownership has + // not been released via `disarm()`, so this is the only close of this + // descriptor. + unsafe { + libc::close(self.fd); + } + } +} + impl RawBlockDevice { /// Internal constructor performs all low level setup. fn new_internal( @@ -421,6 +628,10 @@ impl RawBlockDevice { if fd < 0 { return Err(os_err("open failed")); } + // Take ownership of the fd so it is closed if any fallible setup step + // below returns early before the fd is moved into the RawBlockDevice. + // Disarmed once the struct is successfully constructed. + let fd_guard = FdGuard::new(fd); let size = fd_size_bytes(fd)?; let ( @@ -438,10 +649,18 @@ impl RawBlockDevice { ) = if use_iouring { let ring = IoUring::new(iouring_queue_depth as u32) .map_err(|e| PyRuntimeError::new_err(format!("io_uring init failed: {}", e)))?; + let notify = UringNotify::new() + .map_err(|e| PyRuntimeError::new_err(format!("UringNotify init failed: {}", e)))?; + // Register the CQ eventfd with the ring so the kernel writes to it + // whenever a CQE is posted. Must happen before the ring is wrapped + // in a Mutex / handed to the worker. + ring.submitter() + .register_eventfd(notify.cq_efd) + .map_err(|e| PyRuntimeError::new_err(format!("register_eventfd failed: {}", e)))?; let ring = Arc::new(Mutex::new(ring)); let queue = Arc::new(Mutex::new(Vec::::new())); let shutdown = Arc::new(AtomicBool::new(false)); - let batch_ready = Arc::new(Condvar::new()); + let batch_ready = Arc::new(notify); let in_flight_count = Arc::new(AtomicU64::new(0)); let in_flight_cvar = Arc::new(Condvar::new()); let batched_buffer_objs = Arc::new(Mutex::new(HashMap::>>::new())); @@ -476,48 +695,127 @@ impl RawBlockDevice { // - Reads from the submission queue // - Submits to io_uring // - Processes completions - let worker = thread::spawn(move || { - let mut in_flight: HashMap = HashMap::new(); - let mut next_user_data: u64 = 1; - - while !shutdown_clone.load(Ordering::Relaxed) { - // This drains all completed I/O operations from the completion queue (CQ). - // For each completion: - // - Remove the request from our in_flight tracking HashMap - // - Signal the waiting Python thread via IoCompletion - // - Decrement the in_flight_count atomic - // - Wake up any threads waiting for all I/O to complete - { - let mut ring = ring_clone.lock().unwrap(); - let completions: Vec<_> = ring.completion().collect(); - for cqe in completions { - let user_data = cqe.user_data(); - if let Some(mut sub) = in_flight.remove(&user_data) { - let batch_id = sub.batch_id; - if cqe.result() < 0 { - let code = -cqe.result(); - // Drop any bounce buffer associated with this submission. - let _ = sub.bounce.take(); - sub.completion - .set(Err(PyOSError::new_err((code, "io_uring I/O error")))); - } else { - let bytes_transferred = cqe.result() as usize; - if bytes_transferred < sub.len { - // Short read/write: update offset and length, then resubmit - sub.offset += bytes_transferred as u64; - sub.len -= bytes_transferred; - // Update buffer pointer for writes and direct reads - if sub.is_write || sub.bounce.is_none() { - sub.ptr_addr += bytes_transferred; + let worker = thread::Builder::new() + .name("rust-rawblock-uring".into()) + .spawn(move || { + let mut in_flight: HashMap = HashMap::new(); + let mut next_user_data: u64 = 1; + + while !shutdown_clone.load(Ordering::Relaxed) { + // This drains all completed I/O operations from the completion queue (CQ). + // For each completion: + // - Remove the request from our in_flight tracking HashMap + // - Signal the waiting Python thread via IoCompletion + // - Decrement the in_flight_count atomic + // - Wake up any threads waiting for all I/O to complete + { + let mut ring = ring_clone.lock().unwrap(); + let completions: Vec<_> = ring.completion().collect(); + for cqe in completions { + let user_data = cqe.user_data(); + if let Some(mut sub) = in_flight.remove(&user_data) { + let batch_id = sub.batch_id; + if cqe.result() < 0 { + let code = -cqe.result(); + // Drop any bounce buffer associated with this submission. + let _ = sub.bounce.take(); + sub.completion.set(Err(PyOSError::new_err(( + code, + "io_uring I/O error", + )))); + } else { + let bytes_transferred = cqe.result() as usize; + if bytes_transferred < sub.len { + // Short read/write: update offset and length, then resubmit + sub.offset += bytes_transferred as u64; + sub.len -= bytes_transferred; + // Update buffer pointer for writes and direct reads + if sub.is_write || sub.bounce.is_none() { + sub.ptr_addr += bytes_transferred; + } + // For read with bounce buffer, copy partial data back + if !sub.is_write { + if let ( + Some(bounce), + Some(orig_ptr), + Some(payload_len), + ) = ( + sub.bounce.as_ref(), + sub.original_ptr, + sub.payload_len, + ) { + unsafe { + libc::memcpy( + orig_ptr as *mut libc::c_void, + bounce.as_ptr() as *const libc::c_void, + bytes_transferred.min(payload_len), + ); + } + sub.original_ptr = + Some(orig_ptr + bytes_transferred); + sub.payload_len = Some( + payload_len + .saturating_sub(bytes_transferred), + ); + } + } + // Re-insert into in_flight with updated values + // Don't decrement in_flight_count since we're resubmitting + in_flight.insert(user_data, sub.clone()); + // Push a new SQE for the remaining data + let ptr = sub.ptr_addr as *mut u8; + let sqe = if sub.is_write { + if let Some(idx) = sub.fixed_buffer_idx { + opcode::WriteFixed::new( + Fd(sub.fd), + ptr as *const u8, + sub.len as u32, + idx, + ) + .offset(sub.offset) + .build() + } else { + opcode::Write::new( + Fd(sub.fd), + ptr as *const u8, + sub.len as u32, + ) + .offset(sub.offset) + .build() + } + } else if let Some(idx) = sub.fixed_buffer_idx { + opcode::ReadFixed::new( + Fd(sub.fd), + ptr, + sub.len as u32, + idx, + ) + .offset(sub.offset) + .build() + } else { + opcode::Read::new(Fd(sub.fd), ptr, sub.len as u32) + .offset(sub.offset) + .build() + }; + let sqe = sqe.user_data(user_data); + unsafe { + ring.submission().push(&sqe).expect( + "failed to push sqe for short read/write", + ); + } + // Submit the new SQE to the kernel + let _ = ring.submitter().submit(); + continue; } - // For read with bounce buffer, copy partial data back + // Full completion + // For reads with bounce buffer, copy data back to original buffer if !sub.is_write { if let ( Some(bounce), Some(orig_ptr), Some(payload_len), ) = ( - sub.bounce.as_ref(), + sub.bounce.take(), sub.original_ptr, sub.payload_len, ) { @@ -525,262 +823,199 @@ impl RawBlockDevice { libc::memcpy( orig_ptr as *mut libc::c_void, bounce.as_ptr() as *const libc::c_void, - bytes_transferred.min(payload_len), + payload_len, ); } - sub.original_ptr = - Some(orig_ptr + bytes_transferred); - sub.payload_len = Some( - payload_len.saturating_sub(bytes_transferred), - ); } - } - // Re-insert into in_flight with updated values - // Don't decrement in_flight_count since we're resubmitting - in_flight.insert(user_data, sub.clone()); - // Push a new SQE for the remaining data - let ptr = sub.ptr_addr as *mut u8; - let sqe = if sub.is_write { - if let Some(idx) = sub.fixed_buffer_idx { - opcode::WriteFixed::new( - Fd(sub.fd), - ptr as *const u8, - sub.len as u32, - idx, - ) - .offset(sub.offset) - .build() - } else { - opcode::Write::new( - Fd(sub.fd), - ptr as *const u8, - sub.len as u32, - ) - .offset(sub.offset) - .build() - } - } else if let Some(idx) = sub.fixed_buffer_idx { - opcode::ReadFixed::new( - Fd(sub.fd), - ptr, - sub.len as u32, - idx, - ) - .offset(sub.offset) - .build() } else { - opcode::Read::new(Fd(sub.fd), ptr, sub.len as u32) - .offset(sub.offset) - .build() - }; - let sqe = sqe.user_data(user_data); - unsafe { - ring.submission() - .push(&sqe) - .expect("failed to push sqe for short read/write"); + // Drop any bounce buffer associated with this submission. + let _ = sub.bounce.take(); } - // Submit the new SQE to the kernel - let _ = ring.submitter().submit(); - continue; + sub.completion.set(Ok(())); + } + let prev = + in_flight_count_clone.fetch_sub(1, Ordering::Relaxed); + if prev == 1 { + in_flight_cvar_clone.notify_all(); } - // Full completion - // For reads with bounce buffer, copy data back to original buffer - if !sub.is_write { - if let (Some(bounce), Some(orig_ptr), Some(payload_len)) = - (sub.bounce.take(), sub.original_ptr, sub.payload_len) + // Decrement per-batch in-flight count and notify if batch is complete + if batch_id != 0 { + let batch_map = batch_in_flight_clone.lock().unwrap(); + if let Some((batch_count, batch_cvar)) = + batch_map.get(&batch_id) { - unsafe { - libc::memcpy( - orig_ptr as *mut libc::c_void, - bounce.as_ptr() as *const libc::c_void, - payload_len, - ); + let prev_batch = + batch_count.fetch_sub(1, Ordering::Relaxed); + if prev_batch == 1 { + batch_cvar.notify_all(); } } - } else { - // Drop any bounce buffer associated with this submission. - let _ = sub.bounce.take(); - } - sub.completion.set(Ok(())); - } - let prev = in_flight_count_clone.fetch_sub(1, Ordering::Relaxed); - if prev == 1 { - in_flight_cvar_clone.notify_all(); - } - // Decrement per-batch in-flight count and notify if batch is complete - if batch_id != 0 { - let batch_map = batch_in_flight_clone.lock().unwrap(); - if let Some((batch_count, batch_cvar)) = - batch_map.get(&batch_id) - { - let prev_batch = - batch_count.fetch_sub(1, Ordering::Relaxed); - if prev_batch == 1 { - batch_cvar.notify_all(); - } } } } + ring.submission().sync(); } - ring.submission().sync(); - } - // We use a condition variable with a short timeout (10 microseconds). - // This allows us to: - // - Quickly respond to new requests (batched from Python) - // - Periodically check for shutdown signal - // - Not spin aggressively (which would waste CPU) - let timeout = Duration::from_micros(10); - let q = queue_clone.lock().unwrap(); - let (mut q, _) = batch_ready_clone - .wait_timeout_while(q, timeout, |q| { - q.is_empty() && !shutdown_clone.load(Ordering::Relaxed) - }) - .unwrap(); - - if !q.is_empty() { - // Take all pending requests from our queue and submit them to io_uring. - // - // - Remove all pending requests from queue - // - Check how much space is available in the ring (max 256 entries) - // - If batch is larger than available space, put excess back in queue - // - Increment in_flight_count for each request we're about to submit - // - Build SQE (Submission Queue Entry) for each request - // - Push SQEs to the ring - // - Call submit() to send them to the kernel - // - // Fixed Buffer Support: - // - If the buffer was pre-registered with register_fixed_buffers(), - // we use ReadFixed/WriteFixed for true zero-copy I/O - // - Otherwise we use regular Read/Write with user-space pointers - let mut batch: Vec = std::mem::take(&mut *q); - let batch_len = batch.len(); - - let mut ring = ring_clone.lock().unwrap(); - - let available = ring_size - ring.submission().len(); - let to_submit_count = std::cmp::min(available, batch_len); - - if to_submit_count < batch_len { - let remaining: Vec<_> = batch[to_submit_count..].to_vec(); - if !remaining.is_empty() { - q.extend(remaining); - } + // Block on epoll only if there's truly nothing pending. The empty + + // shutdown checks short-circuit so we don't sleep when a producer or + // do_close() already left work for us. Race-free against a late + // signal_producer(): eventfd is a counter, so a wake-up between the + // check and wait() is buffered, not lost. + if !shutdown_clone.load(Ordering::Relaxed) + && queue_clone.lock().unwrap().is_empty() + { + batch_ready_clone.wait(); } - drop(q); - - // Track user_data values for each submission to clean up in_flight entries - // if submit() fails or returns partial count - let mut user_data_list: Vec = Vec::with_capacity(to_submit_count); - for sub in batch.iter().take(to_submit_count) { - let user_data = next_user_data; - next_user_data = next_user_data.wrapping_add(1); - user_data_list.push(user_data); - in_flight.insert(user_data, sub.clone()); - - let ptr = sub.ptr_addr as *mut u8; - let sqe = if sub.is_write { - if let Some(idx) = sub.fixed_buffer_idx { - opcode::WriteFixed::new( - Fd(sub.fd), - ptr as *const u8, - sub.len as u32, - idx, - ) - .offset(sub.offset) - .build() - } else { - opcode::Write::new(Fd(sub.fd), ptr as *const u8, sub.len as u32) - .offset(sub.offset) - .build() + let mut q = queue_clone.lock().unwrap(); + if !q.is_empty() { + // Take all pending requests from our queue and submit them to io_uring. + // + // - Remove all pending requests from queue + // - Check how much space is available in the ring (max 256 entries) + // - If batch is larger than available space, put excess back in queue + // - Increment in_flight_count for each request we're about to submit + // - Build SQE (Submission Queue Entry) for each request + // - Push SQEs to the ring + // - Call submit() to send them to the kernel + // + // Fixed Buffer Support: + // - If the buffer was pre-registered with register_fixed_buffers(), + // we use ReadFixed/WriteFixed for true zero-copy I/O + // - Otherwise we use regular Read/Write with user-space pointers + let mut batch: Vec = std::mem::take(&mut *q); + let batch_len = batch.len(); + + let mut ring = ring_clone.lock().unwrap(); + + let available = ring_size - ring.submission().len(); + let to_submit_count = std::cmp::min(available, batch_len); + + if to_submit_count < batch_len { + let remaining: Vec<_> = batch[to_submit_count..].to_vec(); + if !remaining.is_empty() { + q.extend(remaining); } - } else if let Some(idx) = sub.fixed_buffer_idx { - opcode::ReadFixed::new(Fd(sub.fd), ptr, sub.len as u32, idx) - .offset(sub.offset) - .build() - } else { - opcode::Read::new(Fd(sub.fd), ptr, sub.len as u32) - .offset(sub.offset) - .build() - }; - let sqe = sqe.user_data(user_data); - unsafe { - ring.submission().push(&sqe).expect("failed to push sqe"); } - } - let submit_result = ring.submitter().submit(); - // Handle EAGAIN (ring full) and EINTR (interrupted syscall) - match submit_result { - Ok(submitted) => { - // Any remaining requests in batch that weren't submitted - // will be retried in the next iteration of the loop - if submitted < to_submit_count { - // Remove in_flight entries for unsubmitted requests - for user_data in user_data_list[submitted..].iter() { - in_flight.remove(user_data); - } - // Put unsubmitted requests back in the queue for retry - let unsubmitted: Vec<_> = - batch[submitted..to_submit_count].to_vec(); - if !unsubmitted.is_empty() { - drop(ring); - let mut q = queue_clone.lock().unwrap(); - // Insert unsubmitted requests back at the front preserving order - q.splice(0..0, unsubmitted); + drop(q); + + // Track user_data values for each submission to clean up in_flight entries + // if submit() fails or returns partial count + let mut user_data_list: Vec = Vec::with_capacity(to_submit_count); + for sub in batch.iter().take(to_submit_count) { + let user_data = next_user_data; + next_user_data = next_user_data.wrapping_add(1); + user_data_list.push(user_data); + in_flight.insert(user_data, sub.clone()); + + let ptr = sub.ptr_addr as *mut u8; + let sqe = if sub.is_write { + if let Some(idx) = sub.fixed_buffer_idx { + opcode::WriteFixed::new( + Fd(sub.fd), + ptr as *const u8, + sub.len as u32, + idx, + ) + .offset(sub.offset) + .build() + } else { + opcode::Write::new( + Fd(sub.fd), + ptr as *const u8, + sub.len as u32, + ) + .offset(sub.offset) + .build() } + } else if let Some(idx) = sub.fixed_buffer_idx { + opcode::ReadFixed::new(Fd(sub.fd), ptr, sub.len as u32, idx) + .offset(sub.offset) + .build() + } else { + opcode::Read::new(Fd(sub.fd), ptr, sub.len as u32) + .offset(sub.offset) + .build() + }; + let sqe = sqe.user_data(user_data); + unsafe { + ring.submission().push(&sqe).expect("failed to push sqe"); } } - Err(e) => { - // Handle submission errors - let error_code = e.raw_os_error(); - match error_code { - Some(libc::EAGAIN) | Some(libc::EINTR) => { - // Ring is full, or the operation was interrupted due - // to signal. We need to wait for completions and then retry - // Remove in_flight entries for all submissions in this batch - for user_data in user_data_list.iter() { + + let submit_result = ring.submitter().submit(); + // Handle EAGAIN (ring full) and EINTR (interrupted syscall) + match submit_result { + Ok(submitted) => { + // Any remaining requests in batch that weren't submitted + // will be retried in the next iteration of the loop + if submitted < to_submit_count { + // Remove in_flight entries for unsubmitted requests + for user_data in user_data_list[submitted..].iter() { in_flight.remove(user_data); } - // Put unsubmitted requests back in queue for next iteration - if to_submit_count > 0 { - let unsubmitted: Vec<_> = - batch[..to_submit_count].to_vec(); + // Put unsubmitted requests back in the queue for retry + let unsubmitted: Vec<_> = + batch[submitted..to_submit_count].to_vec(); + if !unsubmitted.is_empty() { drop(ring); let mut q = queue_clone.lock().unwrap(); // Insert unsubmitted requests back at the front preserving order q.splice(0..0, unsubmitted); } } - _ => { - // Error: fail all pending submissions in this batch. - // Remove in_flight entries since these won't generate completions - for user_data in user_data_list.iter() { - in_flight.remove(user_data); + } + Err(e) => { + // Handle submission errors + let error_code = e.raw_os_error(); + match error_code { + Some(libc::EAGAIN) | Some(libc::EINTR) => { + // Ring is full, or the operation was interrupted due + // to signal. We need to wait for completions and then retry + // Remove in_flight entries for all submissions in this batch + for user_data in user_data_list.iter() { + in_flight.remove(user_data); + } + // Put unsubmitted requests back in queue for next iteration + if to_submit_count > 0 { + let unsubmitted: Vec<_> = + batch[..to_submit_count].to_vec(); + drop(ring); + let mut q = queue_clone.lock().unwrap(); + // Insert unsubmitted requests back at the front preserving order + q.splice(0..0, unsubmitted); + } } - for sub in batch.iter_mut().take(to_submit_count) { - let batch_id = sub.batch_id; - sub.completion.set(Err(PyRuntimeError::new_err( - format!("io_uring submit error: {:?}", e), - ))); - let _ = sub.bounce.take(); - let prev = in_flight_count_clone - .fetch_sub(1, Ordering::Relaxed); - if prev == 1 { - in_flight_cvar_clone.notify_all(); + _ => { + // Error: fail all pending submissions in this batch. + // Remove in_flight entries since these won't generate completions + for user_data in user_data_list.iter() { + in_flight.remove(user_data); } - // Decrement per-batch in-flight count and notify if batch is complete - if batch_id != 0 { - let batch_map = - batch_in_flight_clone.lock().unwrap(); - if let Some((batch_count, batch_cvar)) = - batch_map.get(&batch_id) - { - let prev_batch = - batch_count.fetch_sub(1, Ordering::Relaxed); - if prev_batch == 1 { - batch_cvar.notify_all(); + for sub in batch.iter_mut().take(to_submit_count) { + let batch_id = sub.batch_id; + sub.completion.set(Err(PyRuntimeError::new_err( + format!("io_uring submit error: {:?}", e), + ))); + let _ = sub.bounce.take(); + let prev = in_flight_count_clone + .fetch_sub(1, Ordering::Relaxed); + if prev == 1 { + in_flight_cvar_clone.notify_all(); + } + // Decrement per-batch in-flight count and notify if batch is complete + if batch_id != 0 { + let batch_map = + batch_in_flight_clone.lock().unwrap(); + if let Some((batch_count, batch_cvar)) = + batch_map.get(&batch_id) + { + let prev_batch = batch_count + .fetch_sub(1, Ordering::Relaxed); + if prev_batch == 1 { + batch_cvar.notify_all(); + } } } } @@ -790,131 +1025,140 @@ impl RawBlockDevice { } } } - } - // SHUTDOWN: Wake up all waiting Python threads - // Drain the queue and wake up all waiting threads with error - { - let mut q = queue_clone - .lock() - .expect("Worker: queue mutex poisoned during shutdown"); - while let Some(mut sub) = q.pop() { - let batch_id = sub.batch_id; - // Drop any bounce buffer associated with this submission. - let _ = sub.bounce.take(); - in_flight_count_clone.fetch_sub(1, Ordering::Relaxed); - sub.completion.set(Err(PyRuntimeError::new_err( - "io_uring worker shutting down", - ))); - // Decrement per-batch in-flight count and notify if batch is complete - if batch_id != 0 { - let batch_map = batch_in_flight_clone.lock().unwrap(); - if let Some((batch_count, batch_cvar)) = batch_map.get(&batch_id) { - let prev_batch = batch_count.fetch_sub(1, Ordering::Relaxed); - if prev_batch == 1 { - batch_cvar.notify_all(); + // SHUTDOWN: Wake up all waiting Python threads + // Drain the queue and wake up all waiting threads with error + { + let mut q = queue_clone + .lock() + .expect("Worker: queue mutex poisoned during shutdown"); + while let Some(mut sub) = q.pop() { + let batch_id = sub.batch_id; + // Drop any bounce buffer associated with this submission. + let _ = sub.bounce.take(); + in_flight_count_clone.fetch_sub(1, Ordering::Relaxed); + sub.completion.set(Err(PyRuntimeError::new_err( + "io_uring worker shutting down", + ))); + // Decrement per-batch in-flight count and notify if batch is complete + if batch_id != 0 { + let batch_map = batch_in_flight_clone.lock().unwrap(); + if let Some((batch_count, batch_cvar)) = batch_map.get(&batch_id) { + let prev_batch = batch_count.fetch_sub(1, Ordering::Relaxed); + if prev_batch == 1 { + batch_cvar.notify_all(); + } } } } } - } - // Process any remaining in-flight requests - // Wait for kernel to complete the requests or force-cancel them - // Note: This 1000 milliseconds is a rough estimate - let graceful_shutdown = Duration::from_millis(1000); - thread::sleep(graceful_shutdown); - { - let mut ring = ring_clone - .lock() - .expect("Worker: ring mutex poisoned during shutdown"); - for cqe in ring.completion() { - let user_data = cqe.user_data(); - if let Some(mut sub) = in_flight.remove(&user_data) { - let batch_id = sub.batch_id; - if cqe.result() < 0 { - let code = -cqe.result(); - // Drop any bounce buffer associated with this submission. - let _ = sub.bounce.take(); - sub.completion - .set(Err(PyOSError::new_err((code, "io_uring I/O error")))); - } else { - let bytes_transferred = cqe.result() as usize; - if bytes_transferred < sub.len { - // Short read/write during shutdown: fail the request - // We cannot resubmit because the worker is about to exit + // Process any remaining in-flight requests + // Wait for kernel to complete the requests or force-cancel them + // Note: This 1000 milliseconds is a rough estimate + let graceful_shutdown = Duration::from_millis(1000); + thread::sleep(graceful_shutdown); + { + let mut ring = ring_clone + .lock() + .expect("Worker: ring mutex poisoned during shutdown"); + for cqe in ring.completion() { + let user_data = cqe.user_data(); + if let Some(mut sub) = in_flight.remove(&user_data) { + let batch_id = sub.batch_id; + if cqe.result() < 0 { + let code = -cqe.result(); // Drop any bounce buffer associated with this submission. let _ = sub.bounce.take(); - sub.completion.set(Err(PyRuntimeError::new_err( + sub.completion + .set(Err(PyOSError::new_err((code, "io_uring I/O error")))); + } else { + let bytes_transferred = cqe.result() as usize; + if bytes_transferred < sub.len { + // Short read/write during shutdown: fail the request + // We cannot resubmit because the worker is about to exit + // Drop any bounce buffer associated with this submission. + let _ = sub.bounce.take(); + sub.completion.set(Err(PyRuntimeError::new_err( "io_uring worker shutting down - short I/O during shutdown", ))); - // Continue to decrement in_flight_count below - } else { - // Full completion - // For reads with bounce buffer, copy data back to original buffer - if !sub.is_write { - if let (Some(bounce), Some(orig_ptr), Some(payload_len)) = - (sub.bounce.take(), sub.original_ptr, sub.payload_len) - { - unsafe { - libc::memcpy( - orig_ptr as *mut libc::c_void, - bounce.as_ptr() as *const libc::c_void, - payload_len, - ); + // Continue to decrement in_flight_count below + } else { + // Full completion + // For reads with bounce buffer, copy data back to original buffer + if !sub.is_write { + if let ( + Some(bounce), + Some(orig_ptr), + Some(payload_len), + ) = ( + sub.bounce.take(), + sub.original_ptr, + sub.payload_len, + ) { + unsafe { + libc::memcpy( + orig_ptr as *mut libc::c_void, + bounce.as_ptr() as *const libc::c_void, + payload_len, + ); + } } + } else { + // Drop any bounce buffer associated with this submission. + let _ = sub.bounce.take(); } - } else { - // Drop any bounce buffer associated with this submission. - let _ = sub.bounce.take(); + sub.completion.set(Ok(())); } - sub.completion.set(Ok(())); } - } - let prev = in_flight_count_clone.fetch_sub(1, Ordering::Relaxed); - if prev == 1 { - in_flight_cvar_clone.notify_all(); - } - // Decrement per-batch in-flight count and notify if batch is complete - if batch_id != 0 { - let batch_map = batch_in_flight_clone.lock().unwrap(); - if let Some((batch_count, batch_cvar)) = batch_map.get(&batch_id) { - let prev_batch = batch_count.fetch_sub(1, Ordering::Relaxed); - if prev_batch == 1 { - batch_cvar.notify_all(); + let prev = in_flight_count_clone.fetch_sub(1, Ordering::Relaxed); + if prev == 1 { + in_flight_cvar_clone.notify_all(); + } + // Decrement per-batch in-flight count and notify if batch is complete + if batch_id != 0 { + let batch_map = batch_in_flight_clone.lock().unwrap(); + if let Some((batch_count, batch_cvar)) = + batch_map.get(&batch_id) + { + let prev_batch = + batch_count.fetch_sub(1, Ordering::Relaxed); + if prev_batch == 1 { + batch_cvar.notify_all(); + } } } } } + ring.submission().sync(); } - ring.submission().sync(); - } - // Any remaining in_flight requests, force wake with error - // (these were submitted to kernel but won't get completions) - for (_user_data, mut sub) in in_flight.drain() { - let batch_id = sub.batch_id; - // Drop any bounce buffer associated with this submission. - let _ = sub.bounce.take(); - in_flight_count_clone.fetch_sub(1, Ordering::Relaxed); - sub.completion.set(Err(PyRuntimeError::new_err( - "io_uring worker shutting down - request cancelled", - ))); - // Decrement per-batch in-flight count and notify if batch is complete - if batch_id != 0 { - let batch_map = batch_in_flight_clone.lock().unwrap(); - if let Some((batch_count, batch_cvar)) = batch_map.get(&batch_id) { - let prev_batch = batch_count.fetch_sub(1, Ordering::Relaxed); - if prev_batch == 1 { - batch_cvar.notify_all(); + // Any remaining in_flight requests, force wake with error + // (these were submitted to kernel but won't get completions) + for (_user_data, mut sub) in in_flight.drain() { + let batch_id = sub.batch_id; + // Drop any bounce buffer associated with this submission. + let _ = sub.bounce.take(); + in_flight_count_clone.fetch_sub(1, Ordering::Relaxed); + sub.completion.set(Err(PyRuntimeError::new_err( + "io_uring worker shutting down - request cancelled", + ))); + // Decrement per-batch in-flight count and notify if batch is complete + if batch_id != 0 { + let batch_map = batch_in_flight_clone.lock().unwrap(); + if let Some((batch_count, batch_cvar)) = batch_map.get(&batch_id) { + let prev_batch = batch_count.fetch_sub(1, Ordering::Relaxed); + if prev_batch == 1 { + batch_cvar.notify_all(); + } } } } - } - // Final notification in case any thread is waiting on in_flight_count - in_flight_cvar_clone.notify_all(); - }); + // Final notification in case any thread is waiting on in_flight_count + in_flight_cvar_clone.notify_all(); + }) + .expect("spawn rust-rawblock-uring worker"); ( Some(ring), @@ -935,6 +1179,11 @@ impl RawBlockDevice { ) }; + // All fallible setup succeeded: hand the fd over to the struct, whose + // Drop is now responsible for closing it. Disarm so the guard does not + // close the same descriptor a second time. + let fd = fd_guard.disarm(); + Ok(Self { fd, size, @@ -1235,7 +1484,7 @@ impl RawBlockDevice { let mut q = queue.lock().unwrap(); q.push(sub); } - batch_ready.notify_one(); + batch_ready.signal_producer(); // Store completion for error checking in wait_iouring { @@ -1455,7 +1704,7 @@ impl RawBlockDevice { q.push(sub); } if let Some(batch_ready) = &self.batch_ready { - batch_ready.notify_one(); + batch_ready.signal_producer(); } py.allow_threads(move || comp.wait()) } else { @@ -1483,7 +1732,7 @@ impl RawBlockDevice { q.push(sub); } if let Some(batch_ready) = &self.batch_ready { - batch_ready.notify_one(); + batch_ready.signal_producer(); } py.allow_threads(move || comp.wait()) }; @@ -1671,7 +1920,7 @@ impl RawBlockDevice { let mut q = queue.lock().unwrap(); q.push(sub); } - batch_ready.notify_one(); + batch_ready.signal_producer(); // Store completion for error checking in wait_iouring { @@ -1971,7 +2220,7 @@ impl RawBlockDevice { shutdown.store(true, Ordering::Relaxed); } if let Some(batch_ready) = &self.batch_ready { - batch_ready.notify_all(); + batch_ready.signal_producer(); } let mutex = Mutex::new(()); From 3e83d5c2020dc5984269d5646c7be5a906cca595 Mon Sep 17 00:00:00 2001 From: Guy Ealey Morag Date: Tue, 26 May 2026 18:57:01 +0200 Subject: [PATCH 63/69] nixl_storage: Fix tests and remove from ignore list (#3200) * nixl_storage: naive support for files + dynamic static is not actually very usable with files: we bump into the OS limits on the open files very quickly, limiting the size of the cache. With tons of VRAM, having G3 of comparable size is not helping much, we really need it to be much bigger. This commit adds support for files in dynamic mode of nixl storage. There are some naive things done: 1. No support for sharing the cache storage. We assume the worker has exclusive access to the files and it is always safe to overwrite existing files. Note that with TP!=1 we will have several workers with the same target directory, that's why it's important to have a worker id as part of the key, so they don't affect each other. 2. No eviction support. As before, dynamic mode has no evicting support. This kinda made sense when it was only working with OBJ storages, but now that is something to be aware of. It's not hard to add eviction though. 3. Flat directory with all the cache files. There are going to be a lot of them, especially provided we don't have eviction. Most filesystems don't optimize for the case of a directory with millions of files, and we constantly open/close files there, that might be a source of additional latency. - There is an existing PR to support sharding the directory, we can merge it as a band aid. - As a more advanced solution, we can do multi-layer subdirectory structure, like gds backend does. 4. We switched directly from having _all_ files open at once, to open/close _every_ single file on access. We might explore having an LRU cache of open files instead. Signed-off-by: Ilya Yanok * Fix formatting Signed-off-by: Guy Ealey Morag * Fix PR Comments Signed-off-by: Guy Ealey Morag * Extract _build_descs Signed-off-by: Guy Ealey Morag * Fix test Signed-off-by: Guy Ealey Morag * Refactor to increase readability Signed-off-by: Guy Ealey Morag * Fix another leak issue Signed-off-by: Guy Ealey Morag * Fix release order Signed-off-by: Guy Ealey Morag * Treat file not found as a miss Signed-off-by: Guy Ealey Morag * Add docstring about file create mode 0o644 Signed-off-by: Guy Ealey Morag * Unlink files on failure Signed-off-by: Guy Ealey Morag * Add missing docs Signed-off-by: Guy Ealey Morag * Fix pre-commit issues Signed-off-by: Guy Ealey Morag * Fix mypy error Signed-off-by: Guy Ealey Morag * Fix test Signed-off-by: Guy Ealey Morag * Make tests pass Signed-off-by: Guy Ealey Morag * Fix formatting Signed-off-by: Guy Ealey Morag * Fix nixl tests and stop ignoring them in CI Signed-off-by: Guy Ealey Morag * Fix path to support GDS Signed-off-by: Guy Ealey Morag * Revert path creation in NixlDynamicStorageBackend Signed-off-by: Guy Ealey Morag * Fix code quality issue Signed-off-by: Guy Ealey Morag --------- Signed-off-by: Ilya Yanok Signed-off-by: Guy Ealey Morag Co-authored-by: Ilya Yanok --- .buildkite/k3_tests/unit/run.sh | 1 - .buildkite/pipeline.yml | 1 - .github/workflows/test.yml | 2 +- AGENTS.md | 9 ++- CONTRIBUTING.md | 1 - tests/v1/test_nixl_storage.py | 104 ++++++++++++++++++++++++++++---- 6 files changed, 96 insertions(+), 22 deletions(-) diff --git a/.buildkite/k3_tests/unit/run.sh b/.buildkite/k3_tests/unit/run.sh index 314e50d1ed3..052ed5931ce 100755 --- a/.buildkite/k3_tests/unit/run.sh +++ b/.buildkite/k3_tests/unit/run.sh @@ -35,7 +35,6 @@ pytest --maxfail=1 --cov=lmcache \ --cov-report term --cov-report=html:coverage-test \ --cov-report=xml:coverage-test.xml --html=durations/test.html \ --ignore=tests/disagg --ignore=tests/v1/test_pos_kernels.py \ - --ignore=tests/v1/test_nixl_storage.py \ --ignore=tests/skipped \ --ignore=tests/v1/storage_backend/test_eic.py diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6d024a7d721..e4517632cee 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -37,7 +37,6 @@ steps: --cov-report term --cov-report=html:coverage-test \ --cov-report=xml:coverage-test.xml --html=durations/test.html \ --ignore=tests/disagg --ignore=tests/v1/test_pos_kernels.py \ - --ignore=tests/v1/test_nixl_storage.py \ --ignore=tests/v1/test_nixl_batched_contains.py \ --ignore=tests/v1/test_device_id_race.py \ --ignore=tests/skipped \ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 62e36f2ff0c..87ab1c34335 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,7 +99,7 @@ jobs: - name: "Run non-CUDA unit tests" run: | - pytest --ignore=tests/disagg --ignore=tests/v1/test_nixl_storage.py \ + pytest --ignore=tests/disagg \ --ignore=tests/v1/multiprocess/ \ --ignore=tests/v1/distributed/ \ --ignore=tests/v1/mp_observability/ \ diff --git a/AGENTS.md b/AGENTS.md index aa1673d0449..aa239e53877 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,11 +47,10 @@ BUILD_WITH_HIP=1 pip install -e . ```bash # Run standard test suite (mirrors CI) pytest -xvs --ignore=tests/disagg \ - --ignore=tests/v1/test_nixl_storage.py \ - --ignore=tests/v1/multiprocess/ \ - --ignore=tests/v1/distributed/ \ - --ignore=tests/skipped \ - --ignore=tests/v1/storage_backend/test_eic.py + --ignore=tests/v1/multiprocess/ \ + --ignore=tests/v1/distributed/ \ + --ignore=tests/skipped \ + --ignore=tests/v1/storage_backend/test_eic.py # Run a single test file pytest -xvs tests/v1/test_cache_engine.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e37e0270b27..506a19bda2c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,7 +47,6 @@ BUILD_WITH_HIP=1 pip install -e . ```bash # Run standard test suite (mirrors CI) pytest -xvs --ignore=tests/disagg \ - --ignore=tests/v1/test_nixl_storage.py \ --ignore=tests/v1/multiprocess/ \ --ignore=tests/v1/distributed/ \ --ignore=tests/skipped \ diff --git a/tests/v1/test_nixl_storage.py b/tests/v1/test_nixl_storage.py index 634c6484cb7..47499d82948 100644 --- a/tests/v1/test_nixl_storage.py +++ b/tests/v1/test_nixl_storage.py @@ -2,9 +2,14 @@ # Standard from pathlib import Path import asyncio +import contextlib +import functools import os +import shutil import sys +import tempfile import threading +import uuid # Third Party import pytest @@ -12,6 +17,10 @@ pytest.importorskip("nixl", reason="nixl package is required for nixl tests") +# Third Party +from nixl._api import nixl_agent as NixlAgent +from nixl._api import nixl_agent_config as NixlAgentConfig + # First Party from lmcache.utils import CacheEngineKey from lmcache.v1.config import LMCacheEngineConfig @@ -22,6 +31,52 @@ NixlStorageBackend, NixlStorageConfig, ) +from lmcache.v1.transfer_channel.transfer_utils import get_correct_device + +# cuFile-based backends (GDS, GDS_MT) need a GDS-capable filesystem +_TEST_TMPDIR = os.environ.get("LMCACHE_TEST_TMPDIR") or None + + +@functools.lru_cache(maxsize=None) +def _can_register_file_with_nixl_backend(backend: str) -> bool: + """Probe ``cuFileHandleRegister`` via NIXL on the test scratch dir.""" + + probe_dir = tempfile.mkdtemp(prefix="nixl_gds_probe_", dir=_TEST_TMPDIR) + probe_path = os.path.join(probe_dir, "probe.bin") + fd = -1 + try: + agent = NixlAgent( + f"NixlGdsProbe_{uuid.uuid4().hex}", + NixlAgentConfig(backends=[]), + ) + agent.create_backend(backend, {}) + fd = os.open(probe_path, os.O_CREAT | os.O_RDWR, 0o600) + os.write(fd, b"\x00" * 4096) + agent.register_memory([(0, 4096, fd, "")], mem_type="FILE") + return True + except Exception: + return False + finally: + if fd >= 0: + with contextlib.suppress(OSError): + os.close(fd) + shutil.rmtree(probe_dir, ignore_errors=True) + + +_GDS_SKIP_REASON = ( + "NIXL {backend} cannot register file handles in this environment; " + "set LMCACHE_TEST_TMPDIR to a GDS-capable mount (ext4/xfs) to enable." +) + + +@pytest.fixture +def nixl_tmp_path(): + """Per-test scratch dir, honoring ``LMCACHE_TEST_TMPDIR``.""" + path = tempfile.mkdtemp(prefix="nixl_test_", dir=_TEST_TMPDIR) + try: + yield path + finally: + shutil.rmtree(path, ignore_errors=True) def create_key(chunk_hash: str): @@ -70,7 +125,9 @@ def run(config: LMCacheEngineConfig, shape, dtype): config, metadata, thread_loop, - dst_device=config.nixl_buffer_device, # Pass the device directly + dst_device=get_correct_device( + config.nixl_buffer_device, metadata.worker_id + ), ) assert len(backends) == 2 # NixlStorageBackend + LocalCPUBackend assert BACKEND_NAME in backends @@ -201,62 +258,82 @@ def test_eviction(new_idx, old_idx): @pytest.mark.no_shared_allocator @pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA") -def test_nixl_gds_mt_cuda_backend(): +@pytest.mark.skipif( + not _can_register_file_with_nixl_backend("GDS_MT"), + reason=_GDS_SKIP_REASON.format(backend="GDS_MT"), +) +def test_nixl_gds_mt_cuda_backend(nixl_tmp_path): BASE_DIR = Path(__file__).parent config = LMCacheEngineConfig.from_file(BASE_DIR / "data/nixl.yaml") dtype = torch.bfloat16 - shape = [2048, 2048] + shape = torch.Size([2048, 2048]) - config.nixl_buffer_device = "cuda:0" # Use explicit device + config.nixl_buffer_device = "cuda" config.extra_config["nixl_backend"] = "GDS_MT" config.extra_config["enable_cuda"] = True + config.extra_config["nixl_path"] = nixl_tmp_path run(config, shape, dtype) @pytest.mark.no_shared_allocator -def test_nixl_gds_mt_cpu_backend(): +@pytest.mark.skipif( + not _can_register_file_with_nixl_backend("GDS_MT"), + reason=_GDS_SKIP_REASON.format(backend="GDS_MT"), +) +def test_nixl_gds_mt_cpu_backend(nixl_tmp_path): BASE_DIR = Path(__file__).parent config = LMCacheEngineConfig.from_file(BASE_DIR / "data/nixl.yaml") dtype = torch.bfloat16 - shape = [2048, 2048] + shape = torch.Size([2048, 2048]) config.nixl_buffer_device = "cpu" config.extra_config["nixl_backend"] = "GDS_MT" config.extra_config["enable_cuda"] = False + config.extra_config["nixl_path"] = nixl_tmp_path run(config, shape, dtype) @pytest.mark.no_shared_allocator @pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA") -def test_nixl_gds_cuda_backend(): +@pytest.mark.skipif( + not _can_register_file_with_nixl_backend("GDS"), + reason=_GDS_SKIP_REASON.format(backend="GDS"), +) +def test_nixl_gds_cuda_backend(nixl_tmp_path): BASE_DIR = Path(__file__).parent config = LMCacheEngineConfig.from_file(BASE_DIR / "data/nixl.yaml") dtype = torch.bfloat16 - shape = [2048, 2048] + shape = torch.Size([2048, 2048]) - config.nixl_buffer_device = "cuda:0" # Use explicit device + config.nixl_buffer_device = "cuda" config.extra_config["nixl_backend"] = "GDS" config.extra_config["enable_cuda"] = True + config.extra_config["nixl_path"] = nixl_tmp_path run(config, shape, dtype) @pytest.mark.no_shared_allocator -def test_nixl_gds_cpu_backend(): +@pytest.mark.skipif( + not _can_register_file_with_nixl_backend("GDS"), + reason=_GDS_SKIP_REASON.format(backend="GDS"), +) +def test_nixl_gds_cpu_backend(nixl_tmp_path): BASE_DIR = Path(__file__).parent config = LMCacheEngineConfig.from_file(BASE_DIR / "data/nixl.yaml") dtype = torch.bfloat16 - shape = [2048, 2048] + shape = torch.Size([2048, 2048]) config.nixl_buffer_device = "cpu" config.extra_config["nixl_backend"] = "GDS" config.extra_config["enable_cuda"] = False + config.extra_config["nixl_path"] = nixl_tmp_path run(config, shape, dtype) @@ -305,16 +382,17 @@ def test_nixl_endpoint_list_malformed_url_raises(): @pytest.mark.no_shared_allocator -def test_nixl_posix_backend(): +def test_nixl_posix_backend(nixl_tmp_path): BASE_DIR = Path(__file__).parent config = LMCacheEngineConfig.from_file(BASE_DIR / "data/nixl.yaml") dtype = torch.bfloat16 - shape = [2048, 2048] + shape = torch.Size([2048, 2048]) config.nixl_buffer_device = "cpu" config.extra_config["nixl_backend"] = "POSIX" config.extra_config["enable_cuda"] = False + config.extra_config["nixl_path"] = nixl_tmp_path run(config, shape, dtype) From a68bd0a0d1465f5a1a6b69d1ac8ca2e0be54932b Mon Sep 17 00:00:00 2001 From: Ilya Yanok Date: Tue, 26 May 2026 19:10:29 +0200 Subject: [PATCH 64/69] Iyanok/huge pages (#2643) * memory: huge page support (C bits) Add 3 pairs of alloc/free functions for huge pages. The 4th option is for SHM and it's missing, since normal SHM use cases don't support huge pages. Old API is left untouched, except for the small refactor: common mmap code is factored out. This doesn't affect the behavior. Signed-off-by: Ilya Yanok * memory_management: use new functions for huge pages Change two allocators to support us_huge_pages flag. Signed-off-by: Ilya Yanok * memory_management: log if allocating huge pages fail Signed-off-by: Ilya Yanok * non_cuda_equivalents: change memory functions to match cuda Adding use_hugepages to alloc and size to free. There is no actual huge page support though. It should be pretty easy to add to non-shm cases (mmap and give torch a buffer). For shm we need to look into shared_memory module implementation. Signed-off-by: Ilya Yanok * local_cpu: support for huge pages Signed-off-by: Ilya Yanok * nixl_storage: add config option to alloc huge pages We want to be able to alloc huge pages for the NIXL buffer in DRAM. Signed-off-by: Ilya Yanok * PR comments fixes, Code cleanup Signed-off-by: Guy Ealey Morag * Skip test if not enough free huge pages Signed-off-by: Guy Ealey Morag * Fix non-cuda fallbacks Signed-off-by: Guy Ealey Morag * Add docs Signed-off-by: Guy Ealey Morag * Remove unused func Signed-off-by: Guy Ealey Morag * Fix hugepage info to check for 2 MiB only Signed-off-by: Guy Ealey Morag * Fix pre-commit issues Signed-off-by: Guy Ealey Morag * Merge branch 'dev' into iyanok/huge-pages Signed-off-by: Guy Ealey Morag --------- Signed-off-by: Ilya Yanok Signed-off-by: Guy Ealey Morag Co-authored-by: Guy Ealey Morag Co-authored-by: Yihua Cheng --- csrc/mem_alloc.cpp | 95 ++++++++-- csrc/mem_alloc.h | 10 ++ csrc/pybind.cpp | 6 + docs/source/api_reference/configurations.rst | 5 + .../kv_cache/storage_backends/cpu_ram.rst | 50 ++++++ .../source/kv_cache/storage_backends/nixl.rst | 5 +- lmcache/python_ops_fallback.py | 37 ++++ lmcache/v1/config.py | 5 + lmcache/v1/memory_management.py | 166 ++++++++++++++---- .../v1/storage_backend/local_cpu_backend.py | 7 + .../storage_backend/nixl_storage_backend.py | 22 ++- tests/v1/test_memory_management.py | 58 ++++++ 12 files changed, 417 insertions(+), 49 deletions(-) diff --git a/csrc/mem_alloc.cpp b/csrc/mem_alloc.cpp index b659b9cae52..9ac21c1af76 100644 --- a/csrc/mem_alloc.cpp +++ b/csrc/mem_alloc.cpp @@ -1,8 +1,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -10,6 +12,24 @@ #include // for MPOL_BIND, MPOL_MF_MOVE, MPOL_MF_STRICT #include "mem_alloc.h" +static constexpr size_t HUGEPAGE_SIZE = 2UL * 1024 * 1024; // MAP_HUGE_2MB + +static inline size_t _align_hugepage(size_t size) { + return (size + HUGEPAGE_SIZE - 1) & ~(HUGEPAGE_SIZE - 1); +} + +static void* _mmap_anon(size_t size, bool hugepages) { + int flags = MAP_PRIVATE | MAP_ANONYMOUS; + if (hugepages) { + flags |= MAP_HUGETLB | MAP_HUGE_2MB; + } + void* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, flags, -1, 0); + if (ptr == MAP_FAILED) { + throw std::runtime_error(std::string("mmap failed: ") + strerror(errno)); + } + return ptr; +} + uintptr_t alloc_pinned_ptr(size_t size, unsigned int flags) { void* ptr = nullptr; cudaError_t err = cudaHostAlloc(&ptr, size, flags); @@ -26,6 +46,36 @@ void free_pinned_ptr(uintptr_t ptr) { } } +uintptr_t alloc_hugepage_pinned_ptr(size_t size, unsigned int flags) { + size = _align_hugepage(size); + void* ptr = _mmap_anon(size, true); + + cudaError_t st = cudaHostRegister(ptr, size, flags); + if (st != cudaSuccess) { + munmap(ptr, size); + throw std::runtime_error(std::string("cudaHostRegister failed: ") + + cudaGetErrorString(st)); + } + + return reinterpret_cast(ptr); +} + +void free_hugepage_pinned_ptr(uintptr_t ptr, size_t size) { + size = _align_hugepage(size); + void* p = reinterpret_cast(ptr); + + // Unpin first, then unmap. + cudaError_t st = cudaHostUnregister(p); + if (st != cudaSuccess) { + munmap(p, size); + throw std::runtime_error(std::string("cudaHostUnregister failed: ") + + cudaGetErrorString(st)); + } + if (munmap(p, size) != 0) { + throw std::runtime_error(std::string("munmap failed: ") + strerror(errno)); + } +} + void batched_memcpy(const std::vector& src_ptrs, const std::vector& dst_ptrs, const std::vector& sizes) { @@ -43,10 +93,11 @@ void batched_memcpy(const std::vector& src_ptrs, } } -static void first_touch(void* p, size_t size) { - const long ps = sysconf(_SC_PAGESIZE); +static void first_touch(void* p, size_t size, bool hugepages) { + const size_t ps = + hugepages ? HUGEPAGE_SIZE : static_cast(sysconf(_SC_PAGESIZE)); for (size_t off = 0; off < size; off += ps) { - volatile char* c = (volatile char*)p + off; + volatile char* c = static_cast(p) + off; *c = 0; } } @@ -58,11 +109,12 @@ static inline int mbind_sys(void* addr, unsigned long len, int mode, return (rc == -1) ? -errno : 0; } -uintptr_t alloc_numa_ptr(size_t size, int node) { - void* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (ptr == MAP_FAILED) - throw std::runtime_error(std::string("mmap failed: ") + strerror(errno)); +static uintptr_t _alloc_numa_impl(size_t size, int node, bool hugepages) { + if (hugepages) { + assert(size % HUGEPAGE_SIZE == 0); + } + + void* ptr = _mmap_anon(size, hugepages); // Maximum of 64 numa nodes unsigned long mask = 1UL << node; @@ -74,11 +126,15 @@ uintptr_t alloc_numa_ptr(size_t size, int node) { throw std::runtime_error(std::string("mbind failed: ") + strerror(err)); } - first_touch(ptr, size); + first_touch(ptr, size, hugepages); return reinterpret_cast(ptr); } +uintptr_t alloc_numa_ptr(size_t size, int node) { + return _alloc_numa_impl(size, node, false); +} + void free_numa_ptr(uintptr_t ptr, size_t size) { void* p = reinterpret_cast(ptr); if (munmap(p, size) != 0) { @@ -86,8 +142,9 @@ void free_numa_ptr(uintptr_t ptr, size_t size) { } } -uintptr_t alloc_pinned_numa_ptr(size_t size, int node) { - void* ptr = reinterpret_cast(alloc_numa_ptr(size, node)); +static uintptr_t _alloc_pinned_numa_impl(size_t size, int node, + bool hugepages) { + void* ptr = reinterpret_cast(_alloc_numa_impl(size, node, hugepages)); cudaError_t st = cudaHostRegister(ptr, size, 0); if (st != cudaSuccess) { @@ -99,6 +156,15 @@ uintptr_t alloc_pinned_numa_ptr(size_t size, int node) { return reinterpret_cast(ptr); } +uintptr_t alloc_pinned_numa_ptr(size_t size, int node) { + return _alloc_pinned_numa_impl(size, node, false); +} + +uintptr_t alloc_hugepage_pinned_numa_ptr(size_t size, int node) { + size = _align_hugepage(size); + return _alloc_pinned_numa_impl(size, node, true); +} + void free_pinned_numa_ptr(uintptr_t ptr, size_t size) { void* p = reinterpret_cast(ptr); // Unpin first, then unmap. @@ -113,6 +179,11 @@ void free_pinned_numa_ptr(uintptr_t ptr, size_t size) { } } +void free_hugepage_pinned_numa_ptr(uintptr_t ptr, size_t size) { + size = _align_hugepage(size); + free_pinned_numa_ptr(ptr, size); +} + uintptr_t alloc_shm_pinned_ptr(size_t size, const std::string& shm_name) { int fd = shm_open(shm_name.c_str(), O_CREAT | O_RDWR, 0600); if (fd < 0) @@ -133,7 +204,7 @@ uintptr_t alloc_shm_pinned_ptr(size_t size, const std::string& shm_name) { throw std::runtime_error(std::string("mmap failed: ") + strerror(errno)); } - first_touch(ptr, size); + first_touch(ptr, size, false); cudaError_t st = cudaHostRegister(ptr, size, 0); if (st != cudaSuccess) { diff --git a/csrc/mem_alloc.h b/csrc/mem_alloc.h index 3e189ef2c2f..58c83b7fa8e 100644 --- a/csrc/mem_alloc.h +++ b/csrc/mem_alloc.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 + #include #include #include @@ -15,3 +17,11 @@ void free_numa_ptr(uintptr_t ptr, size_t size); void free_pinned_numa_ptr(uintptr_t ptr, size_t size); void free_shm_pinned_ptr(uintptr_t ptr, size_t size, const std::string& shm_name); + +// Hugepage variants (MAP_HUGETLB). Not available for shm: /dev/shm usually +// uses tmpfs, and tmpfs does not support MAP_HUGETLB. +uintptr_t alloc_hugepage_pinned_ptr(size_t size, unsigned int flags); +uintptr_t alloc_hugepage_pinned_numa_ptr(size_t size, int node); + +void free_hugepage_pinned_ptr(uintptr_t ptr, size_t size); +void free_hugepage_pinned_numa_ptr(uintptr_t ptr, size_t size); diff --git a/csrc/pybind.cpp b/csrc/pybind.cpp index 2228795d59a..817295bef5b 100644 --- a/csrc/pybind.cpp +++ b/csrc/pybind.cpp @@ -61,9 +61,15 @@ PYBIND11_MODULE(c_ops, m) { m.def("alloc_pinned_ptr", &alloc_pinned_ptr, py::call_guard()); m.def("free_pinned_ptr", &free_pinned_ptr); + m.def("alloc_hugepage_pinned_ptr", &alloc_hugepage_pinned_ptr, + py::call_guard()); + m.def("free_hugepage_pinned_ptr", &free_hugepage_pinned_ptr); m.def("alloc_pinned_numa_ptr", &alloc_pinned_numa_ptr, py::call_guard()); m.def("free_pinned_numa_ptr", &free_pinned_numa_ptr); + m.def("alloc_hugepage_pinned_numa_ptr", &alloc_hugepage_pinned_numa_ptr, + py::call_guard()); + m.def("free_hugepage_pinned_numa_ptr", &free_hugepage_pinned_numa_ptr); m.def("alloc_numa_ptr", &alloc_numa_ptr, py::call_guard()); m.def("free_numa_ptr", &free_numa_ptr); diff --git a/docs/source/api_reference/configurations.rst b/docs/source/api_reference/configurations.rst index 85b16f1bb9c..8c3506d7350 100644 --- a/docs/source/api_reference/configurations.rst +++ b/docs/source/api_reference/configurations.rst @@ -34,6 +34,9 @@ Basic cache settings that control the core functionality of LMCache. * - max_local_cpu_size - LMCACHE_MAX_LOCAL_CPU_SIZE - Maximum CPU cache size in GB. Default: 5.0 + * - local_cpu_use_hugepages + - LMCACHE_LOCAL_CPU_USE_HUGEPAGES + - Whether to use Linux hugepages (2 MB) for CPU-pinned KV cache memory. Not compatible with P2P mode or shared memory (multiprocess). Requires pre-allocated hugepages (``sysctl vm.nr_hugepages``). Values: true/false. Default: false * - local_disk - LMCACHE_LOCAL_DISK - Path (or comma-separated paths) to local disk cache directories. Format: ``"file:///path/to/cache"`` or ``"/path/a,/path/b"`` for multi-device I/O. See ``local_disk_path_sharding`` for how paths are assigned to GPUs. @@ -354,6 +357,8 @@ Settings for using Nixl as a storage backend instead of disaggregated prefill. T - Number of files or objects in the storage pool * - nixl_endpoint_list - List of object-storage endpoint URLs for per-worker distribution. Each TP worker selects an entry round-robin by ``local_worker_id``, overriding ``nixl_backend_params.endpoint_override``. Only applied when ``nixl_backend`` is ``"OBJ"`` (silently ignored otherwise). Each entry must start with ``http://`` or ``https://``; an empty list raises ``ValueError`` at engine init. + * - nixl_use_hugepages + - Whether to use Linux hugepages (2 MiB) for the NIXL CPU buffer. Requires pre-allocated hugepages (``sysctl vm.nr_hugepages``). Values: true/false. Default: false Additional Storage Configurations diff --git a/docs/source/kv_cache/storage_backends/cpu_ram.rst b/docs/source/kv_cache/storage_backends/cpu_ram.rst index ce1253615cd..c29d5c482c7 100644 --- a/docs/source/kv_cache/storage_backends/cpu_ram.rst +++ b/docs/source/kv_cache/storage_backends/cpu_ram.rst @@ -63,6 +63,56 @@ tokens into the pinned CPU RAM from the disk or remote storage (*if* the KV cach tokens are already stored there). This can preemptively avoid the latency of the disk and remote KV transfer if we predict these tokens will be requested soon (e.g. structured or agentic workflows). +.. _cpu_ram-hugepage-support: + +Hugepage Support +----------------- + +By default LMCache allocates CPU-pinned memory using regular 4 KiB pages. +For large KV cache buffers (multiple gigabytes), enabling **Linux hugepages** +(2 MiB pages) can reduce TLB (Translation Lookaside Buffer) pressure and +improve memory access performance. + +**System prerequisite** + +Hugepages must be pre-allocated at the OS level before LMCache starts. +TO find the number of pages needed, divide the desired buffer size by 2 MiB and round up. +For example, 5 GB requires at least 2560 pages: + +.. code-block:: bash + + # Allocate 2560 hugepages (5 GB) + sudo sysctl -w vm.nr_hugepages=2560 + + # Make persistent across reboots + echo 'vm.nr_hugepages=2560' | sudo tee -a /etc/sysctl.conf + +Verify that pages are available: + +.. code-block:: bash + + grep HugePages /proc/meminfo + # HugePages_Total: 2560 + # HugePages_Free: 2560 + +**Configuration** + +.. code-block:: yaml + + local_cpu_use_hugepages: true + +Or via environment variable: + +.. code-block:: bash + + export LMCACHE_LOCAL_CPU_USE_HUGEPAGES=true + +**Restrictions** + +- Hugepages are **not compatible with P2P mode** (``enable_p2p: true``). +- Hugepages are **not compatible with shared memory** (``shm_name`` is set). +- On non-CUDA platforms, hugepages are not supported. Regular allocation will be used as fallback. + .. _cpu_ram-online-inference-example: Online Inference Example diff --git a/docs/source/kv_cache/storage_backends/nixl.rst b/docs/source/kv_cache/storage_backends/nixl.rst index 9f599ab1ef6..bc689b78b31 100644 --- a/docs/source/kv_cache/storage_backends/nixl.rst +++ b/docs/source/kv_cache/storage_backends/nixl.rst @@ -37,7 +37,8 @@ Example ``lmcache-config.yaml`` for POSIX backend: nixl_backend: POSIX nixl_pool_size: 64 nixl_path: /mnt/nixl/cache/ - use_direct_io: True + use_direct_io: true + nixl_use_hugepages: true # optional, requires pre-allocated hugepages Key settings: @@ -51,6 +52,8 @@ Key settings: - ``nixl_backend``: configuration of which nixl backend to use for storage. +- ``nixl_use_hugepages``: whether to use Linux hugepages (2 MiB) for the NIXL CPU buffer. Not supported for GPU buffers. Requires pre-allocated hugepages (``sysctl vm.nr_hugepages``). Default: ``false``. + .. note:: Supported backends are: ["GDS", "GDS_MT", "POSIX", "HF3FS", "OBJ", "AZURE_BLOB"]. diff --git a/lmcache/python_ops_fallback.py b/lmcache/python_ops_fallback.py index 10fce1d4afa..5c19d96f8b4 100644 --- a/lmcache/python_ops_fallback.py +++ b/lmcache/python_ops_fallback.py @@ -10,6 +10,7 @@ from typing import Optional, Tuple import ctypes import ctypes.util +import warnings # Third Party from numba import njit @@ -438,6 +439,42 @@ def free_shm_pinned_ptr(ptr: int, size: int = 0, shm_name: str = "") -> None: shm.unlink() +# Hugepage variants: non-CUDA platforms do not support hugepages, so these +# fall back to the same regular pinned allocation. + + +def alloc_hugepage_pinned_ptr(size: int, device_id: int = 0) -> int: + """Non-CUDA fallback for alloc_hugepage_pinned_ptr (no hugepage support).""" + warnings.warn( + "Hugepages requested but not available on non-CUDA platforms; " + "falling back to regular allocation.", + RuntimeWarning, + stacklevel=2, + ) + return alloc_pinned_ptr(size, device_id) + + +def free_hugepage_pinned_ptr(ptr: int, size: int = 0) -> None: + """Non-CUDA fallback for free_hugepage_pinned_ptr (no hugepage support).""" + free_pinned_ptr(ptr) + + +def alloc_hugepage_pinned_numa_ptr(size: int, numa_id: int = 0) -> int: + """Non-CUDA fallback for alloc_hugepage_pinned_numa_ptr (no hugepage support).""" + warnings.warn( + "Hugepages requested but not available on non-CUDA platforms; " + "falling back to regular allocation.", + RuntimeWarning, + stacklevel=2, + ) + return alloc_pinned_numa_ptr(size, numa_id) + + +def free_hugepage_pinned_numa_ptr(ptr: int, size: int = 0) -> None: + """Non-CUDA fallback for free_hugepage_pinned_numa_ptr (no hugepage support).""" + free_pinned_numa_ptr(ptr, size) + + def alloc_numa_ptr(size: int, numa_id: int = 0) -> int: """Non-CUDA equivalent of allocating numa memory and returning pointer to it. Note: Numa memory is not supported on non-CUDA.""" diff --git a/lmcache/v1/config.py b/lmcache/v1/config.py index c7b317d99ca..d52693cafed 100644 --- a/lmcache/v1/config.py +++ b/lmcache/v1/config.py @@ -75,6 +75,11 @@ "env_converter": _to_bool, }, "max_local_cpu_size": {"type": float, "default": 5.0, "env_converter": float}, + "local_cpu_use_hugepages": { + "type": bool, + "default": False, + "env_converter": _to_bool, + }, "reserve_local_cpu_size": {"type": float, "default": 0.0, "env_converter": float}, "local_disk": { "type": Optional[str], diff --git a/lmcache/v1/memory_management.py b/lmcache/v1/memory_management.py index 9e1a5182b3c..18509941c8e 100644 --- a/lmcache/v1/memory_management.py +++ b/lmcache/v1/memory_management.py @@ -372,25 +372,44 @@ def parent(self) -> Optional["MemoryAllocatorInterface"]: raise NotImplementedError +@dataclass +class PinnedAllocFree: + """Resolved alloc/free function pair for pinned CPU memory.""" + + alloc_fn: Any + alloc_args: tuple + free_fn: Any + free_args: tuple + + def alloc(self) -> int: + """Allocate pinned memory and return the raw pointer.""" + return self.alloc_fn(*self.alloc_args) + + def free(self, ptr: int) -> None: + """Free a previously allocated pinned-memory pointer.""" + self.free_fn(ptr, *self.free_args) + + def _resolve_pinned_alloc_free( numa_mapping: Optional[NUMAMapping] = None, shm_name: Optional[str] = None, size: Optional[int] = None, -) -> Tuple[ - tuple, # (alloc_fn, *alloc_args) - tuple, # (free_fn, *free_args_after_ptr) -]: + use_hugepages: bool = False, +) -> PinnedAllocFree: """Resolve the alloc/free function pair based on memory type. Returns: - A tuple of (alloc_info, free_info) where: - - alloc_info: (alloc_fn, *args) to call as alloc_fn(size, *args) - - free_info: (free_fn, *args) to call as free_fn(ptr, *args) + A PinnedAllocFree with the resolved functions and their extra + arguments. Call ``ptr = resolved.alloc()`` and ``resolved.free(ptr)``. """ if shm_name: - return ( - (lmc_ops.alloc_shm_pinned_ptr, shm_name), - (lmc_ops.free_shm_pinned_ptr, size, shm_name), + if use_hugepages: + raise ValueError("Hugepages are not supported with shared memory (shm)") + return PinnedAllocFree( + alloc_fn=lmc_ops.alloc_shm_pinned_ptr, + alloc_args=(size, shm_name), + free_fn=lmc_ops.free_shm_pinned_ptr, + free_args=(size, shm_name), ) elif numa_mapping: if torch_dev.is_available(): @@ -402,31 +421,103 @@ def _resolve_pinned_alloc_free( f"Current device {current_device_id} is not in the GPU NUMA mapping." ) numa_id = gpu_to_numa_mapping[current_device_id] - return ( - (lmc_ops.alloc_pinned_numa_ptr, numa_id), - (lmc_ops.free_pinned_numa_ptr, size), - ) + if use_hugepages: + return PinnedAllocFree( + alloc_fn=lmc_ops.alloc_hugepage_pinned_numa_ptr, + alloc_args=(size, numa_id), + free_fn=lmc_ops.free_hugepage_pinned_numa_ptr, + free_args=(size,), + ) + else: + return PinnedAllocFree( + alloc_fn=lmc_ops.alloc_pinned_numa_ptr, + alloc_args=(size, numa_id), + free_fn=lmc_ops.free_pinned_numa_ptr, + free_args=(size,), + ) else: - return ( - (lmc_ops.alloc_pinned_ptr, 0), - (lmc_ops.free_pinned_ptr,), - ) + flags = 0 + if use_hugepages: + return PinnedAllocFree( + alloc_fn=lmc_ops.alloc_hugepage_pinned_ptr, + alloc_args=(size, flags), + free_fn=lmc_ops.free_hugepage_pinned_ptr, + free_args=(size,), + ) + else: + return PinnedAllocFree( + alloc_fn=lmc_ops.alloc_pinned_ptr, + alloc_args=(size, flags), + free_fn=lmc_ops.free_pinned_ptr, + free_args=(), + ) + + +def _read_hugepage_info() -> Optional[Tuple[int, int, int]]: + """Read hugepage pool stats from sysfs. + + NOTE: We only use 2 MiB hugepages, so the pool stats are taken from + the 2 MiB pool directly rather than the system default pool reported in + ``/proc/meminfo`` (which can be 1 GiB on some hosts). + + Returns: + ``(nr_hugepages, free_hugepages, page_size_mb)`` for the hugepage + pool, or ``None`` if the sysfs entries are unavailable. + """ + base = "/sys/kernel/mm/hugepages/hugepages-2048kB" + try: + with open(f"{base}/nr_hugepages") as f: + total = int(f.read().strip()) + with open(f"{base}/free_hugepages") as f: + free = int(f.read().strip()) + return total, free, 2 + except (OSError, ValueError): + return None def _allocate_cpu_memory( size: int, numa_mapping: Optional[NUMAMapping] = None, shm_name: Optional[str] = None, + use_hugepages: bool = False, ) -> torch.Tensor: if size == 0: return torch.empty(0, dtype=torch.uint8) - alloc_info, _ = _resolve_pinned_alloc_free( + resolved = _resolve_pinned_alloc_free( numa_mapping, shm_name, + size, + use_hugepages, ) - alloc_fn, *alloc_args = alloc_info - ptr = alloc_fn(size, *alloc_args) + + try: + ptr = resolved.alloc() + except RuntimeError as e: + if use_hugepages and "mmap failed" in str(e): + diag = _read_hugepage_info() + if diag is not None: + total, free, page_mb = diag + page_bytes = page_mb * 1024 * 1024 + needed = (size + page_bytes - 1) // page_bytes + logger.error( + "Failed to allocate huge pages. " + "Pool has %d pages (%d free, each %d MiB). " + "Requested %d bytes (%d pages). " + "Please grow the %d MiB hugepage pool.", + total, + free, + page_mb, + size, + needed, + page_mb, + ) + else: + logger.error( + "Failed to allocate huge pages. " + "Please grow the 2 MiB hugepage pool." + ) + raise array_type = ctypes.c_uint8 * size buf = array_type.from_address(ptr) @@ -440,17 +531,18 @@ def _free_cpu_memory( size: int | None = None, numa_mapping: Optional[NUMAMapping] = None, shm_name: Optional[str] = None, + use_hugepages: bool = False, ) -> None: if torch_dev.is_available(): torch_dev.synchronize() - _, free_info = _resolve_pinned_alloc_free( + resolved = _resolve_pinned_alloc_free( numa_mapping, shm_name, - size=size, + size, + use_hugepages, ) - free_fn, *free_args = free_info - free_fn(buffer.data_ptr(), *free_args) + resolved.free(buffer.data_ptr()) def _allocate_gpu_memory( @@ -534,6 +626,7 @@ def get_shape(self) -> torch.Size: return self.meta.shape def get_dtype(self) -> torch.dtype: + assert self.meta.dtype is not None return self.meta.dtype def get_shapes(self) -> list[torch.Size]: @@ -1819,12 +1912,12 @@ def memcheck(self): def __str__(self): return "PagedTensorMemoryAllocator" - def get_paged_buffers(self) -> list[torch.Tensor]: + def get_paged_buffers(self) -> tuple[torch.Tensor, ...]: """ - Get the list of paged buffers for fixed buffer registration. + Get the paged buffers for fixed buffer registration. Returns: - List of paged buffer tensors that can be registered with io_uring + Tuple of paged buffer tensors that can be registered with io_uring for true zero copy operations. """ return self.paged_buffers @@ -2062,12 +2155,16 @@ class MixedMemoryAllocator(MemoryAllocatorInterface): (2) byte_array buffer memory. """ - def __init__(self, size: int, use_paging: bool = False, **kwargs): + def __init__( + self, size: int, use_paging: bool = False, use_hugepages: bool = False, **kwargs + ): """ :param int size: The size of the pinned memory in bytes. + :param bool use_hugepages: Whether to use hugepages. """ self.numa_mapping = kwargs.get("numa_mapping", None) + self.use_hugepages = use_hugepages self.align_bytes = kwargs.get("align_bytes", AddressManager.ALIGN_BYTES) if self.align_bytes <= 0 or self.align_bytes & (self.align_bytes - 1) != 0: raise ValueError("align_bytes must be a positive power of two") @@ -2083,7 +2180,9 @@ def __init__(self, size: int, use_paging: bool = False, **kwargs): self.size = size - self.buffer = _allocate_cpu_memory(size, self.numa_mapping, self.shm_name) + self.buffer = _allocate_cpu_memory( + size, self.numa_mapping, self.shm_name, use_hugepages=use_hugepages + ) self._unregistered = False @@ -2218,15 +2317,16 @@ def close(self): self.size, self.numa_mapping, self.shm_name, + use_hugepages=self.use_hugepages, ) self._unregistered = True - def get_paged_buffers(self) -> Optional[list[torch.Tensor]]: + def get_paged_buffers(self) -> Optional[tuple[torch.Tensor, ...]]: """ - Get the list of paged buffers for fixed buffer registration. + Get the paged buffers for fixed buffer registration. Returns: - List of paged buffer tensors if using paged allocator, None otherwise. + Tuple of paged buffer tensors if using paged allocator, None otherwise. These buffers can be registered with io_uring for true zero copy operations. """ if isinstance(self.pin_allocator, PagedTensorMemoryAllocator): diff --git a/lmcache/v1/storage_backend/local_cpu_backend.py b/lmcache/v1/storage_backend/local_cpu_backend.py index c072465db92..fbc2e461e98 100644 --- a/lmcache/v1/storage_backend/local_cpu_backend.py +++ b/lmcache/v1/storage_backend/local_cpu_backend.py @@ -350,6 +350,7 @@ def initialize_allocator( metadata: Optional[LMCacheMetadata] = None, ) -> MemoryAllocatorInterface: cpu_size = config.max_local_cpu_size + use_hugepages = config.local_cpu_use_hugepages if metadata is not None: # save_only_first_rank only works when use mla @@ -381,6 +382,9 @@ def initialize_allocator( ) if config.enable_p2p: + if use_hugepages: + raise ValueError("Hugepages are not supported with P2P mode") + # TODO(baoloongmao): Add lazy memory allocator support for P2P mode # For now, keep the original P2P implementation assert metadata is not None @@ -483,6 +487,7 @@ def initialize_allocator( return MixedMemoryAllocator( align_cpu_size_bytes, use_paging=True, + use_hugepages=False, **kwargs, ) @@ -492,11 +497,13 @@ def initialize_allocator( cpu_size_bytes, numa_mapping=numa_mapping, align_bytes=allocator_align_bytes, + use_hugepages=use_hugepages, ) return MixedMemoryAllocator( cpu_size_bytes, numa_mapping=numa_mapping, config=config, + use_hugepages=use_hugepages, ) @staticmethod diff --git a/lmcache/v1/storage_backend/nixl_storage_backend.py b/lmcache/v1/storage_backend/nixl_storage_backend.py index 856f351d8dc..a6895b31f8d 100644 --- a/lmcache/v1/storage_backend/nixl_storage_backend.py +++ b/lmcache/v1/storage_backend/nixl_storage_backend.py @@ -90,6 +90,7 @@ class NixlStorageConfig: enable_async_put: bool use_direct_io: bool path: str + use_hugepages: bool enable_prog_thread: bool sync_mode: Optional[Any] # nixl_thread_sync_t, None if unsupported @@ -149,6 +150,7 @@ def from_cache_engine_config( len(endpoint_list), ) path = extra_config.get("nixl_path") + use_hugepages = extra_config.get("nixl_use_hugepages", False) enable_prog_thread = extra_config.get("nixl_enable_prog_thread", True) sync_mode_str = extra_config.get("nixl_sync_mode", None) if sync_mode_str is not None and not _NIXL_SYNC_MODE_SUPPORTED: @@ -215,6 +217,7 @@ def from_cache_engine_config( enable_async_put=enable_async_put, use_direct_io=use_direct_io, path=path, + use_hugepages=use_hugepages, enable_prog_thread=enable_prog_thread, sync_mode=sync_mode, ) @@ -706,6 +709,7 @@ def __init__( self.progress_lock = threading.RLock() self.progress_set: Set[CacheEngineKey] = set() + self.nixl_config = nixl_config self.memory_allocator = self.initialize_allocator(config, metadata) def initialize_allocator( @@ -718,15 +722,23 @@ def initialize_allocator( "enable_nixl_storage" ) assert enable_nixl_storage + corrected_device = get_correct_device( config.nixl_buffer_device, metadata.worker_id, ) + self.use_hugepages = self.nixl_config.use_hugepages + self.buffer_size = config.nixl_buffer_size if corrected_device == "cpu": - self.buffer = _allocate_cpu_memory(config.nixl_buffer_size) + self.buffer = _allocate_cpu_memory( + config.nixl_buffer_size, use_hugepages=self.use_hugepages + ) self.free_pinned_buffer = True else: + if self.use_hugepages: + logger.warning("Hugepages are not supported for GPU memory allocation") + self.use_hugepages = False base_buffer, self.buffer = _allocate_gpu_memory( config.nixl_buffer_size, corrected_device ) @@ -1141,7 +1153,9 @@ def close(self) -> None: self.memory_allocator.close() if self.free_pinned_buffer: - _free_cpu_memory(self.buffer) + _free_cpu_memory( + self.buffer, self.buffer_size, use_hugepages=self.use_hugepages + ) class NixlDynamicStorageBackend(NixlStorageBackend): @@ -1862,4 +1876,6 @@ def close(self) -> None: self.memory_allocator.close() if self.free_pinned_buffer: - _free_cpu_memory(self.buffer) + _free_cpu_memory( + self.buffer, self.buffer_size, use_hugepages=self.use_hugepages + ) diff --git a/tests/v1/test_memory_management.py b/tests/v1/test_memory_management.py index aad1a0feb47..9f6429ce9ed 100644 --- a/tests/v1/test_memory_management.py +++ b/tests/v1/test_memory_management.py @@ -21,9 +21,14 @@ PinMemoryAllocator, TensorMemoryAllocator, TensorMemoryObj, + _allocate_cpu_memory, + _free_cpu_memory, + _read_hugepage_info, ) from lmcache.v1.pin_monitor import PinMonitor +HUGEPAGE_SIZE = 2 * 1024 * 1024 # MAP_HUGE_2MB + def check_allocator(allocator, max_size): # 512 * 512 * 4 = 1MB @@ -885,3 +890,56 @@ def test_allocation_and_free_interleaved(self, lazy_allocator_cls): assert allocator.memcheck() allocator.close() + + +def _get_num_free_hugepages() -> int: + """Return the number of free huge pages, or 0 if unknown.""" + info = _read_hugepage_info() + if info is None: + return 0 + _, free, _ = info + return free + + +@pytest.mark.skipif( + _get_num_free_hugepages() < 1, + reason="Requires at least 1 free huge page (sysctl vm.nr_hugepages)", +) +class TestHugepageAllocation: + """Tests for hugepage-backed CPU memory allocation. + + Skipped unless the system has pre-allocated huge pages. + """ + + def test_allocate_and_free(self): + """Allocate one huge page worth of memory and free it.""" + buf = _allocate_cpu_memory(HUGEPAGE_SIZE, use_hugepages=True) + assert buf.numel() == HUGEPAGE_SIZE + assert buf.dtype == torch.uint8 + buf[0] = 42 + buf[-1] = 99 + assert buf[0].item() == 42 + assert buf[-1].item() == 99 + _free_cpu_memory(buf, size=HUGEPAGE_SIZE, use_hugepages=True) + + @pytest.mark.skipif( + _get_num_free_hugepages() < 4, + reason="Requires at least 4 free huge pages (sysctl vm.nr_hugepages)", + ) + def test_allocate_multiple_pages(self): + """Allocate several huge pages and verify the buffer is usable.""" + size = 4 * HUGEPAGE_SIZE + buf = _allocate_cpu_memory(size, use_hugepages=True) + assert buf.numel() == size + buf.fill_(7) + assert buf[size // 2].item() == 7 + _free_cpu_memory(buf, size=size, use_hugepages=True) + + def test_read_hugepage_info(self): + """_read_hugepage_info returns valid data on Linux.""" + info = _read_hugepage_info() + assert info is not None + total, free, page_mb = info + assert total > 0 + assert free >= 0 + assert page_mb == 2 From bda3af18db83a9e1a2da9d8ed8ee7ca064b890ef Mon Sep 17 00:00:00 2001 From: Yihua Cheng Date: Tue, 26 May 2026 12:43:04 -0500 Subject: [PATCH 65/69] [MP][Core] Refactor MPCacheEngine for better extendability (#3391) * [Add] refactoring for the LMCache MP cache engine Signed-off-by: Yihua Cheng --- lmcache/v1/multiprocess/config.py | 21 +- lmcache/v1/multiprocess/engine_context.py | 157 ++ lmcache/v1/multiprocess/engine_module.py | 65 + lmcache/v1/multiprocess/http_server.py | 7 +- lmcache/v1/multiprocess/modules/__init__.py | 1 + .../{blend_server_v2.py => modules/blend.py} | 699 +++----- .../v1/multiprocess/modules/gpu_transfer.py | 670 +++++++ lmcache/v1/multiprocess/modules/lookup.py | 467 +++++ lmcache/v1/multiprocess/modules/management.py | 129 ++ .../multiprocess/modules/non_gpu_transfer.py | 347 ++++ lmcache/v1/multiprocess/server.py | 1543 ++--------------- tests/v1/multiprocess/test_blend_server_v2.py | 75 +- tests/v1/multiprocess/test_cache_server.py | 2 +- tests/v1/multiprocess/test_free_locks.py | 47 +- .../test_non_cuda_data_transfer.py | 42 +- .../v1/multiprocess/test_query_lookup_hits.py | 63 +- 16 files changed, 2375 insertions(+), 1960 deletions(-) create mode 100644 lmcache/v1/multiprocess/engine_context.py create mode 100644 lmcache/v1/multiprocess/engine_module.py create mode 100644 lmcache/v1/multiprocess/modules/__init__.py rename lmcache/v1/multiprocess/{blend_server_v2.py => modules/blend.py} (68%) create mode 100644 lmcache/v1/multiprocess/modules/gpu_transfer.py create mode 100644 lmcache/v1/multiprocess/modules/lookup.py create mode 100644 lmcache/v1/multiprocess/modules/management.py create mode 100644 lmcache/v1/multiprocess/modules/non_gpu_transfer.py diff --git a/lmcache/v1/multiprocess/config.py b/lmcache/v1/multiprocess/config.py index 8873bd24d08..71670947a80 100644 --- a/lmcache/v1/multiprocess/config.py +++ b/lmcache/v1/multiprocess/config.py @@ -39,9 +39,13 @@ class MPServerConfig: engine_type: str = "default" """Cache engine backend type - ('default' for MPCacheEngine, 'blend' for BlendEngineV2). + ('default' for standard prefix caching, 'blend' when cacheblend is enabled). """ + transfer_mode: str = "gpu" + """Transfer mode: 'gpu' for GPU-based IPC transfer (STORE/RETRIEVE), + 'non_gpu' for non-GPU-based transfer (PREPARE/COMMIT).""" + runtime_plugin_config: "RuntimePluginConfig" = field( default_factory=lambda: RuntimePluginConfig() ) @@ -146,9 +150,17 @@ def add_mp_server_args( type=str, default="default", choices=["default", "blend"], - help="Cache engine backend type. 'default' uses MPCacheEngine, " - "'blend' uses BlendEngineV2 for cross-request KV reuse. " - "Default is 'default'.", + help="Cache engine backend type. 'default' uses standard prefix caching, " + "'blend' when cacheblend is enabled. Default is 'default'.", + ) + mp_group.add_argument( + "--transfer-mode", + type=str, + default="gpu", + choices=["gpu", "non_gpu"], + help="Transfer mode: 'gpu' for GPU-based IPC transfer " + "(STORE/RETRIEVE), 'non_gpu' for non-GPU-based transfer " + "(PREPARE/COMMIT). Default is 'gpu'.", ) mp_group.add_argument( "--runtime-plugin-locations", @@ -198,6 +210,7 @@ def parse_args_to_mp_server_config( max_cpu_workers=max_cpu, hash_algorithm=args.hash_algorithm, engine_type=args.engine_type, + transfer_mode=args.transfer_mode, runtime_plugin_config=RuntimePluginConfig( locations=(args.runtime_plugin_locations or []), extra_config=plugin_extra, diff --git a/lmcache/v1/multiprocess/engine_context.py b/lmcache/v1/multiprocess/engine_context.py new file mode 100644 index 00000000000..890a56ca8c4 --- /dev/null +++ b/lmcache/v1/multiprocess/engine_context.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Shared context and layout descriptor registry for engine modules.""" + +# Standard +import threading + +# First Party +from lmcache.logging import init_logger +from lmcache.v1.distributed.api import ( + MemoryLayoutDesc, + ObjectKey, + ipc_key_to_object_keys, +) +from lmcache.v1.distributed.config import StorageManagerConfig +from lmcache.v1.distributed.storage_manager import StorageManager +from lmcache.v1.mp_observability.event_bus import EventBus, get_event_bus +from lmcache.v1.multiprocess.custom_types import IPCCacheEngineKey +from lmcache.v1.multiprocess.session import SessionManager +from lmcache.v1.multiprocess.token_hasher import TokenHasher + +logger = init_logger(__name__) + + +class LayoutDescRegistry: + """Thread-safe registry mapping (model_name, world_size) to MemoryLayoutDesc. + + Modules write to this registry when KV caches are registered. + Consumers (e.g. LookupModule) read from it to find layout descriptors + for prefetch tasks. + """ + + def __init__(self) -> None: + # Key: (model_name, world_size) -> MemoryLayoutDesc + self._registry: dict[tuple[str, int], MemoryLayoutDesc] = {} + self._lock = threading.Lock() + + def register( + self, + model_name: str, + world_size: int, + layout_desc: MemoryLayoutDesc, + ) -> None: + """Register a layout descriptor for a (model_name, world_size) pair. + + Args: + model_name: The model name. + world_size: The world size. + layout_desc: The memory layout descriptor. + """ + with self._lock: + self._registry[(model_name, world_size)] = layout_desc + + def unregister(self, model_name: str, world_size: int) -> None: + """Remove a layout descriptor for a (model_name, world_size) pair. + + Args: + model_name: The model name. + world_size: The world size. + """ + with self._lock: + self._registry.pop((model_name, world_size), None) + + def find(self, model_name: str, world_size: int) -> MemoryLayoutDesc | None: + """Look up a layout descriptor by (model_name, world_size). + + Args: + model_name: The model name. + world_size: The world size. + + Returns: + The layout descriptor if found, otherwise None. + """ + with self._lock: + return self._registry.get((model_name, world_size)) + + +class MPCacheEngineContext: + """Shared infrastructure for all engine modules. + + Holds the storage manager, token hasher, session manager, event bus, + and layout descriptor registry. Modules receive this context at init + and use it for shared operations. + + Args: + storage_manager_config: Configuration for the storage manager. + chunk_size: Chunk size for KV cache operations. + hash_algorithm: Hash algorithm for token hashing. + """ + + def __init__( + self, + storage_manager_config: StorageManagerConfig, + chunk_size: int = 256, + hash_algorithm: str = "blake3", + ) -> None: + self._chunk_size = chunk_size + self._storage_manager = StorageManager(storage_manager_config) + self._token_hasher = TokenHasher( + chunk_size=chunk_size, hash_algorithm=hash_algorithm + ) + self._session_manager = SessionManager(self._token_hasher) + self._event_bus = get_event_bus() + self._layout_desc_registry = LayoutDescRegistry() + + @property + def chunk_size(self) -> int: + """Chunk size for KV cache operations.""" + return self._chunk_size + + @property + def storage_manager(self) -> StorageManager: + """The storage manager instance.""" + return self._storage_manager + + @property + def token_hasher(self) -> TokenHasher: + """The token hasher for computing chunk hashes.""" + return self._token_hasher + + @property + def session_manager(self) -> SessionManager: + """The session manager for request lifecycle tracking.""" + return self._session_manager + + @property + def event_bus(self) -> EventBus: + """The event bus for observability events.""" + return self._event_bus + + @property + def layout_desc_registry(self) -> LayoutDescRegistry: + """Registry mapping (model_name, world_size) to MemoryLayoutDesc.""" + return self._layout_desc_registry + + def resolve_obj_keys(self, key: IPCCacheEngineKey) -> list[ObjectKey]: + """Resolve object keys from an IPC cache key. + + Uses the session manager to track token state and the token hasher + to compute chunk hashes for the requested range. + + Args: + key: IPC cache key describing model/session/token range. + + Returns: + Resolved object keys for the requested token range. + + Raises: + ValueError: If ``key.worker_id`` is ``None``. + """ + session = self.session_manager.get_or_create(key.request_id) + session.set_tokens(list(key.token_ids)) + chunk_hashes = [ + TokenHasher.hash_to_bytes(h) for h in session.get_hashes(key.start, key.end) + ] + if key.worker_id is None: + raise ValueError("Must resolve keys with worker_id != None") + return ipc_key_to_object_keys(key, chunk_hashes) diff --git a/lmcache/v1/multiprocess/engine_module.py b/lmcache/v1/multiprocess/engine_module.py new file mode 100644 index 00000000000..592d124adae --- /dev/null +++ b/lmcache/v1/multiprocess/engine_module.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Protocol and types for pluggable engine modules.""" + +# Future +from __future__ import annotations + +# Standard +from dataclasses import dataclass +from enum import Enum, auto +from typing import TYPE_CHECKING, Callable, Protocol + +# First Party +from lmcache.v1.multiprocess.protocol import RequestType + +if TYPE_CHECKING: + # First Party + from lmcache.v1.multiprocess.engine_context import MPCacheEngineContext + + +class ThreadPoolType(Enum): + """Declares which thread pool a handler should run in.""" + + SYNC = auto() + AFFINITY = auto() + NORMAL = auto() + + +@dataclass +class HandlerSpec: + """Specification for a single message queue handler. + + Args: + request_type: The ZMQ request type this handler serves. + handler: The callable that processes the request. + pool: Which thread pool the handler runs in. + """ + + request_type: RequestType + handler: Callable + pool: ThreadPoolType + + +class EngineModule(Protocol): + """Protocol for pluggable engine modules. + + Each module owns its internal state and exposes handlers + that the compositor registers with the message queue server. + """ + + @property + def context(self) -> MPCacheEngineContext: + """Return the shared engine context. Exposed for testing only.""" + ... + + def get_handlers(self) -> list[HandlerSpec]: + """Return handler specs for all request types this module serves.""" + ... + + def report_status(self) -> dict: + """Return module-specific status information.""" + ... + + def close(self) -> None: + """Release resources owned by this module.""" + ... diff --git a/lmcache/v1/multiprocess/http_server.py b/lmcache/v1/multiprocess/http_server.py index c83540314ca..2e1f6d45f0d 100644 --- a/lmcache/v1/multiprocess/http_server.py +++ b/lmcache/v1/multiprocess/http_server.py @@ -35,6 +35,7 @@ from lmcache.v1.multiprocess.mp_runtime_plugin_launcher import ( MPRuntimePluginLauncher, ) +from lmcache.v1.multiprocess.server import run_cache_server logger = init_logger(__name__) @@ -61,12 +62,6 @@ async def lifespan(app: FastAPI): torch_dev.is_available(), ) mp_config = _configs["mp"] - if mp_config.engine_type == "blend": - # First Party - from lmcache.v1.multiprocess.blend_server_v2 import run_cache_server - else: - # First Party - from lmcache.v1.multiprocess.server import run_cache_server result = run_cache_server( mp_config=mp_config, diff --git a/lmcache/v1/multiprocess/modules/__init__.py b/lmcache/v1/multiprocess/modules/__init__.py new file mode 100644 index 00000000000..9881313609a --- /dev/null +++ b/lmcache/v1/multiprocess/modules/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/lmcache/v1/multiprocess/blend_server_v2.py b/lmcache/v1/multiprocess/modules/blend.py similarity index 68% rename from lmcache/v1/multiprocess/blend_server_v2.py rename to lmcache/v1/multiprocess/modules/blend.py index 1d355179a63..22b82419fac 100644 --- a/lmcache/v1/multiprocess/blend_server_v2.py +++ b/lmcache/v1/multiprocess/modules/blend.py @@ -1,41 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -""" -Overview --------- -This server enables KV cache reuse across requests that share token -sub-sequences at *arbitrary positions*, not only at a common prefix. - -Workflow (example: chunk_size = 3) ------------------------------------ -1. cb_store_pre_computed([1,2,3,4,5,6]) - Tokens are split into full chunks ([1,2,3] and [4,5,6]). Each chunk - is stored in the underlying storage under its normal rolling prefix - hash, and the chunk fingerprints are registered in - BlendTokenRangeMatcher for fast sub-sequence lookup. Because normal - hashes are used, these chunks are also accessible via the standard - lookup/retrieve path. - -2. cb_lookup_pre_computed([x,y,z, a,b,c, 4,5,6, m,n,p]) - BlendTokenRangeMatcher slides a rolling polynomial hash over the new - request's tokens and detects that the window at positions [6, 9) - matches the stored chunk [4,5,6]. A prefetch task is submitted for - that chunk using its stored hash as the storage key. Only chunks - confirmed present in storage are returned as CBMatchResult objects - (with cur_st/cur_ed pointing to their location in the new request). - -3. cb_retrieve_pre_computed(...) - The (prefetched) KV cache for each matched chunk is copied (CPU→GPU) - into the correct slot of the new request's KV cache buffer (at - cur_st + offset), so the LLM can skip recomputing those tokens. - -4. cb_store_final([x,y,z, a,b,c, 4,5,6, m,n,p]) - After inference completes on the new request, all its chunks are - stored under normal prefix hashes. Future requests sharing - any prefix of the new request will get standard prefix-cache hits. - Future requests sharing any prefix of the first request will also - get hits because cb_store_pre_computed already stored those chunks - under normal hashes. -""" +"""Blend (context-blend / cross-request KV reuse) module for MPCacheEngine.""" # Standard from typing import Any @@ -44,7 +8,6 @@ # Third Party import numpy as np -import zmq # First Party from lmcache import torch_dev, torch_device_type @@ -53,44 +16,26 @@ from lmcache.v1.distributed.api import ( MemoryLayoutDesc, ObjectKey, + PrefetchHandle, ipc_key_to_object_keys, ) -from lmcache.v1.distributed.config import ( - StorageManagerConfig, - parse_args_to_config, -) -from lmcache.v1.distributed.storage_manager import PrefetchHandle from lmcache.v1.gpu_connector.gpu_ops import ( lmcache_memcpy_async_d2h, lmcache_memcpy_async_h2d, ) -from lmcache.v1.mp_observability.config import ( - ObservabilityConfig, - init_observability, - parse_args_to_observability_config, -) from lmcache.v1.mp_observability.event import Event, EventType -from lmcache.v1.mp_observability.event_bus import get_event_bus -from lmcache.v1.mp_observability.trace import maybe_initialize_trace_recorder -from lmcache.v1.multiprocess.config import ( - MPServerConfig, - parse_args_to_mp_server_config, -) from lmcache.v1.multiprocess.custom_types import ( CBMatchResult, IPCCacheEngineKey, KVCache, ) -from lmcache.v1.multiprocess.gpu_context import ( - PlainGPUCacheContext, -) -from lmcache.v1.multiprocess.mq import MessageQueueServer -from lmcache.v1.multiprocess.protocol import ( - RequestType, - get_handler_type, - get_payload_classes, +from lmcache.v1.multiprocess.engine_context import MPCacheEngineContext +from lmcache.v1.multiprocess.engine_module import ( + HandlerSpec, + ThreadPoolType, ) -from lmcache.v1.multiprocess.server import MPCacheEngine, parse_args +from lmcache.v1.multiprocess.gpu_context import PlainGPUCacheContext +from lmcache.v1.multiprocess.protocols.base import RequestType from lmcache.v1.multiprocess.token_hasher import ( chunk_hash_windows_numba, rolling_hash_windows_numba, @@ -105,43 +50,46 @@ class BlendTokenRangeMatcher: """Fast token-range matcher using polynomial rolling/chunk hashes and a direct-address lookup table. - Table layout: poly_chunk_hash (u64) → compact_chunk_id (i64, sequential 0…N-1). + Table layout: poly_chunk_hash (u64) -> compact_chunk_id (i64, sequential 0...N-1). Because compact IDs are bounded by _TABLE_SIZE, unique_hits_direct_id_numba - can use a fixed `seen` array of _TABLE_SIZE bytes (~1 MB) rather than one - sized by an arbitrary max hash — no memory explosion. + can use a fixed ``seen`` array of _TABLE_SIZE bytes (~1 MB) rather than one + sized by an arbitrary max hash -- no memory explosion. Auxiliary storage: _chunk_token_hash[i] : token_hash for chunk i (None if evicted) - _token_hash_to_start : token_hash → start position in seq + _token_hash_to_start : token_hash -> start position in seq _compact_id_to_slot[i] : table slot for compact_id i - _token_hash_to_compact_id : token_hash → compact_chunk_id + _token_hash_to_compact_id : token_hash -> compact_chunk_id Methods: - on_new_token_hashes – register a sequence; builds fingerprints - and writes compact IDs. - match_sub_sequence – sliding-window probe → compact IDs → - token_hash → start. Skips evicted entries. - remove_chunks – lazily evict stale entries. Clears the - table slot and auxiliary maps. + on_new_token_hashes -- register a sequence; builds fingerprints + and writes compact IDs. + match_sub_sequence -- sliding-window probe -> compact IDs -> + token_hash -> start. Skips evicted entries. + remove_chunks -- lazily evict stale entries. Clears the + table slot and auxiliary maps. + + Args: + chunk_size: Number of tokens per chunk for fingerprint computation. """ - _TABLE_BITS: int = 20 # 2^20 ≈ 1 M entries + _TABLE_BITS: int = 20 # 2^20 ~ 1 M entries _TABLE_SIZE: int = 1 << _TABLE_BITS _BASE: np.uint64 = np.uint64(0x9E3779B97F4A7C15) # Fibonacci-hashing constant def __init__(self, chunk_size: int = 256): self.chunk_size = chunk_size - # poly_chunk_hash → compact_chunk_id; -1 = empty + # poly_chunk_hash -> compact_chunk_id; -1 = empty self._table_id = np.full(self._TABLE_SIZE, -1, dtype=np.int64) self._mask = np.uint64(self._TABLE_SIZE - 1) - # compact_chunk_id → caller-supplied token_hash (full bytes) + # compact_chunk_id -> caller-supplied token_hash (full bytes) self._chunk_token_hash: list[bytes | None] = [] - # token_hash → start position in its registered sequence + # token_hash -> start position in its registered sequence self._token_hash_to_start: dict[bytes, int] = {} - # compact_chunk_id → table slot index (for reverse lookup during eviction) + # compact_chunk_id -> table slot index (for reverse lookup during eviction) self._compact_id_to_slot = np.full(self._TABLE_SIZE, -1, dtype=np.int64) - # token_hash → compact_chunk_id (for eviction lookup) + # token_hash -> compact_chunk_id (for eviction lookup) self._token_hash_to_compact_id: dict[bytes, int] = {} self._lock = threading.Lock() @@ -149,7 +97,7 @@ def on_new_token_hashes( self, token_ids: list[int], token_hashes: list[bytes], - ): + ) -> None: """Register a new token sequence and index its non-overlapping chunks. Args: @@ -207,10 +155,10 @@ def on_new_token_hashes( ) compact_ids = np.arange(base_id, base_id + n_new, dtype=np.int64) - # Write table: poly_chunk_hash → compact_chunk_id + # Write table: poly_chunk_hash -> compact_chunk_id update_table_id_numba(new_chunk_hashes, self._table_id, compact_ids) - # Persist compact_id → token_hash, token_hash → start, and reverse maps + # Persist compact_id -> token_hash, token_hash -> start, and reverse maps for k, orig_i in enumerate(new_idxs): th = token_hashes[orig_i] cid = int(compact_ids[k]) @@ -363,28 +311,108 @@ def _unique_token_coverage(results: list[CBMatchResult]) -> int: return coverage -# Main class and main functions -class BlendEngineV2(MPCacheEngine): - def __init__( - self, - storage_manager_config: StorageManagerConfig, - chunk_size: int = 256, - hash_algorithm: str = "blake3", - ): - super().__init__( - storage_manager_config, chunk_size, hash_algorithm=hash_algorithm - ) +class BlendModule: + """Handles blend (context-blend / cross-request KV reuse) operations. - self._cb_gpu_contexts: dict[int, PlainGPUCacheContext] = {} + Owns CB-specific GPU context registrations and the token range matcher. + Provides handlers for CB register, unregister, store, retrieve, and lookup. + + Args: + ctx: The shared engine context. + """ - # CB GPU ID -> (model name, world size) as metadata - # NOTE: This is mainly for determining the layout desc during prefetch + def __init__(self, ctx: MPCacheEngineContext) -> None: + self._ctx = ctx + self._cb_gpu_contexts: dict[int, PlainGPUCacheContext] = {} self._cb_gpu_context_meta: dict[int, tuple[str, int]] = {} + self._token_range_matcher = BlendTokenRangeMatcher(ctx.chunk_size) + self._gpu_copy_lock = threading.Lock() + + @property + def context(self) -> MPCacheEngineContext: + """Return the shared engine context. Exposed for testing only.""" + return self._ctx + + def get_handlers(self) -> list[HandlerSpec]: + """Return handler specs for all request types this module serves. + + Returns: + A list of HandlerSpec entries mapping request types to + their handler callables and thread pool assignments. + """ + return [ + HandlerSpec( + RequestType.CB_REGISTER_KV_CACHE, + self.cb_register_kv_cache, + ThreadPoolType.SYNC, + ), + HandlerSpec( + RequestType.CB_UNREGISTER_KV_CACHE, + self.cb_unregister_kv_cache, + ThreadPoolType.SYNC, + ), + HandlerSpec( + RequestType.CB_STORE_PRE_COMPUTED, + self.cb_store_pre_computed, + ThreadPoolType.AFFINITY, + ), + HandlerSpec( + RequestType.CB_RETRIEVE_PRE_COMPUTED_V2, + self.cb_retrieve_pre_computed, + ThreadPoolType.AFFINITY, + ), + HandlerSpec( + RequestType.CB_STORE_FINAL, + self.cb_store_final, + ThreadPoolType.AFFINITY, + ), + HandlerSpec( + RequestType.CB_LOOKUP_PRE_COMPUTED_V2, + self.cb_lookup_pre_computed, + ThreadPoolType.NORMAL, + ), + ] + + def report_status(self) -> dict: + """Return blend module status information. + + Returns: + A dict containing registered CB GPU instance IDs and + per-instance KV cache layout metadata. + """ + cb_gpu_context_meta: dict[str, dict] = {} + for gpu_id, meta in self._cb_gpu_context_meta.items(): + model_name, world_size = meta + entry: dict = { + "model_name": model_name, + "world_size": world_size, + } + ctx = self._cb_gpu_contexts.get(gpu_id) + if ctx is not None: + # bytes per token = 2 (K+V) * num_layers * hidden_dim_size * + # itemsize; num_tokens is the cache capacity, not a per-token + # cost. + cache_size_per_token = ( + 2 * ctx.num_layers * ctx.hidden_dim_size * ctx.dtype.itemsize + ) + entry["kv_cache_layout"] = { + "num_layers": ctx.num_layers, + "num_tokens": ctx.num_tokens, + "hidden_dim_size": ctx.hidden_dim_size, + "dtype": str(ctx.dtype), + "cache_size_per_token": cache_size_per_token, + } + cb_gpu_context_meta[str(gpu_id)] = entry - # Fast local matcher: indexes pre-computed chunk hashes for sub-sequence lookup - self._token_range_matcher = BlendTokenRangeMatcher(chunk_size) + return { + "registered_cb_gpu_ids": list(self._cb_gpu_contexts.keys()), + "cb_gpu_context_meta": cb_gpu_context_meta, + } - self._event_bus = get_event_bus() + def close(self) -> None: + """Release resources owned by this module.""" + self._cb_gpu_contexts.clear() + self._cb_gpu_context_meta.clear() def cb_register_kv_cache( self, @@ -393,18 +421,24 @@ def cb_register_kv_cache( model_name: str, world_size: int, ) -> None: - """ - Register the KV cache buffer from the blend engine + """Register the KV cache buffer from the blend engine. Args: - instance_id: Unique identifier for the blend engine instance - kv_caches: KVCache object containing the GPU buffer pointers + instance_id: Unique identifier for the blend engine instance. + kv_caches: KVCache object containing the GPU buffer pointers. model_name: The name of the model associated with this KV cache. world_size: The world size associated with this KV cache. """ - gpu_context = PlainGPUCacheContext(kv_caches, self.chunk_size) + gpu_context = PlainGPUCacheContext(kv_caches, self._ctx.chunk_size) self._cb_gpu_contexts[instance_id] = gpu_context self._cb_gpu_context_meta[instance_id] = (model_name, world_size) + + layout_desc = MemoryLayoutDesc( + shapes=[gpu_context.get_kv_buffer_shape(self._ctx.chunk_size)], + dtypes=[gpu_context.dtype], + ) + self._ctx.layout_desc_registry.register(model_name, world_size, layout_desc) + logger.info( "Registered CB KV cache for instance_id %d with %d layers", instance_id, @@ -412,11 +446,11 @@ def cb_register_kv_cache( ) def cb_unregister_kv_cache(self, instance_id: int) -> None: - """ - Unregister the KV cache buffer for the given instance_id + """Unregister the KV cache buffer for the given instance_id. Args: - instance_id: Unique identifier for the blend engine instance to unregister + instance_id: Unique identifier for the blend engine instance + to unregister. """ if instance_id in self._cb_gpu_contexts: del self._cb_gpu_contexts[instance_id] @@ -428,46 +462,8 @@ def cb_unregister_kv_cache(self, instance_id: int) -> None: instance_id, ) - def report_status(self) -> dict: - """Return a status dict for the entire cache engine. - - Extends the base dict with ``registered_cb_gpu_ids`` and - ``cb_gpu_context_meta`` so CB-only deployments are distinguishable - from "no engine connected" to ``/api/status`` clients. - """ - status = super().report_status() - - cb_gpu_context_meta: dict[str, dict] = {} - for gpu_id, meta in self._cb_gpu_context_meta.items(): - model_name, world_size = meta - entry: dict = { - "model_name": model_name, - "world_size": world_size, - } - ctx = self._cb_gpu_contexts.get(gpu_id) - if ctx is not None: - # bytes per token = 2 (K+V) * num_layers * hidden_dim_size * - # itemsize; num_tokens is the cache capacity, not a per-token - # cost. - cache_size_per_token = ( - 2 * ctx.num_layers * ctx.hidden_dim_size * ctx.dtype.itemsize - ) - entry["kv_cache_layout"] = { - "num_layers": ctx.num_layers, - "num_tokens": ctx.num_tokens, - "hidden_dim_size": ctx.hidden_dim_size, - "dtype": str(ctx.dtype), - "cache_size_per_token": cache_size_per_token, - } - cb_gpu_context_meta[str(gpu_id)] = entry - - status["registered_cb_gpu_ids"] = list(self._cb_gpu_contexts.keys()) - status["cb_gpu_context_meta"] = cb_gpu_context_meta - return status - def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: - """ - Lookup the pre-computed chunks in the underlying storage. + """Lookup the pre-computed chunks in the underlying storage. Uses BlendTokenRangeMatcher for a fast local pre-filter, then submits prefetch tasks for matched chunks using their stored hashes directly. @@ -475,20 +471,20 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: storage are lazily evicted from the matcher via remove_chunks. Args: - key: IPCCacheEngineKey containing the token ids to lookup + key: IPCCacheEngineKey containing the token ids to lookup. Returns: List of CBMatchResult for chunks that were actually found in storage, ready to be passed to cb_retrieve_pre_computed. """ num_tokens = len(key.token_ids) - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_REQUEST_START, session_id=key.request_id, ) ) - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_LOOKUP_START, session_id=key.request_id, @@ -496,18 +492,11 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: ) ) - # Sub-sequence fingerprint match: find CB-stored chunks anywhere in the - # query using polynomial rolling hashes (context-independent, so a chunk - # stored at position 0 is found even when it appears at CHUNK_SIZE in the - # query). Only chunks registered via cb_store_pre_computed / cb_store_final - # are in the fingerprint table, which gives CB isolation for free — chunks - # stored via the normal STORE path are never registered and therefore - # never returned here. cb_match_result = self._token_range_matcher.match_sub_sequence( list(key.token_ids) ) if not cb_match_result: - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_LOOKUP_END, session_id=key.request_id, @@ -519,12 +508,12 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: "stale_chunks": 0, "no_gpu_context": False, "hit_tokens": 0, - "requested_tokens": (num_tokens // self.chunk_size) - * self.chunk_size, + "requested_tokens": (num_tokens // self._ctx.chunk_size) + * self._ctx.chunk_size, }, ) ) - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_REQUEST_END, session_id=key.request_id, @@ -564,7 +553,7 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: if m_name == model_name and w_size == world_size: cb_ctx = self._cb_gpu_contexts[gpu_id] layout_desc = MemoryLayoutDesc( - shapes=[cb_ctx.get_kv_buffer_shape(self.chunk_size)], + shapes=[cb_ctx.get_kv_buffer_shape(self._ctx.chunk_size)], dtypes=[cb_ctx.dtype], ) break @@ -576,7 +565,7 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: model_name, world_size, ) - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_LOOKUP_END, session_id=key.request_id, @@ -588,12 +577,12 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: "stale_chunks": 0, "no_gpu_context": True, "hit_tokens": 0, - "requested_tokens": (num_tokens // self.chunk_size) - * self.chunk_size, + "requested_tokens": (num_tokens // self._ctx.chunk_size) + * self._ctx.chunk_size, }, ) ) - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_REQUEST_END, session_id=key.request_id, @@ -607,7 +596,7 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: for group in groups: chunk_hashes = [r.hash for r in group] obj_keys = ipc_key_to_object_keys(key, chunk_hashes) - handle = self.storage_manager.submit_prefetch_task( + handle = self._ctx.storage_manager.submit_prefetch_task( obj_keys, layout_desc, external_request_id=key.request_id, @@ -620,17 +609,14 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: group[0].cur_st, ) - # TODO(Jiayi): We need to follow how lookup is handled in server.py - # to optimize performance. # Collect only the CBMatchResults for chunks actually found in storage stale_hashes: list[bytes] = [] for handle, group in zip(prefetch_handles, groups, strict=False): found_count = None while True: - found_count = self.storage_manager.query_prefetch_status(handle) + found_count = self._ctx.storage_manager.query_prefetch_status(handle) if found_count is not None: break - time.sleep(0.001) # Real found count after dedup the TP @@ -664,14 +650,14 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: "Evicted %d stale chunks from fingerprint table", len(stale_hashes), ) - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_CHUNKS_EVICTED, metadata={"num_chunks": len(stale_hashes)}, ) ) - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_LOOKUP_END, session_id=key.request_id, @@ -683,8 +669,8 @@ def cb_lookup_pre_computed(self, key: IPCCacheEngineKey) -> list[CBMatchResult]: "stale_chunks": len(stale_hashes), "no_gpu_context": False, "hit_tokens": _unique_token_coverage(found_cb_match_result), - "requested_tokens": (num_tokens // self.chunk_size) - * self.chunk_size, + "requested_tokens": (num_tokens // self._ctx.chunk_size) + * self._ctx.chunk_size, }, ) ) @@ -698,8 +684,7 @@ def _cb_store_gpu_copy( event_ipc_handle: bytes, start_event: Event | None = None, ) -> tuple[Any, dict]: - """ - Helper function to perform GPU-to-CPU copy operations for storing chunks. + """Helper function to perform GPU-to-CPU copy operations for storing chunks. Args: obj_keys: List of object keys to store. @@ -736,16 +721,18 @@ def _cb_store_gpu_copy( vllm_event.wait(stream=gpu_context.stream) if start_event is not None: - self._event_bus.publish_on_stream(gpu_context.cupy_stream, start_event) + self._ctx.event_bus.publish_on_stream( + gpu_context.cupy_stream, start_event + ) # Prepare for the copy - num_tokens = self.chunk_size + num_tokens = self._ctx.chunk_size cpu_shape = gpu_context.get_kv_buffer_shape(num_tokens) layout_desc = MemoryLayoutDesc( shapes=[cpu_shape], dtypes=[gpu_context.dtype] ) - reserved_dict = self.storage_manager.reserve_write( + reserved_dict = self._ctx.storage_manager.reserve_write( obj_keys, layout_desc, "new" ) @@ -755,22 +742,22 @@ def _cb_store_gpu_copy( else: continue - offset_start = idx * self.chunk_size + offset - offset_end = offset_start + self.chunk_size + offset_start = idx * self._ctx.chunk_size + offset + offset_end = offset_start + self._ctx.chunk_size # Copy from GPU to CPU tmp_buffer = gpu_context.get_tmp_gpu_buffer(offset_end - offset_start) gpu_kv_slice = gpu_context.slice_kv_cache_on_tokens( offset_start, offset_end ) - with self.lock: + with self._gpu_copy_lock: tmp_buffer.copy_(gpu_kv_slice, non_blocking=True) lmcache_memcpy_async_d2h(tmp_buffer, memory_obj) event.record() # Call finish_write after the copy is done gpu_context.cupy_stream.launch_host_func( - self.storage_manager.finish_write, + self._ctx.storage_manager.finish_write, list(reserved_dict.keys()), ) @@ -783,46 +770,49 @@ def cb_store_pre_computed( instance_id: int, event_ipc_handle: bytes, ) -> tuple[bytes, bool]: - """ - Store the pre-computed chunks in the underlying storage for later retrieval. + """Store the pre-computed chunks in the underlying storage for later retrieval. Args: - key: IPCCacheEngineKey containing the token ids for which the pre-computed - chunks are stored. + key: IPCCacheEngineKey containing the token ids for which the + pre-computed chunks are stored. offset: The starting offset in the CB KV cache buffer where the - pre-computed - instance_id: The instance_id of the blend engine instance to store the - pre-computed chunks for. + pre-computed chunks begin. + instance_id: The instance_id of the blend engine instance to store + the pre-computed chunks for. event_ipc_handle: The IPC handle for the CUDA event that signals the completion of LLM inference. Returns: - IPC handle bytes for the event that signals the completion of storing the - pre-computed chunks, and a boolean flag indicating if the store is - successful. + IPC handle bytes for the event that signals the completion of storing + the pre-computed chunks, and a boolean flag indicating if the store + is successful. + + Raises: + ValueError: If instance_id is not registered for CB KV cache. Note: - The input tokens should not have any separator in it. It should just be - one "paragraph". - This function will discard the last partial chunk and only store the full - chunks + The input tokens should not have any separator in it. It should just + be one "paragraph". + This function will discard the last partial chunk and only store the + full chunks. """ num_tokens = key.end - key.start - assert instance_id in self._cb_gpu_contexts, ( - f"Instance ID {instance_id} not registered for CB KV cache" - ) + if instance_id not in self._cb_gpu_contexts: + raise ValueError( + f"Instance ID {instance_id} not registered for CB KV cache" + ) gpu_context = self._cb_gpu_contexts[instance_id] # CPU-synchronous sentinel: GPU store is about to be enqueued. - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_STORE_PRE_COMPUTED_SUBMITTED, session_id=key.request_id, metadata={"instance_id": instance_id}, ) ) - self._event_bus.publish_on_stream( + self._ctx.event_bus.publish_on_stream( gpu_context.cupy_stream, Event( event_type=EventType.CB_STORE_PRE_COMPUTED_START, @@ -833,7 +823,7 @@ def cb_store_pre_computed( # Compute normal prefix hashes so these chunks are accessible both via # the CB lookup path and via the standard lookup/retrieve path. - chunk_hashes = self.token_hasher.compute_chunk_hashes(list(key.token_ids)) + chunk_hashes = self._ctx.token_hasher.compute_chunk_hashes(list(key.token_ids)) # convert to object key obj_keys = ipc_key_to_object_keys(key, chunk_hashes) @@ -852,7 +842,7 @@ def cb_store_pre_computed( self._token_range_matcher.on_new_token_hashes( list(key.token_ids), token_hashes ) - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_FINGERPRINTS_REGISTERED, session_id=key.request_id, @@ -870,7 +860,7 @@ def cb_store_pre_computed( ) except Exception: logger.exception("Cannot store pre-computed chunks due to exception") - self._event_bus.publish_on_stream( + self._ctx.event_bus.publish_on_stream( gpu_context.cupy_stream, Event( event_type=EventType.CB_STORE_PRE_COMPUTED_END, @@ -885,7 +875,7 @@ def cb_store_pre_computed( ) raise - self._event_bus.publish_on_stream( + self._ctx.event_bus.publish_on_stream( gpu_context.cupy_stream, Event( event_type=EventType.CB_STORE_PRE_COMPUTED_END, @@ -908,33 +898,37 @@ def cb_retrieve_pre_computed( instance_id: int, event_ipc_handle: bytes, ) -> tuple[bytes, bool]: - """ - Retrieve the pre-computed chunks from the underlying storage and copy them to - the CB KV cache buffer. + """Retrieve pre-computed chunks from storage and copy them to the CB KV buffer. Args: - key: IPCCacheEngineKey containing the token ids for which the pre-computed - chunks are retrieved. - cb_match_result: List of CBMatchResult returned by cb_lookup_pre_computed, - containing the per-chunk hashes and query positions. - offset: The starting offset in the CB KV cache buffer to copy the retrieved - chunks to. - instance_id: The instance_id of the blend engine instance to retrieve the - pre-computed chunks for. - event_ipc_handle: The IPC handle for the CUDA event that signals the - completion of LLM inference. + key: IPCCacheEngineKey containing the token ids for which the + pre-computed chunks are retrieved. + cb_match_result: List of CBMatchResult returned by + cb_lookup_pre_computed, containing the per-chunk hashes and + query positions. + offset: The starting offset in the CB KV cache buffer to copy the + retrieved chunks to. + instance_id: The instance_id of the blend engine instance to + retrieve the pre-computed chunks for. + event_ipc_handle: The IPC handle for the CUDA event that signals + the completion of LLM inference. Returns: - IPC handle bytes for the event that signals the completion of retrieving the - pre-computed chunks, and a boolean flag indicating if the retrieval is - successful. + IPC handle bytes for the event that signals the completion of + retrieving the pre-computed chunks, and a boolean flag indicating + if the retrieval is successful. + + Raises: + ValueError: If instance_id is not registered for CB KV cache. Note: - We must call `cb_lookup_pre_computed` first before calling this function + cb_lookup_pre_computed must be called first before calling this + function. """ - assert instance_id in self._cb_gpu_contexts, ( - f"Instance ID {instance_id} not registered for CB KV cache" - ) + if instance_id not in self._cb_gpu_contexts: + raise ValueError( + f"Instance ID {instance_id} not registered for CB KV cache" + ) gpu_context = self._cb_gpu_contexts[instance_id] # One obj_key per match_result, in cur_st order @@ -944,7 +938,7 @@ def cb_retrieve_pre_computed( all_obj_keys = ipc_key_to_object_keys(key, chunk_hashes) # CPU-synchronous sentinel: GPU retrieve is about to be enqueued. - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_RETRIEVE_SUBMITTED, session_id=key.request_id, @@ -962,22 +956,25 @@ def cb_retrieve_pre_computed( check_interprocess_event_support() event = torch_dev.Event(interprocess=True) - self._event_bus.publish_on_stream( + self._ctx.event_bus.publish_on_stream( gpu_context.cupy_stream, Event( event_type=EventType.CB_RETRIEVE_START, session_id=key.request_id, - metadata={"instance_id": instance_id, "num_chunks": num_chunks}, + metadata={ + "instance_id": instance_id, + "num_chunks": num_chunks, + }, ), ) try: - with self.storage_manager.read_prefetched_results( + with self._ctx.storage_manager.read_prefetched_results( all_obj_keys ) as memory_objs: if memory_objs is None: logger.error("Some keys not found during CB retrieve!") - self._event_bus.publish_on_stream( + self._ctx.event_bus.publish_on_stream( gpu_context.cupy_stream, Event( event_type=EventType.CB_RETRIEVE_END, @@ -995,18 +992,20 @@ def cb_retrieve_pre_computed( cb_match_result, memory_objs, strict=False ): gpu_st = r.cur_st + offset - gpu_ed = gpu_st + self.chunk_size - tmp_buffer = gpu_context.get_tmp_gpu_buffer(self.chunk_size) + gpu_ed = gpu_st + self._ctx.chunk_size + tmp_buffer = gpu_context.get_tmp_gpu_buffer( + self._ctx.chunk_size + ) target_buffer = gpu_context.slice_kv_cache_on_tokens( gpu_st, gpu_ed ) - with self.lock: + with self._gpu_copy_lock: lmcache_memcpy_async_h2d(memory_obj, tmp_buffer) target_buffer.copy_(tmp_buffer, non_blocking=True) except Exception: logger.exception("Error during retrieving prefetched results") - self._event_bus.publish_on_stream( + self._ctx.event_bus.publish_on_stream( gpu_context.cupy_stream, Event( event_type=EventType.CB_RETRIEVE_END, @@ -1027,7 +1026,8 @@ def cb_retrieve_pre_computed( # We should consider not unlocking objects in read_prefetched_results # if error happens. gpu_context.cupy_stream.launch_host_func( - self.storage_manager.finish_read_prefetched, all_obj_keys + self._ctx.storage_manager.finish_read_prefetched, + all_obj_keys, ) logger.info( @@ -1035,7 +1035,7 @@ def cb_retrieve_pre_computed( len(cb_match_result), offset, ) - self._event_bus.publish_on_stream( + self._ctx.event_bus.publish_on_stream( gpu_context.cupy_stream, Event( event_type=EventType.CB_RETRIEVE_END, @@ -1056,43 +1056,48 @@ def cb_store_final( instance_id: int, event_ipc_handle: bytes, ) -> tuple[bytes, bool]: - """ - Store the final chunks in the underlying storage after processing. The stored - chunk should be accessible for normal mode LLMs. + """Store the final chunks in the underlying storage after processing. + + The stored chunks should be accessible for normal mode LLMs. Args: - key: IPCCacheEngineKey containing the token ids for which the final chunks - are stored. - offset: The starting offset in the CB KV cache buffer where the final - chunks are stored. - instance_id: The instance_id of the blend engine instance to store the final - chunks for. - event_ipc_handle: The IPC handle for the CUDA event that signals the - completion of LLM inference. + key: IPCCacheEngineKey containing the token ids for which the + final chunks are stored. + offset: The starting offset in the CB KV cache buffer where the + final chunks are stored. + instance_id: The instance_id of the blend engine instance to + store the final chunks for. + event_ipc_handle: The IPC handle for the CUDA event that signals + the completion of LLM inference. Returns: - IPC handle bytes for the event that signals the completion of storing the - final chunks, and a boolean flag indicating if the store is successful. + IPC handle bytes for the event that signals the completion of + storing the final chunks, and a boolean flag indicating if the + store is successful. + + Raises: + ValueError: If instance_id is not registered for CB KV cache. """ num_tokens = key.end - key.start # Get GPU context - assert instance_id in self._cb_gpu_contexts, ( - f"Instance ID {instance_id} not registered for CB KV cache" - ) + if instance_id not in self._cb_gpu_contexts: + raise ValueError( + f"Instance ID {instance_id} not registered for CB KV cache" + ) gpu_context = self._cb_gpu_contexts[instance_id] # CPU-synchronous sentinels: SUBMITTED before SESSION_END so the # tracing subscriber's in-flight counter is non-zero when SESSION_END # arrives and correctly defers root span closure. - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_STORE_FINAL_SUBMITTED, session_id=key.request_id, metadata={"instance_id": instance_id}, ) ) - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_REQUEST_END, session_id=key.request_id, @@ -1100,7 +1105,7 @@ def cb_store_final( ) # Compute normal hash for the keys - chunk_hashes = self.token_hasher.compute_chunk_hashes(list(key.token_ids)) + chunk_hashes = self._ctx.token_hasher.compute_chunk_hashes(list(key.token_ids)) # convert to object key obj_keys = ipc_key_to_object_keys(key, chunk_hashes) @@ -1115,7 +1120,10 @@ def cb_store_final( start_event=Event( event_type=EventType.CB_STORE_FINAL_START, session_id=key.request_id, - metadata={"instance_id": instance_id, "num_tokens": num_tokens}, + metadata={ + "instance_id": instance_id, + "num_tokens": num_tokens, + }, ), ) @@ -1126,7 +1134,7 @@ def cb_store_final( self._token_range_matcher.on_new_token_hashes( list(key.token_ids), list(chunk_hashes) ) - self._event_bus.publish( + self._ctx.event_bus.publish( Event( event_type=EventType.CB_FINGERPRINTS_REGISTERED, session_id=key.request_id, @@ -1144,7 +1152,7 @@ def cb_store_final( ) except Exception: logger.exception("Cannot store final chunks due to exception") - self._event_bus.publish_on_stream( + self._ctx.event_bus.publish_on_stream( gpu_context.cupy_stream, Event( event_type=EventType.CB_STORE_FINAL_END, @@ -1159,7 +1167,7 @@ def cb_store_final( ) raise - self._event_bus.publish_on_stream( + self._ctx.event_bus.publish_on_stream( gpu_context.cupy_stream, Event( event_type=EventType.CB_STORE_FINAL_END, @@ -1173,172 +1181,3 @@ def cb_store_final( ), ) return event.ipc_handle(), True - - -def add_handler_helper( - server: MessageQueueServer, request_type: RequestType, handler_function -): - payload_classes = get_payload_classes(request_type) - handler_type = get_handler_type(request_type) - server.add_handler( - request_type, - payload_classes, - handler_type, - handler_function, - ) - - -def run_cache_server( - mp_config: MPServerConfig, - storage_manager_config: StorageManagerConfig, - obs_config: ObservabilityConfig, - return_engine: bool = False, - start_prometheus_http_server: bool = True, -): - """ - Run the LMCache cache server with ZMQ message queue. - - Args: - mp_config: Configuration for the ZMQ multiprocess server - storage_manager_config: Configuration for the storage manager - obs_config: Configuration for the observability stack - return_engine: If True, return (server, engine) after starting; - if False, run blocking loop to keep server alive - start_prometheus_http_server: Whether to start a standalone - Prometheus HTTP server in a background thread. Set to - ``False`` when an external HTTP framework already serves - ``/metrics`` to avoid port conflicts or redundant servers. - - Returns: - If return_engine is True: tuple of (MessageQueueServer, BlendEngineV2) - If return_engine is False: None (blocks until interrupted) - """ - event_bus = init_observability( - obs_config, start_prometheus_http_server=start_prometheus_http_server - ) - - # Wire up the trace recorder (no-op when --trace-level is unset). - maybe_initialize_trace_recorder(event_bus, obs_config, storage_manager_config) - - # Initialize the engine (loggers self-register with the global controller) - engine = BlendEngineV2( - storage_manager_config=storage_manager_config, - chunk_size=mp_config.chunk_size, - hash_algorithm=mp_config.hash_algorithm, - ) - - # Initialize the message queue server - context = zmq.Context.instance() - server = MessageQueueServer( - bind_url=f"tcp://{mp_config.host}:{mp_config.port}", - context=context, - ) - - # Add handlers for original server - add_handler_helper(server, RequestType.REGISTER_KV_CACHE, engine.register_kv_cache) - add_handler_helper( - server, RequestType.UNREGISTER_KV_CACHE, engine.unregister_kv_cache - ) - add_handler_helper(server, RequestType.STORE, engine.store) - add_handler_helper(server, RequestType.LOOKUP, engine.lookup) - add_handler_helper( - server, RequestType.QUERY_PREFETCH_STATUS, engine.query_prefetch_status - ) - add_handler_helper(server, RequestType.FREE_LOOKUP_LOCKS, engine.free_lookup_locks) - add_handler_helper(server, RequestType.RETRIEVE, engine.retrieve) - add_handler_helper(server, RequestType.CLEAR, engine.clear) - add_handler_helper(server, RequestType.GET_CHUNK_SIZE, engine.get_chunk_size) - add_handler_helper(server, RequestType.END_SESSION, engine.end_session) - add_handler_helper(server, RequestType.NOOP, engine.debug) - add_handler_helper( - server, - RequestType.REPORT_BLOCK_ALLOCATION, - engine.report_block_allocations, - ) - # Add handler for blend operations - add_handler_helper( - server, RequestType.CB_REGISTER_KV_CACHE, engine.cb_register_kv_cache - ) - add_handler_helper( - server, RequestType.CB_UNREGISTER_KV_CACHE, engine.cb_unregister_kv_cache - ) - add_handler_helper( - server, RequestType.CB_LOOKUP_PRE_COMPUTED_V2, engine.cb_lookup_pre_computed - ) - add_handler_helper( - server, RequestType.CB_STORE_PRE_COMPUTED, engine.cb_store_pre_computed - ) - add_handler_helper( - server, RequestType.CB_RETRIEVE_PRE_COMPUTED_V2, engine.cb_retrieve_pre_computed - ) - add_handler_helper(server, RequestType.CB_STORE_FINAL, engine.cb_store_final) - add_handler_helper(server, RequestType.PING, engine.ping) - - # Assign thread pools - server.add_affinity_thread_pool( - [ - RequestType.STORE, - RequestType.RETRIEVE, - RequestType.CB_STORE_PRE_COMPUTED, - RequestType.CB_RETRIEVE_PRE_COMPUTED_V2, - RequestType.CB_STORE_FINAL, - ], - max_workers=mp_config.max_gpu_workers, - ) - server.add_normal_thread_pool( - [ - RequestType.LOOKUP, - RequestType.QUERY_PREFETCH_STATUS, - RequestType.FREE_LOOKUP_LOCKS, - RequestType.END_SESSION, - RequestType.CLEAR, - RequestType.CB_LOOKUP_PRE_COMPUTED_V2, - RequestType.PING, - RequestType.REPORT_BLOCK_ALLOCATION, - ], - max_workers=mp_config.max_cpu_workers, - ) - - logger.info( - "LMCache ZMQ cache server is running on tcp://%s:%d", - mp_config.host, - mp_config.port, - ) - # Start the ZMQ server - # Not all backends expose init(); some auto-initialize on first use - if not hasattr(torch_dev, "init"): - logger.warning( - "Backend '%s' does not support init(), skipping device init", - torch_device_type, - ) - else: - torch_dev.init() - server.start() - - logger.info("LMCache cache blend v2 server is running...") - - # Return server and engine if requested (for HTTP server integration) - if return_engine: - return server, engine - - # Dummy loop to keep the server running - try: - while True: - time.sleep(1) - except KeyboardInterrupt: - logger.info("Shutting down server...") - event_bus.stop() - server.close() - engine.close() - - -if __name__ == "__main__": - args = parse_args() - mp_config = parse_args_to_mp_server_config(args) - storage_manager_config = parse_args_to_config(args) - obs_config = parse_args_to_observability_config(args) - run_cache_server( - mp_config=mp_config, - storage_manager_config=storage_manager_config, - obs_config=obs_config, - ) diff --git a/lmcache/v1/multiprocess/modules/gpu_transfer.py b/lmcache/v1/multiprocess/modules/gpu_transfer.py new file mode 100644 index 00000000000..536cafb20c5 --- /dev/null +++ b/lmcache/v1/multiprocess/modules/gpu_transfer.py @@ -0,0 +1,670 @@ +# SPDX-License-Identifier: Apache-2.0 +"""GPU-based KV cache transfer operations for the MPCacheEngine.""" + +# Standard +from dataclasses import dataclass +from itertools import islice +from typing import Generator +import time + +# First Party +from lmcache import torch_dev, torch_device_type +from lmcache.logging import init_logger +from lmcache.utils import ( + EngineType, + _lmcache_nvtx_annotate, + check_interprocess_event_support, +) +from lmcache.v1.distributed.api import ( + MemoryLayoutDesc, + ObjectKey, +) +from lmcache.v1.gpu_connector.gpu_ops import ( + lmcache_memcpy_async_d2h, + lmcache_memcpy_async_h2d, +) +from lmcache.v1.gpu_connector.utils import LayoutHints +from lmcache.v1.memory_management import MemoryObj +from lmcache.v1.mp_observability.event import Event, EventType +from lmcache.v1.multiprocess.custom_types import ( + IPCCacheEngineKey, + KVCache, +) +from lmcache.v1.multiprocess.engine_context import MPCacheEngineContext +from lmcache.v1.multiprocess.engine_module import ( + HandlerSpec, + ThreadPoolType, +) +from lmcache.v1.multiprocess.gpu_context import GPUCacheContext +from lmcache.v1.multiprocess.native_completion import ( + DeviceHostFuncDispatcher, + submit_callback_to_stream, +) +from lmcache.v1.multiprocess.protocols.base import RequestType +import lmcache.c_ops as lmc_ops + +logger = init_logger(__name__) + + +def get_layout_desc(gpu_context: GPUCacheContext, num_tokens: int) -> MemoryLayoutDesc: + """Get the memory layout description for a given GPU context and number of tokens. + + Supports multiple KV layer groups with different shapes and dtypes. + + Args: + gpu_context: The GPU cache context containing the KV cache information. + num_tokens: The number of tokens to determine the layout for. + + Returns: + MemoryLayoutDesc: The memory layout description containing shapes and dtypes. + """ + num_groups = gpu_context.kv_layer_groups_manager.num_groups + shapes = [ + gpu_context.get_kv_buffer_shape(num_tokens, group_idx) + for group_idx in range(num_groups) + ] + dtypes = [ + gpu_context.kv_layer_groups_manager.kv_layer_groups[group_idx].dtype + for group_idx in range(num_groups) + ] + return MemoryLayoutDesc(shapes=shapes, dtypes=dtypes) + + +def batched_iteration(lst: list, batch_size: int) -> Generator[tuple, None, None]: + """Utility function to iterate over a list in batches. + + Args: + lst: The list to iterate over. + batch_size: The size of each batch. + + Yields: + Batches of the list as tuples. + + Raises: + ValueError: If batch_size is less than 1. + """ + if batch_size < 1: + raise ValueError("batch size must be at least one") + it = iter(lst) + while batch := tuple(islice(it, batch_size)): + yield batch + + +@dataclass +class GPUContextEntry: + """Registered GPU context metadata for a single worker instance. + + Args: + gpu_context: The GPU cache context managing shape and pointers + to vLLM GPU KV cache tensors. + model_name: The name of the model associated with this KV cache. + world_size: The world size associated with this KV cache. + """ + + gpu_context: GPUCacheContext + model_name: str + world_size: int + + +class GPUTransferModule: + """Handles GPU-based KV cache transfer operations. + + Owns GPU context registrations and provides handlers for + register, unregister, store, and retrieve of GPU KV caches. + + Args: + ctx: The shared engine context. + """ + + def __init__(self, ctx: MPCacheEngineContext) -> None: + self._ctx = ctx + self._gpu_contexts: dict[int, GPUContextEntry] = {} + + # Route finish_write / finish_read_prefetched through a C++ host + # callback so the driver thread doesn't acquire the GIL. + self._device_host_func_dispatcher = DeviceHostFuncDispatcher() + self._device_host_func_dispatcher.register( + "finish_write", + self._ctx.storage_manager.finish_write, + payload_type=list[ObjectKey], + ) + self._device_host_func_dispatcher.register( + "finish_read_prefetched", + self._ctx.storage_manager.finish_read_prefetched, + payload_type=list[ObjectKey], + ) + self._device_host_func_dispatcher.start() + + @property + def context(self) -> MPCacheEngineContext: + """Return the shared engine context. Exposed for testing only.""" + return self._ctx + + def get_handlers(self) -> list[HandlerSpec]: + """Return handler specs for all request types this module serves. + + Returns: + A list of HandlerSpec entries mapping request types to + their handler callables and thread pool assignments. + """ + return [ + HandlerSpec( + RequestType.REGISTER_KV_CACHE, + self.register_kv_cache, + ThreadPoolType.SYNC, + ), + HandlerSpec( + RequestType.UNREGISTER_KV_CACHE, + self.unregister_kv_cache, + ThreadPoolType.SYNC, + ), + HandlerSpec( + RequestType.STORE, + self.store, + ThreadPoolType.AFFINITY, + ), + HandlerSpec( + RequestType.RETRIEVE, + self.retrieve, + ThreadPoolType.AFFINITY, + ), + ] + + def report_status(self) -> dict: + """Return GPU transfer module status information. + + Returns: + A dict containing registered GPU instance IDs and + per-instance KV cache layout metadata. + """ + registered_gpu_ids: list[int] = [] + gpu_context_meta: dict[str, dict] = {} + + for instance_id, entry in self._gpu_contexts.items(): + registered_gpu_ids.append(instance_id) + ctx = entry.gpu_context + gpu_context_meta[str(instance_id)] = { + "model_name": entry.model_name, + "world_size": entry.world_size, + "kv_cache_layout": { + "num_layers": ctx.num_layers, + "inference_engine_logical_block_size": ( + ctx.kv_layer_groups_manager.inference_engine_logical_block_size + ), + "group_physical_block_sizes": ctx.group_physical_block_sizes, + "group_compress_ratios": ctx.group_compress_ratios, + "hidden_dim_sizes": str(ctx.hidden_dim_sizes), + "dtype": str(ctx.dtype), + "is_mla": ctx.is_mla, + "num_blocks": ctx.num_blocks, + "gpu_kv_format": ctx.gpu_kv_format_name, + "gpu_kv_shape": ctx.gpu_kv_shape, + "gpu_kv_concrete_shape": ctx.concrete_gpu_kv_shape, + "attention_backend": ctx.attention_backend, + "cache_size_per_token": ctx.cache_size_per_token(), + }, + } + + return { + "registered_gpu_ids": registered_gpu_ids, + "gpu_context_meta": gpu_context_meta, + } + + def close(self) -> None: + """Release GPU resources owned by this module.""" + # Stop the drain thread before storage_manager.close() so any + # in-flight completions reach a live storage manager. + self._device_host_func_dispatcher.stop() + + had_contexts = len(self._gpu_contexts) > 0 + self._gpu_contexts.clear() + if had_contexts: + torch_dev.empty_cache() + + def register_kv_cache( + self, + instance_id: int, + kv_caches: KVCache, + model_name: str, + world_size: int, + engine_type: EngineType, + layout_hints: LayoutHints, + ) -> None: + """Register the KV cache tensors for a given GPU instance ID. + + Args: + instance_id: The GPU instance ID (such as PID). + kv_caches: The KV cache tensor wrappers from the + serving engine. + model_name: The name of the model associated with this KV cache. + world_size: The world size associated with this KV cache. + engine_type: Which serving engine produced the caches. + Forwarded to GPUCacheContext for format detection. + layout_hints: See LayoutHints. Forwarded to + GPUCacheContext for GPU KV format detection. + """ + if instance_id in self._gpu_contexts: + logger.warning( + "Instance %s's KV cache is already registered, " + "skipping the new registration", + instance_id, + ) + return + + gpu_context = GPUCacheContext( + kv_caches, + self._ctx.chunk_size, + layout_hints=layout_hints or None, + engine_type=engine_type, + ) + self._gpu_contexts[instance_id] = GPUContextEntry( + gpu_context=gpu_context, + model_name=model_name, + world_size=world_size, + ) + + layout_desc = get_layout_desc(gpu_context, self._ctx.chunk_size) + self._ctx.layout_desc_registry.register(model_name, world_size, layout_desc) + + logger.info( + "Registered KV cache for GPU ID %d with %d layers", + instance_id, + gpu_context.num_layers, + ) + + def unregister_kv_cache(self, instance_id: int) -> None: + """Unregister the KV cache tensors for a given GPU instance ID. + + Args: + instance_id: The GPU instance ID (such as PID). + """ + entry = self._gpu_contexts.pop(instance_id, None) + if entry is None: + logger.warning( + "No registered GPU context found for instance ID %d", instance_id + ) + return + + self._ctx.layout_desc_registry.unregister(entry.model_name, entry.world_size) + logger.info("Unregistered KV cache for GPU ID %d", instance_id) + torch_dev.empty_cache() + + @_lmcache_nvtx_annotate + def store( + self, + key: IPCCacheEngineKey, + instance_id: int, + gpu_block_ids: list[int], + event_ipc_handle: bytes, + ) -> tuple[bytes, bool]: + """Store the GPU KV cache blocks to CPU. + + Args: + key: The IPC key for the KV cache blocks. + Must have worker_id != None (worker store operation). + instance_id: The GPU instance ID (such as PID). + gpu_block_ids: The GPU block IDs to store. + event_ipc_handle: The IPC handle of the event to wait on. + + Returns: + A tuple where the first element is the IPC handle of the event + that signals the completion of the store operation, and the second + element indicates whether the store operation was successful. + + Raises: + ValueError: If no GPU context is registered for the given instance ID. + RuntimeError: If the backend does not support IPC event handles. + """ + st = time.perf_counter() + obj_keys = self._ctx.resolve_obj_keys(key) + + entry = self._gpu_contexts.get(instance_id) + if entry is None: + raise ValueError(f"No GPU context registered for instance ID {instance_id}") + gpu_context = entry.gpu_context + model_name = entry.model_name + + # ``blocks_per_chunk`` is counted in inference-engine-side + # blocks (each block addresses + # ``inference_engine_logical_block_size`` *logical* tokens). + # For compressed groups the per-group physical slot count + # differs, but the block-id indexing is shared with the engine + # and therefore uses the engine logical block size here. + blocks_per_chunk = ( + self._ctx.chunk_size + // gpu_context.kv_layer_groups_manager.inference_engine_logical_block_size + ) + + with ( + torch_dev.device(gpu_context.device), + torch_dev.stream(gpu_context.stream), + ): + check_interprocess_event_support() + event = torch_dev.Event(interprocess=True) + + all_block_ids_gpu = gpu_context.stage_block_ids(gpu_block_ids) + + if not hasattr(torch_dev.Event, "from_ipc_handle"): + raise RuntimeError( + f"Backend '{torch_device_type}' does not support IPC event " + "handles (Event.from_ipc_handle not available). " + "Multiprocess IPC requires CUDA." + ) + vllm_event = torch_dev.Event.from_ipc_handle( + gpu_context.device, event_ipc_handle + ) + vllm_event.wait(stream=gpu_context.stream) + + # CPU-synchronous sentinel: a GPU store is about to be enqueued. + # Must be published via publish() (not publish_on_stream) so the + # drain thread sees it before MP_REQUEST_END can race MP_STORE_END. + self._ctx.event_bus.publish( + Event( + event_type=EventType.MP_STORE_SUBMITTED, + session_id=key.request_id, + metadata={"device": str(gpu_context.device)}, + ) + ) + + self._ctx.event_bus.publish_on_stream( + gpu_context.cupy_stream, + Event( + event_type=EventType.MP_STORE_START, + session_id=key.request_id, + metadata={ + "device": str(gpu_context.device), + "engine_id": instance_id, + "model_name": model_name, + }, + ), + ) + + reserved_dict: dict[ObjectKey, MemoryObj] = {} + try: + layout_desc = get_layout_desc(gpu_context, self._ctx.chunk_size) + reserved_dict = self._ctx.storage_manager.reserve_write( + obj_keys, layout_desc, "new" + ) + + # NOTE: Store is not batched because some obj_keys may be + # skipped (not in reserved_dict), making block_ids + # non-contiguous. Batching would require torch.cat to + # reassemble block_ids, negating the benefit. + num_groups = gpu_context.kv_layer_groups_manager.num_groups + for idx, obj_key in enumerate(obj_keys): + if obj_key in reserved_dict: + memory_obj = reserved_dict[obj_key] + else: + continue + + chunk_block_ids_gpu = all_block_ids_gpu[ + idx * blocks_per_chunk : (idx + 1) * blocks_per_chunk + ] + + # Copy from GPU paged buffer to tmp buffer, then to CPU — per group + for group_idx in range(num_groups): + tmp_buffer = gpu_context.get_tmp_chunk_gpu_buffer(group_idx) + group_kv_pointers = gpu_context.get_group_kv_pointers(group_idx) + # Kernel contract: ``group_lmcache_chunk_size`` here is the + # number of *physical* slots per chunk for this group + # (= logical chunk_size // compress_ratio). + group_lmcache_chunk_size = gpu_context.get_physical_chunk_size( + group_idx + ) + lmc_ops.multi_layer_block_kv_transfer( + group_kv_pointers, + [tmp_buffer.data_ptr()], + chunk_block_ids_gpu, + gpu_context.device, + lmc_ops.TransferDirection.D2H, + gpu_context.get_shape_desc(group_idx), + group_lmcache_chunk_size, + gpu_context.gpu_kv_format_, + 0, + ) + # Store is not batched, so we always use chunk_idx=0 (single slot) + lmcache_memcpy_async_d2h( + gpu_context.get_tmp_gpu_buffer_flat(chunk_idx=0), memory_obj + ) + except Exception: + logger.exception("Cannot store keys due to exception") + finally: + event.record() + if reserved_dict: + submit_callback_to_stream( + gpu_context.cupy_stream, + "finish_write", + list(reserved_dict.keys()), + ) + # All reserved MemoryObjs share one layout_desc, so per-object + # size is identical — avoid summing N identical values. + total_bytes = ( + next(iter(reserved_dict.values())).get_size() * len(reserved_dict) + if reserved_dict + else 0 + ) + self._ctx.event_bus.publish_on_stream( + gpu_context.cupy_stream, + Event( + event_type=EventType.MP_STORE_END, + session_id=key.request_id, + metadata={ + "stored_count": len(reserved_dict), + "device": str(gpu_context.device), + "engine_id": instance_id, + "model_name": model_name, + "total_bytes": total_bytes, + }, + ), + ) + + ed = time.perf_counter() + if length := len(reserved_dict): + logger.info( + "Stored %d tokens in %.3f seconds", + length * self._ctx.chunk_size, + ed - st, + ) + return event.ipc_handle(), True + + @_lmcache_nvtx_annotate + def retrieve( + self, + key: IPCCacheEngineKey, + instance_id: int, + gpu_block_ids: list[int], + event_ipc_handle: bytes, + skip_first_n_tokens: int = 0, + ) -> tuple[bytes, bool]: + """Retrieve the CPU KV cache and put into GPU blocks. + + Args: + key: The IPC key for the KV cache blocks. + Must have worker_id != None (worker retrieve operation). + instance_id: The GPU instance ID (such as PID). + gpu_block_ids: The GPU block IDs to retrieve into. + event_ipc_handle: The IPC handle of the event to wait on. + skip_first_n_tokens: Number of tokens to skip writing at + the start of the retrieve range. This avoids overwriting + APC-shared GPU blocks that may be read concurrently by other + requests. + + Returns: + A tuple where the first element is the IPC handle of the event + that signals the completion of the retrieve operation, and the + second element indicates whether the key was successfully retrieved. + + Raises: + ValueError: If no GPU context is registered for the given instance ID. + """ + st = time.perf_counter() + obj_keys = self._ctx.resolve_obj_keys(key) + + entry = self._gpu_contexts.get(instance_id) + if entry is None: + raise ValueError(f"No GPU context registered for instance ID {instance_id}") + gpu_context = entry.gpu_context + model_name = entry.model_name + + # CPU-synchronous sentinel: a GPU retrieve is about to be enqueued. + # Must be published via publish() (not publish_on_stream) so the + # drain thread sees it before MP_REQUEST_END can race MP_RETRIEVE_END. + self._ctx.event_bus.publish( + Event( + event_type=EventType.MP_RETRIEVE_SUBMITTED, + session_id=key.request_id, + metadata={"device": str(gpu_context.device)}, + ) + ) + + self._ctx.event_bus.publish_on_stream( + gpu_context.cupy_stream, + Event( + event_type=EventType.MP_RETRIEVE_START, + session_id=key.request_id, + metadata={ + "device": str(gpu_context.device), + "engine_id": instance_id, + "model_name": model_name, + }, + ), + ) + + # ``skip_*_in_chunk`` is expressed in engine-block units + # (logical tokens), which is what the kernel's + # ``skip_blocks_in_chunk`` argument expects regardless + # of per-group compression. + ie_logical_block_size = ( + gpu_context.kv_layer_groups_manager.inference_engine_logical_block_size + ) + blocks_per_chunk = self._ctx.chunk_size // ie_logical_block_size + + def _retrieve_loop(keys: list[ObjectKey], memory_objs: list[MemoryObj]) -> None: + _BATCH_SIZE = gpu_context.max_batch_size + num_groups = gpu_context.kv_layer_groups_manager.num_groups + for batch_idx, memory_obj_batch in enumerate( + batched_iteration(memory_objs, batch_size=_BATCH_SIZE) + ): + batch_len = len(memory_obj_batch) + chunk_start = batch_idx * self._ctx.chunk_size * _BATCH_SIZE + chunk_end = chunk_start + self._ctx.chunk_size * batch_len + + effective_start = max(chunk_start, skip_first_n_tokens) + if effective_start >= chunk_end: + # Entire batch is within APC range, skip it + continue + + skip_tokens_in_chunk = max( + 0, + min( + effective_start - chunk_start, + self._ctx.chunk_size * batch_len - 1, + ), + ) + if skip_tokens_in_chunk % ie_logical_block_size != 0: + logger.error( + "skip_first_n_tokens (%d) is not aligned to " + "inference_engine_logical_block_size (%d), " + "rounding down from %d tokens to %d blocks", + skip_first_n_tokens, + ie_logical_block_size, + skip_tokens_in_chunk, + skip_tokens_in_chunk // ie_logical_block_size, + ) + skip_blocks_in_chunk = skip_tokens_in_chunk // ie_logical_block_size + + start_chunk_id = batch_idx * _BATCH_SIZE + end_chunk_id = start_chunk_id + batch_len + chunk_block_ids_gpu = all_block_ids_gpu[ + start_chunk_id * blocks_per_chunk : end_chunk_id * blocks_per_chunk + ] + + # Copy from CPU to GPU tmp buffers, then scatter to paged KV — per group + # H2D copy: each memory_obj maps to its own batch slot + for chunk_idx, memory_obj in enumerate(memory_obj_batch): + lmcache_memcpy_async_h2d( + memory_obj, + gpu_context.get_tmp_gpu_buffer_flat(chunk_idx=chunk_idx), + ) + for group_idx in range(num_groups): + tmp_buffers = gpu_context.get_tmp_chunk_gpu_buffer_batched( + batch_len, group_idx + ) + group_kv_pointers = gpu_context.get_group_kv_pointers(group_idx) + group_lmcache_chunk_size = gpu_context.get_physical_chunk_size( + group_idx + ) + + lmc_ops.multi_layer_block_kv_transfer( + group_kv_pointers, + [tb.data_ptr() for tb in tmp_buffers], + chunk_block_ids_gpu, + gpu_context.device, + lmc_ops.TransferDirection.H2D, + gpu_context.get_shape_desc(group_idx), + group_lmcache_chunk_size, + gpu_context.gpu_kv_format_, + skip_blocks_in_chunk, + ) + + with ( + torch_dev.device(gpu_context.device), + torch_dev.stream(gpu_context.stream), + ): + # Stage all block_ids to GPU once before the loop + all_block_ids_gpu = gpu_context.stage_block_ids(gpu_block_ids) + + check_interprocess_event_support() + event = torch_dev.Event(interprocess=True) + + prefetched_keys: list[ObjectKey] = [] + retrieve_succeeded = False + total_bytes = 0 + try: + with self._ctx.storage_manager.read_prefetched_results( + obj_keys + ) as memory_objs: + if not memory_objs or len(memory_objs) != len(obj_keys): + logger.error("Some keys not found during retrieve!") + return event.ipc_handle(), False + + prefetched_keys = obj_keys[: len(memory_objs)] + total_bytes = sum(mo.get_size() for mo in memory_objs) + _retrieve_loop(obj_keys, memory_objs) + # Only set True when with-block exits normally + retrieve_succeeded = True + except Exception: + logger.exception("Cannot retrieve keys due to exception") + return event.ipc_handle(), False + finally: + event.record() + if retrieve_succeeded: + submit_callback_to_stream( + gpu_context.cupy_stream, + "finish_read_prefetched", + prefetched_keys, + ) + self._ctx.event_bus.publish_on_stream( + gpu_context.cupy_stream, + Event( + event_type=EventType.MP_RETRIEVE_END, + session_id=key.request_id, + metadata={ + "retrieved_count": len(prefetched_keys), + "device": str(gpu_context.device), + "engine_id": instance_id, + "model_name": model_name, + "cache_salt": key.cache_salt, + "total_bytes": total_bytes, + }, + ), + ) + tokens_retrieved = len(obj_keys) * self._ctx.chunk_size + ed = time.perf_counter() + logger.info( + "Retrieved %d tokens in %.3f seconds", + tokens_retrieved, + ed - st, + ) + + return event.ipc_handle(), True diff --git a/lmcache/v1/multiprocess/modules/lookup.py b/lmcache/v1/multiprocess/modules/lookup.py new file mode 100644 index 00000000000..d7df4e296f6 --- /dev/null +++ b/lmcache/v1/multiprocess/modules/lookup.py @@ -0,0 +1,467 @@ +# SPDX-License-Identifier: Apache-2.0 +"""LookupModule: lookup, prefetch polling, and session lifecycle.""" + +# Standard +from dataclasses import dataclass +from functools import partial +import threading +import time + +# First Party +from lmcache.logging import init_logger +from lmcache.v1.distributed.api import ( + PrefetchHandle, + ipc_key_to_object_keys, +) +from lmcache.v1.mp_observability.event import Event, EventType +from lmcache.v1.mp_observability.otel_init import register_gauge +from lmcache.v1.multiprocess.custom_types import IPCCacheEngineKey +from lmcache.v1.multiprocess.engine_context import MPCacheEngineContext +from lmcache.v1.multiprocess.engine_module import ( + HandlerSpec, + ThreadPoolType, +) +from lmcache.v1.multiprocess.protocol import RequestType +from lmcache.v1.multiprocess.token_hasher import TokenHasher + +logger = init_logger(__name__) + + +def compute_extra_count( + tp_size: int, + world_size: int, +) -> int: + """Compute extra count for MLA multi-reader locking. + + Non-MLA: each TP worker owns a distinct KV shard, + so each ObjectKey is retrieved by exactly 1 + worker -> extra_count = 0. + MLA: TP does not split KV caches, all TP workers + share the same object. vLLM passes world_size + already divided by tp_size (e.g. world_size=1 + for TP=4 PP=1), so ipc_keys_to_object_keys + only produces 1 ObjectKey per chunk. All TP + workers retrieve that same ObjectKey, hence + extra_count = tp_size - 1. + + Detection: tp > world_size means MLA (world_size + was divided by tp on the vLLM side). + + Fallback: old vLLM (<= 0.8.5) does not send + tp_size (defaults to 1); we fall back to + world_size which gives extra_count = 0 + (safe but may under-lock for MLA). + + TODO: world_size currently carries an overloaded + meaning (total ranks for non-MLA vs total/tp for + MLA). Consider a dedicated field in the future. + + Args: + tp_size: Tensor-parallel size from the client. + world_size: World size from the cache key. + + Returns: + Number of extra count (0 for non-MLA). + """ + tp = tp_size if tp_size > 1 else world_size + return tp - 1 if tp > world_size else 0 + + +@dataclass +class _PrefetchJob: + handle: PrefetchHandle + world_size: int + request_id: str + # Number of tokens submitted for lookup (denominator for the L1+L2 + # token-level hit-rate metric). Equals ``len(chunk_hashes) * chunk_size`` + # on the happy path; 0 for early-exit paths (no GPU context matches + # or chunk_hashes is empty). Consumed at ``MP_LOOKUP_PREFETCH_END`` + # emission time in ``query_prefetch_status``. + requested_tokens: int + # Captured at lookup time so the ``MP_LOOKUP_PREFETCH_END`` event can + # carry them as labels. ``model_name`` lets dashboards slice hit rate + # per model in multi-model deployments; ``cache_salt`` slices per + # tenant / isolation domain (an empty string means no salt set). + model_name: str = "" + cache_salt: str = "" + + +class LookupModule: + """Handles lookup, prefetch polling, lock release, and session lifecycle. + + Owns the prefetch-job bookkeeping (``_prefetch_jobs``) and exposes + handlers for the LOOKUP, QUERY_PREFETCH_STATUS, + QUERY_PREFETCH_LOOKUP_HITS, FREE_LOOKUP_LOCKS, and END_SESSION + request types. + + Args: + ctx: Shared engine context providing storage manager, token hasher, + session manager, event bus, layout descriptor registry, and + chunk size. + """ + + def __init__(self, ctx: MPCacheEngineContext) -> None: + self._ctx = ctx + self._prefetch_jobs: dict[str, _PrefetchJob] = {} + self._prefetch_job_lock = threading.Lock() + self._setup_metrics() + + @property + def context(self) -> MPCacheEngineContext: + """Return the shared engine context. Exposed for testing only.""" + return self._ctx + + def get_handlers(self) -> list[HandlerSpec]: + """Return handler specs for all request types this module serves. + + Returns: + List of handler specs for lookup-related request types. + """ + return [ + HandlerSpec(RequestType.LOOKUP, self.lookup, ThreadPoolType.NORMAL), + HandlerSpec( + RequestType.QUERY_PREFETCH_STATUS, + self.query_prefetch_status, + ThreadPoolType.NORMAL, + ), + HandlerSpec( + RequestType.QUERY_PREFETCH_LOOKUP_HITS, + self.query_prefetch_lookup_hits, + ThreadPoolType.NORMAL, + ), + HandlerSpec( + RequestType.FREE_LOOKUP_LOCKS, + self.free_lookup_locks, + ThreadPoolType.NORMAL, + ), + HandlerSpec( + RequestType.END_SESSION, + self.end_session, + ThreadPoolType.NORMAL, + ), + ] + + def report_status(self) -> dict[str, int]: + """Return module-specific status information. + + Returns: + Dictionary with the count of active prefetch jobs. + """ + return { + "active_prefetch_jobs": self._active_prefetch_count(), + } + + def close(self) -> None: + """Release resources owned by this module (no-op).""" + pass + + # ----------------------------------------------------------------- + # Handlers + # ----------------------------------------------------------------- + + def lookup( + self, + key: IPCCacheEngineKey, + tp_size: int, + ) -> None: + """Submit a prefix lookup. + + Hashes the key, submits a prefetch task to the storage manager, + and registers the job under ``key.request_id`` for later polling + via query_prefetch_status. + + Args: + key: Cache key with request_id embedded. + tp_size: Tensor-parallel size for MLA multi-reader locking. + """ + model_name, world_size = key.model_name, key.world_size + self._ctx.event_bus.publish( + Event( + event_type=EventType.MP_REQUEST_START, + session_id=key.request_id, + ) + ) + self._ctx.event_bus.publish( + Event( + event_type=EventType.MP_LOOKUP_PREFETCH_START, + session_id=key.request_id, + ) + ) + + layout_desc = self._ctx.layout_desc_registry.find(model_name, world_size) + if layout_desc is None: + logger.error( + "No GPU context found for model %s with world size %d during lookup!", + model_name, + world_size, + ) + self._register_prefetch_job( + _PrefetchJob( + handle=PrefetchHandle( + prefetch_request_id=-1, + external_request_id=key.request_id, + l1_prefix_hit_count=0, + total_requested_keys=0, + submit_time=time.monotonic(), + ), + world_size=1, + request_id=key.request_id, + requested_tokens=0, + model_name=model_name, + cache_salt=key.cache_salt, + ) + ) + return + + extra_count = compute_extra_count(tp_size, world_size) + + chunk_hashes = self._ctx.token_hasher.compute_chunk_hashes(list(key.token_ids)) + if not chunk_hashes: + self._register_prefetch_job( + _PrefetchJob( + handle=PrefetchHandle( + prefetch_request_id=-1, + external_request_id=key.request_id, + l1_prefix_hit_count=0, + total_requested_keys=0, + submit_time=time.monotonic(), + ), + world_size=1, + request_id=key.request_id, + requested_tokens=0, + model_name=model_name, + cache_salt=key.cache_salt, + ) + ) + return + + # Total chunk-aligned tokens submitted for lookup; surfaces as the + # denominator of the L1+L2 token-level hit-rate via the + # ``requested_tokens`` field on ``MP_LOOKUP_PREFETCH_END``. Sub-chunk + # trailing tokens are intentionally excluded — they cannot hit at + # chunk granularity. + requested_tokens = len(chunk_hashes) * self._ctx.chunk_size + + # Guard with has_subscribers() to avoid allocating the metadata dict + # (including dtype/shape list comprehensions) when no subscriber is + # listening (e.g. lookup hash logger is disabled). + if self._ctx.event_bus.has_subscribers(EventType.MP_LOOKUP): + self._ctx.event_bus.publish( + Event( + event_type=EventType.MP_LOOKUP, + session_id=key.request_id, + metadata={ + "request_id": key.request_id, + "chunk_hashes": chunk_hashes, + "model_name": model_name, + "chunk_size": self._ctx.chunk_size, + "seq_len": len(key.token_ids), + "dtypes": [str(d) for d in layout_desc.dtypes], + "shapes": [list(s) for s in layout_desc.shapes], + }, + ) + ) + + session = self._ctx.session_manager.get_or_create(key.request_id) + session.set_tokens(list(key.token_ids)) + session.lookup_ipc_key = key + + obj_keys = ipc_key_to_object_keys(key, chunk_hashes) + + handle = self._ctx.storage_manager.submit_prefetch_task( + obj_keys, + layout_desc, + extra_count=extra_count, + external_request_id=key.request_id, + ) + self._register_prefetch_job( + _PrefetchJob( + handle=handle, + world_size=key.world_size, + request_id=key.request_id, + requested_tokens=requested_tokens, + model_name=model_name, + cache_salt=key.cache_salt, + ) + ) + + def query_prefetch_lookup_hits( + self, + request_id: str, + ) -> int | None: + """Query the number of hits for a prefetch request before it's finished. + + Args: + request_id: The external request ID passed in the lookup key. + + Returns: + The number of hits for the prefetched keys if the lookup phase is + done. None if the lookup phase is still in progress. 0 if the + request_id is unknown (already completed and consumed, or invalid). + """ + with self._prefetch_job_lock: + job = self._prefetch_jobs.get(request_id) + + if job is None: + logger.warning( + "Prefetch job for request %s not found (already completed or invalid)", + request_id, + ) + return 0 + + found_count = self._ctx.storage_manager.query_prefetch_lookup_hits(job.handle) + if found_count is None: + return None + + found_count = found_count // job.world_size + return found_count + + def query_prefetch_status( + self, + request_id: str, + ) -> int | None: + """Poll the status of a prefetch job by request_id. + + Returns the chunk count when the prefetch is complete, or None + if it is still in progress. The job entry is automatically + removed once a non-None result is returned (exactly-once + semantics). + + Args: + request_id: The external request ID passed in the lookup key. + + Returns: + Chunk count (int) when done, None if still in progress, + 0 if the request_id is unknown (already completed and consumed, + or invalid). + """ + with self._prefetch_job_lock: + job = self._prefetch_jobs.get(request_id) + if job is None: + logger.warning( + "Prefetch job for request %s not found (already completed or invalid)", + request_id, + ) + return 0 + + found_count = self._ctx.storage_manager.query_prefetch_status(job.handle) + if found_count is None: + return None + + # NOTE(Kuntai): this assumes two things: + # 1. the world size is the same between keys + # 2. the lookup sort the keys in prefix order and breaks at the + # first failure + found_count = found_count // job.world_size + + self._ctx.event_bus.publish( + Event( + event_type=EventType.MP_LOOKUP_PREFETCH_END, + session_id=job.request_id, + metadata={ + "found_count": found_count, + "requested_tokens": job.requested_tokens, + "hit_tokens": found_count * self._ctx.chunk_size, + "model_name": job.model_name, + "cache_salt": job.cache_salt, + }, + ) + ) + + with self._prefetch_job_lock: + self._prefetch_jobs.pop(request_id, None) + + return found_count + + def free_lookup_locks( + self, + key: IPCCacheEngineKey, + tp_size: int, + ) -> None: + """Release read locks acquired during lookup. + + Hashes are computed only for chunks in ``[start, end)`` to avoid + unnecessary work on tokens outside that range. + ``start`` and ``end`` must be aligned to ``chunk_size``; it is the + caller's responsibility to align the boundaries as desired. + + Computes the extra reader count from ``tp_size`` and + ``world_size`` the same way :meth:`lookup` does, so + the correct number of locks is released. + + Args: + key: Cache key whose read locks should be released. + tp_size: Tensor-parallel size for MLA + multi-reader locking. + """ + chunk_hashes = self._ctx.token_hasher.compute_chunk_hashes( + list(key.token_ids), start=key.start, end=key.end + ) + if not chunk_hashes: + return + obj_keys = ipc_key_to_object_keys(key, chunk_hashes) + + extra_count = compute_extra_count(tp_size, key.world_size) + + self._ctx.storage_manager.finish_read_prefetched( + obj_keys, extra_count=extra_count + ) + + def end_session(self, request_id: str) -> None: + """Remove the session for a finished request. + + Args: + request_id: The request ID whose session should be removed. + """ + self._ctx.event_bus.publish( + Event( + event_type=EventType.MP_VLLM_END_SESSION, + metadata={"request_id": request_id}, + ) + ) + session = self._ctx.session_manager.remove(request_id) + self._ctx.event_bus.publish( + Event( + event_type=EventType.MP_REQUEST_END, + session_id=request_id, + ) + ) + if session is None: + logger.warning("Session %s not found, skipping touch", request_id) + return + if session.lookup_ipc_key is None: + logger.warning( + "Session %s has no lookup ipc key, skipping touch", + request_id, + ) + return + + chunk_hashes = [TokenHasher.hash_to_bytes(h) for h in session.get_hashes(0)] + obj_keys = ipc_key_to_object_keys(session.lookup_ipc_key, chunk_hashes) + # unified touch of all keys, which include retrieved and stored keys + # TODO(chunxiaozheng): when l2 is enabled, the prefetched keys from l2 are temp + # and will be deleted after finish_read_prefetched, when we touch all keys, + # these keys has been deleted and will not be touched. + self._ctx.storage_manager.touch_l1_keys(obj_keys) + + # ----------------------------------------------------------------- + # Internal helpers + # ----------------------------------------------------------------- + + def _register_prefetch_job(self, job: _PrefetchJob) -> None: + with self._prefetch_job_lock: + self._prefetch_jobs[job.request_id] = job + + def _active_prefetch_count(self) -> int: + """Return the number of active prefetch jobs (thread-safe).""" + with self._prefetch_job_lock: + return len(self._prefetch_jobs) + + def _setup_metrics(self) -> None: + """Register OTel observable gauges for lookup module metrics.""" + _gauge = partial(register_gauge, "lmcache.mp_engine") + _gauge( + "lmcache_mp.active_prefetch_jobs", + "Number of active prefetch jobs", + self._active_prefetch_count, + ) diff --git a/lmcache/v1/multiprocess/modules/management.py b/lmcache/v1/multiprocess/modules/management.py new file mode 100644 index 00000000000..4cdfcb2d164 --- /dev/null +++ b/lmcache/v1/multiprocess/modules/management.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Management and utility operations for the MPCacheEngine.""" + +# Standard +import threading + +# First Party +from lmcache.logging import init_logger +from lmcache.v1.mp_observability.event import Event, EventType +from lmcache.v1.multiprocess.custom_types import BlockAllocationRecord +from lmcache.v1.multiprocess.engine_context import MPCacheEngineContext +from lmcache.v1.multiprocess.engine_module import ( + HandlerSpec, + ThreadPoolType, +) +from lmcache.v1.multiprocess.protocols.base import RequestType + +logger = init_logger(__name__) + + +class ManagementModule: + """Handles management and utility operations for the cache engine. + + Owns the lock used during cache clearing and provides handlers for + ping, chunk-size queries, clear, debug, and block-allocation reporting. + + Args: + ctx: The shared engine context. + """ + + def __init__(self, ctx: MPCacheEngineContext) -> None: + self._ctx = ctx + self._clear_lock = threading.Lock() + + @property + def context(self) -> MPCacheEngineContext: + """Return the shared engine context. Exposed for testing only.""" + return self._ctx + + def get_handlers(self) -> list[HandlerSpec]: + """Return handler specs for all request types this module serves. + + Returns: + A list of HandlerSpec entries mapping request types to + their handler callables and thread pool assignments. + """ + return [ + HandlerSpec(RequestType.CLEAR, self.clear, ThreadPoolType.NORMAL), + HandlerSpec( + RequestType.GET_CHUNK_SIZE, + self.get_chunk_size, + ThreadPoolType.SYNC, + ), + HandlerSpec(RequestType.PING, self.ping, ThreadPoolType.NORMAL), + HandlerSpec(RequestType.NOOP, self.debug, ThreadPoolType.SYNC), + HandlerSpec( + RequestType.REPORT_BLOCK_ALLOCATION, + self.report_block_allocations, + ThreadPoolType.NORMAL, + ), + ] + + def report_status(self) -> dict: + """Return module-specific status information. + + Returns: + An empty dict; management has no module-level metrics. + """ + return {} + + def close(self) -> None: + """Release resources owned by this module.""" + pass + + def ping(self) -> bool: + """Respond to a ping request. + + Returns: + Always True. + """ + return True + + def get_chunk_size(self) -> int: + """Return the chunk size used for KV cache operations. + + Returns: + The chunk size. + """ + return self._ctx.chunk_size + + def clear(self) -> None: + """Clear all stored KV cache data from the storage manager.""" + with self._clear_lock: + self._ctx.storage_manager.memcheck() + self._ctx.storage_manager.clear(force=True) + self._ctx.storage_manager.memcheck() + + def debug(self) -> str: + """Return a simple health-check string. + + Returns: + The literal string ``"OK"``. + """ + return "OK" + + def report_block_allocations( + self, + instance_id: int, + model_name: str, + records: list[BlockAllocationRecord], + ) -> None: + """Publish vLLM block allocation records to the EventBus. + + Args: + instance_id: The scheduler instance ID. + model_name: The model name from the adapter. + records: List of BlockAllocationRecord with per-request + block and token allocation deltas. + """ + self._ctx.event_bus.publish( + Event( + event_type=EventType.MP_VLLM_BLOCK_ALLOCATION, + metadata={ + "instance_id": instance_id, + "model_name": model_name, + "records": records, + }, + ) + ) diff --git a/lmcache/v1/multiprocess/modules/non_gpu_transfer.py b/lmcache/v1/multiprocess/modules/non_gpu_transfer.py new file mode 100644 index 00000000000..f92291b230f --- /dev/null +++ b/lmcache/v1/multiprocess/modules/non_gpu_transfer.py @@ -0,0 +1,347 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Non-GPU (pickle-based) KV cache transfer operations for the MPCacheEngine.""" + +# Standard +from dataclasses import dataclass +import pickle + +# Third Party +import torch + +# First Party +from lmcache.logging import init_logger +from lmcache.utils import _lmcache_nvtx_annotate +from lmcache.v1.distributed.api import ( + MemoryLayoutDesc, + ObjectKey, +) +from lmcache.v1.multiprocess.custom_types import ( + IPCCacheEngineKey, + RegisterNonGpuContextPayload, +) +from lmcache.v1.multiprocess.engine_context import MPCacheEngineContext +from lmcache.v1.multiprocess.engine_module import ( + HandlerSpec, + ThreadPoolType, +) +from lmcache.v1.multiprocess.non_gpu_context import NonGpuContextMetadata +from lmcache.v1.multiprocess.protocols.base import RequestType +from lmcache.v1.multiprocess.protocols.engine import ( + PrepareRetrieveResponse, + PrepareStoreResponse, +) + +logger = init_logger(__name__) + + +@dataclass +class NonGPUContextEntry: + """Registered non-GPU context metadata for a single worker instance. + + Attributes: + metadata: Layout metadata describing the non-CUDA chunk format. + model_name: The name of the model associated with this context. + world_size: The world size associated with this context. + """ + + metadata: NonGpuContextMetadata + model_name: str + world_size: int + + +class NonGPUTransferModule: + """Handles non-GPU (pickle-based) KV cache transfer operations. + + Owns non-GPU context registrations and provides handlers for + register, unregister, prepare/commit store, and prepare/commit retrieve + of CPU-serialized KV caches. + + Args: + ctx: The shared engine context. + """ + + def __init__(self, ctx: MPCacheEngineContext) -> None: + self._ctx = ctx + self._non_gpu_contexts: dict[int, NonGPUContextEntry] = {} + + @property + def context(self) -> MPCacheEngineContext: + """Return the shared engine context. Exposed for testing only.""" + return self._ctx + + def get_handlers(self) -> list[HandlerSpec]: + """Return handler specs for all request types this module serves. + + Returns: + A list of HandlerSpec entries mapping request types to + their handler callables and thread pool assignments. + """ + return [ + HandlerSpec( + RequestType.REGISTER_KV_CACHE_NON_GPU_CONTEXT, + self.register_kv_cache_non_gpu_context, + ThreadPoolType.SYNC, + ), + HandlerSpec( + RequestType.UNREGISTER_KV_CACHE, + self.unregister_kv_cache, + ThreadPoolType.SYNC, + ), + HandlerSpec( + RequestType.PREPARE_STORE, + self.prepare_store, + ThreadPoolType.AFFINITY, + ), + HandlerSpec( + RequestType.COMMIT_STORE, + self.commit_store, + ThreadPoolType.AFFINITY, + ), + HandlerSpec( + RequestType.PREPARE_RETRIEVE, + self.prepare_retrieve, + ThreadPoolType.AFFINITY, + ), + HandlerSpec( + RequestType.COMMIT_RETRIEVE, + self.commit_retrieve, + ThreadPoolType.AFFINITY, + ), + ] + + def report_status(self) -> dict: + """Return non-GPU transfer module status information. + + Returns: + A dict containing registered non-CUDA instance IDs and + per-instance context metadata. + """ + registered_non_cuda_ids: list[int] = [] + non_cuda_context_meta: dict[str, dict] = {} + + for instance_id, entry in self._non_gpu_contexts.items(): + registered_non_cuda_ids.append(instance_id) + non_cuda_context_meta[str(instance_id)] = { + "model_name": entry.model_name, + "world_size": entry.world_size, + "block_size": entry.metadata.block_size, + "use_mla": entry.metadata.use_mla, + } + + return { + "registered_non_cuda_instance_ids": registered_non_cuda_ids, + "non_cuda_context_meta": non_cuda_context_meta, + } + + def close(self) -> None: + """Release resources owned by this module.""" + self._non_gpu_contexts.clear() + + def register_kv_cache_non_gpu_context( + self, + payload: RegisterNonGpuContextPayload, + ) -> None: + """Register non-CUDA KV layout metadata for non-GPU context mode. + + Args: + payload: Struct containing all registration fields + (instance_id, model_name, world_size, block_size, + num_layers, hidden_dim_size, dtype_str, use_mla). + + Raises: + ValueError: If ``payload.dtype_str`` is not a valid torch dtype name. + """ + if payload.instance_id in self._non_gpu_contexts: + logger.warning( + "Instance %s's KV cache is already registered, " + "skipping the new registration", + payload.instance_id, + ) + return + + dtype = getattr(torch, payload.dtype_str, None) + if dtype is None or not isinstance(dtype, torch.dtype): + raise ValueError( + f"Invalid dtype_str '{payload.dtype_str}': must be a valid torch dtype " + "attribute name (e.g. 'float16' for torch.float16, " + "'bfloat16' for torch.bfloat16, 'float32' for torch.float32)." + ) + + shape = ( + torch.Size( + [payload.num_layers, self._ctx.chunk_size, payload.hidden_dim_size] + ) + if payload.use_mla + else torch.Size( + [2, payload.num_layers, self._ctx.chunk_size, payload.hidden_dim_size] + ) + ) + layout_desc = MemoryLayoutDesc(shapes=[shape], dtypes=[dtype]) + metadata = NonGpuContextMetadata( + layout_desc=layout_desc, + block_size=payload.block_size, + use_mla=payload.use_mla, + ) + self._non_gpu_contexts[payload.instance_id] = NonGPUContextEntry( + metadata=metadata, + model_name=payload.model_name, + world_size=payload.world_size, + ) + + self._ctx.layout_desc_registry.register( + payload.model_name, payload.world_size, layout_desc + ) + + def unregister_kv_cache(self, instance_id: int) -> None: + """Unregister a non-GPU KV cache context for the given instance ID. + + Args: + instance_id: The worker instance identifier. + """ + entry = self._non_gpu_contexts.pop(instance_id, None) + if entry is None: + logger.warning( + "No registered non-GPU context found for instance ID %d", + instance_id, + ) + return + + self._ctx.layout_desc_registry.unregister(entry.model_name, entry.world_size) + logger.info("Unregistered non-CUDA context for instance ID %d", instance_id) + + @_lmcache_nvtx_annotate + def prepare_store( + self, + key: IPCCacheEngineKey, + instance_id: int, + ) -> PrepareStoreResponse: + """Prepare a store operation. For pickle mode, returns empty slots. + + Args: + key: Cache key for the token range to store. + instance_id: Worker instance identifier. + + Returns: + PrepareStoreResponse with empty slots for pickle mode. + """ + return PrepareStoreResponse(context={}) + + @_lmcache_nvtx_annotate + def commit_store( + self, + key: IPCCacheEngineKey, + instance_id: int, + cpu_data: bytes, + ) -> bool: + """Commit serialized CPU chunks to storage. + + Args: + key: Cache key for the token range to store. + instance_id: Worker instance identifier. + cpu_data: Pickled list of CPU tensors produced by the worker. + + Returns: + ``True`` when all reserved objects are written, otherwise ``False``. + + Raises: + ValueError: If no non-GPU context is registered for the given + instance ID. + """ + obj_keys = self._ctx.resolve_obj_keys(key) + + entry = self._non_gpu_contexts.get(instance_id) + if entry is None: + raise ValueError( + f"non-CUDA context not registered for instance ID {instance_id}" + ) + ctx = entry.metadata + chunks: list[torch.Tensor] = pickle.loads(cpu_data) + reserved_dict = self._ctx.storage_manager.reserve_write( + obj_keys, ctx.layout_desc, "new" + ) + written_keys: list[ObjectKey] = [] + try: + for idx, obj_key in enumerate(obj_keys): + if obj_key not in reserved_dict: + continue + if idx >= len(chunks): + continue + memory_obj = reserved_dict[obj_key] + if memory_obj.tensor is None: + continue + chunk_cpu = chunks[idx] + if chunk_cpu.shape != memory_obj.tensor.shape: + continue + memory_obj.tensor.copy_(chunk_cpu) + written_keys.append(obj_key) + finally: + if written_keys: + self._ctx.storage_manager.finish_write(written_keys) + + return len(written_keys) == len(reserved_dict) + + @_lmcache_nvtx_annotate + def prepare_retrieve( + self, + key: IPCCacheEngineKey, + instance_id: int, + ) -> PrepareRetrieveResponse: + """Retrieve prefetched chunks and return serialized CPU tensors. + + Args: + key: Cache key for the token range to retrieve. + instance_id: Worker instance identifier. + + Returns: + PrepareRetrieveResponse with serialized data on hit. + + Raises: + ValueError: If no non-GPU context is registered for the given + instance ID. + """ + obj_keys = self._ctx.resolve_obj_keys(key) + + entry = self._non_gpu_contexts.get(instance_id) + if entry is None: + raise ValueError( + f"non-CUDA context not registered for instance ID {instance_id}" + ) + + prefetched_keys: list[ObjectKey] = [] + try: + with self._ctx.storage_manager.read_prefetched_results( + obj_keys + ) as memory_objs: + if not memory_objs or len(memory_objs) != len(obj_keys): + return PrepareRetrieveResponse(success=False, data=b"", context={}) + prefetched_keys = obj_keys[: len(memory_objs)] + chunks = [] + for memory_obj in memory_objs: + if memory_obj.tensor is None: + return PrepareRetrieveResponse( + success=False, data=b"", context={} + ) + chunks.append(memory_obj.tensor.cpu().clone()) + return PrepareRetrieveResponse( + success=True, data=pickle.dumps(chunks), context={} + ) + finally: + if prefetched_keys: + self._ctx.storage_manager.finish_read_prefetched(prefetched_keys) + + @_lmcache_nvtx_annotate + def commit_retrieve( + self, + key: IPCCacheEngineKey, + instance_id: int, + ) -> bool: + """Finalize a retrieve operation. No-op for pickle mode. + + Args: + key: Cache key (unused for pickle). + instance_id: Worker instance identifier (unused for pickle). + + Returns: + Always ``True``. + """ + return True diff --git a/lmcache/v1/multiprocess/server.py b/lmcache/v1/multiprocess/server.py index b8cedd73256..01be515df74 100644 --- a/lmcache/v1/multiprocess/server.py +++ b/lmcache/v1/multiprocess/server.py @@ -1,1398 +1,116 @@ # SPDX-License-Identifier: Apache-2.0 +"""MPCacheEngine compositor and unified cache server entry point.""" + # Standard -from dataclasses import dataclass -from functools import partial -from itertools import islice -from typing import Generator import argparse -import pickle -import threading import time # Third Party -import torch import zmq # First Party from lmcache import torch_dev, torch_device_type from lmcache.logging import init_logger -from lmcache.utils import ( - EngineType, - _lmcache_nvtx_annotate, - check_interprocess_event_support, -) -from lmcache.v1.distributed.api import ( - MemoryLayoutDesc, - ObjectKey, - ipc_key_to_object_keys, -) from lmcache.v1.distributed.config import ( StorageManagerConfig, add_storage_manager_args, parse_args_to_config, ) -from lmcache.v1.distributed.storage_manager import PrefetchHandle, StorageManager -from lmcache.v1.gpu_connector.gpu_ops import ( - lmcache_memcpy_async_d2h, - lmcache_memcpy_async_h2d, -) -from lmcache.v1.gpu_connector.utils import LayoutHints -from lmcache.v1.memory_management import MemoryObj from lmcache.v1.mp_observability.config import ( ObservabilityConfig, add_observability_args, init_observability, parse_args_to_observability_config, ) -from lmcache.v1.mp_observability.event import Event, EventType -from lmcache.v1.mp_observability.event_bus import get_event_bus -from lmcache.v1.mp_observability.otel_init import register_gauge from lmcache.v1.mp_observability.trace import maybe_initialize_trace_recorder from lmcache.v1.multiprocess.config import ( MPServerConfig, add_mp_server_args, parse_args_to_mp_server_config, ) -from lmcache.v1.multiprocess.custom_types import ( - BlockAllocationRecord, - IPCCacheEngineKey, - KVCache, - RegisterNonGpuContextPayload, -) -from lmcache.v1.multiprocess.gpu_context import ( - GPUCacheContext, +from lmcache.v1.multiprocess.engine_context import MPCacheEngineContext +from lmcache.v1.multiprocess.engine_module import ( + EngineModule, + HandlerSpec, + ThreadPoolType, ) +from lmcache.v1.multiprocess.modules.gpu_transfer import GPUTransferModule +from lmcache.v1.multiprocess.modules.lookup import LookupModule +from lmcache.v1.multiprocess.modules.management import ManagementModule +from lmcache.v1.multiprocess.modules.non_gpu_transfer import NonGPUTransferModule from lmcache.v1.multiprocess.mq import MessageQueueServer -from lmcache.v1.multiprocess.native_completion import ( - DeviceHostFuncDispatcher, - submit_callback_to_stream, -) -from lmcache.v1.multiprocess.non_gpu_context import NonGpuContextMetadata from lmcache.v1.multiprocess.protocol import ( RequestType, get_handler_type, get_payload_classes, ) -from lmcache.v1.multiprocess.protocols.engine import ( - PrepareRetrieveResponse, - PrepareStoreResponse, -) -from lmcache.v1.multiprocess.session import SessionManager -from lmcache.v1.multiprocess.token_hasher import TokenHasher -import lmcache.c_ops as lmc_ops logger = init_logger(__name__) -# Helper functions -def compute_extra_count( - tp_size: int, - world_size: int, -) -> int: - """Compute extra count for MLA multi-reader locking. - - Non-MLA: each TP worker owns a distinct KV shard, - so each ObjectKey is retrieved by exactly 1 - worker -> extra_count = 0. - MLA: TP does not split KV caches, all TP workers - share the same object. vLLM passes world_size - already divided by tp_size (e.g. world_size=1 - for TP=4 PP=1), so ipc_keys_to_object_keys - only produces 1 ObjectKey per chunk. All TP - workers retrieve that same ObjectKey, hence - extra_count = tp_size - 1. - - Detection: tp > world_size means MLA (world_size - was divided by tp on the vLLM side). - - Fallback: old vLLM (<= 0.8.5) does not send - tp_size (defaults to 1); we fall back to - world_size which gives extra_count = 0 - (safe but may under-lock for MLA). - - TODO: world_size currently carries an overloaded - meaning (total ranks for non-MLA vs total/tp for - MLA). Consider a dedicated field in the future. - - Args: - tp_size: Tensor-parallel size from the client. - world_size: World size from the cache key. - - Returns: - Number of extra count (0 for non-MLA). - """ - tp = tp_size if tp_size > 1 else world_size - return tp - 1 if tp > world_size else 0 - - -def get_layout_desc(gpu_context: GPUCacheContext, num_tokens: int) -> MemoryLayoutDesc: - """Get the memory layout description for a given GPU context and number of tokens. - - Supports multiple KV layer groups with different shapes and dtypes. - - Args: - gpu_context: The GPU cache context containing the KV cache information. - num_tokens: The number of tokens to determine the layout for. - - Returns: - MemoryLayoutDesc: The memory layout description containing shapes and dtypes. - """ - num_groups = gpu_context.kv_layer_groups_manager.num_groups - shapes = [ - gpu_context.get_kv_buffer_shape(num_tokens, group_idx) - for group_idx in range(num_groups) - ] - dtypes = [ - gpu_context.kv_layer_groups_manager.kv_layer_groups[group_idx].dtype - for group_idx in range(num_groups) - ] - return MemoryLayoutDesc(shapes=shapes, dtypes=dtypes) - +class MPCacheEngine: + """Compositor that assembles pluggable engine modules. -def batched_iteration(lst: list, batch_size: int) -> Generator[tuple, None, None]: - """Utility function to iterate over a list in batches. + Holds the shared :class:`MPCacheEngineContext` and a list of + :class:`EngineModule` instances. Provides aggregated + ``report_status()`` and ``close()`` across all modules. Args: - lst: The list to iterate over. - batch_size: The size of each batch. - - Yields: - Batches of the list as tuples. + context: The shared engine context. + modules: List of engine modules to compose. """ - if batch_size < 1: - raise ValueError("batch size must be at least one") - it = iter(lst) - while batch := tuple(islice(it, batch_size)): - yield batch - - -@dataclass -class _PrefetchJob: - handle: PrefetchHandle - world_size: int - request_id: str - # Number of tokens submitted for lookup (denominator for the L1+L2 - # token-level hit-rate metric). Equals ``len(chunk_hashes) * chunk_size`` - # on the happy path; 0 for early-exit paths (no GPU context matches - # or chunk_hashes is empty). Consumed at ``MP_LOOKUP_PREFETCH_END`` - # emission time in ``query_prefetch_status``. - requested_tokens: int - # Captured at lookup time so the ``MP_LOOKUP_PREFETCH_END`` event can - # carry them as labels. ``model_name`` lets dashboards slice hit rate - # per model in multi-model deployments; ``cache_salt`` slices per - # tenant / isolation domain (an empty string means no salt set). - model_name: str = "" - cache_salt: str = "" - - -@dataclass -class RegisteredContext: - """Registered context metadata for a single worker instance. - - At least one of ``gpu_context`` or ``non_cuda_metadata`` is expected to be - populated for valid registrations. - """ - - model_name: str - world_size: int - gpu_context: GPUCacheContext | None = None - non_cuda_metadata: NonGpuContextMetadata | None = None - - @property - def is_gpu(self) -> bool: - """Return whether this registration uses a GPU transfer context.""" - return self.gpu_context is not None - - def get_layout_desc(self, chunk_size: int) -> MemoryLayoutDesc: - """Return the layout descriptor for this registration. - - Args: - chunk_size: Chunk size in tokens used for GPU layout derivation. - - Returns: - The resolved memory layout descriptor. - Raises: - ValueError: If no GPU context or non-CUDA metadata is configured. - """ - if self.gpu_context is not None: - return get_layout_desc(self.gpu_context, chunk_size) - if self.non_cuda_metadata is None: - raise ValueError( - "Invalid RegisteredContext: no GPU or non-CUDA metadata configured" - ) - return self.non_cuda_metadata.layout_desc - - -# Main class for the mp cache engine -class MPCacheEngine: def __init__( self, - storage_manager_config: StorageManagerConfig, - chunk_size: int = 256, - hash_algorithm: str = "blake3", - ): - # Worker instance ID -> registered context metadata - self.contexts: dict[int, RegisteredContext] = {} - - # chunk size - self.chunk_size = chunk_size - - # Lock for clear() to avoid concurrent storage manager mutations - self.lock = threading.Lock() - - # storage manager - self.storage_manager = StorageManager(storage_manager_config) - - # Token hasher and session manager for token-based operations - self.token_hasher = TokenHasher( - chunk_size=chunk_size, hash_algorithm=hash_algorithm - ) - self.session_manager = SessionManager(self.token_hasher) - - # EventBus for observability - self._event_bus = get_event_bus() - - # Route finish_write / finish_read_prefetched through a C++ host - # callback so the driver thread doesn't acquire the GIL. - self._device_host_func_dispatcher = DeviceHostFuncDispatcher() - self._device_host_func_dispatcher.register( - "finish_write", - self.storage_manager.finish_write, - payload_type=list[ObjectKey], - ) - self._device_host_func_dispatcher.register( - "finish_read_prefetched", - self.storage_manager.finish_read_prefetched, - payload_type=list[ObjectKey], - ) - self._device_host_func_dispatcher.start() - - # Prefetch job tracking for two-phase lookup, keyed by request_id. - # TODO: implement periodic cleanup of stale _prefetch_jobs entries - # for crash resilience (e.g., client calls lookup but never queries) - self._prefetch_jobs: dict[str, _PrefetchJob] = {} - self._prefetch_job_lock = threading.Lock() - - self._setup_metrics() - - @property - def gpu_contexts(self) -> dict[int, GPUCacheContext]: - """Return GPU-only context mapping for backward compatibility.""" - return { - instance_id: ctx.gpu_context - for instance_id, ctx in self.contexts.items() - if ctx.gpu_context is not None - } - - def register_kv_cache( - self, - instance_id: int, - kv_caches: KVCache, - model_name: str, - world_size: int, - engine_type: EngineType, - layout_hints: LayoutHints, + context: MPCacheEngineContext, + modules: list[EngineModule], ) -> None: - """ - Registers the KV cache tensors for a given GPU instance ID. - - Args: - instance_id (int): The GPU instance ID (such as PID). - kv_caches (KVCache): The KV cache tensor wrappers from the - serving engine. - model_name (str): The name of the model associated with this KV cache. - world_size (int): The world size associated with this KV cache. - engine_type: Which serving engine produced the caches. - Forwarded to :class:`GPUCacheContext` for format detection. - layout_hints: See :class:`LayoutHints`. Forwarded to - :class:`GPUCacheContext` for GPU KV format detection. - """ - if instance_id in self.contexts: - logger.warning( - "Instance %s's KV cache is already registered, " - "skipping the new registration", - instance_id, - ) - return - - gpu_context = GPUCacheContext( - kv_caches, - self.chunk_size, - layout_hints=layout_hints or None, - engine_type=engine_type, - ) - self.contexts[instance_id] = RegisteredContext( - model_name=model_name, - world_size=world_size, - gpu_context=gpu_context, - ) - logger.info( - "Registered KV cache for GPU ID %d with %d layers", - instance_id, - gpu_context.num_layers, - ) - - def unregister_kv_cache(self, instance_id: int) -> None: - """ - Unregisters the KV cache tensors for a given GPU instance ID. + self._context = context + self._modules = modules - Args: - instance_id (int): The GPU instance ID (such as PID). - """ - context = self.contexts.pop(instance_id, None) - if context is None: - logger.warning( - "No registered context found for instance ID %d", instance_id - ) - return - - if context.is_gpu: - logger.info("Unregistered KV cache for GPU ID %d", instance_id) - torch_dev.empty_cache() - else: - logger.info("Unregistered non-CUDA context for instance ID %d", instance_id) - - def register_kv_cache_non_gpu_context( - self, - payload: RegisterNonGpuContextPayload, - ) -> None: - """Register non-CUDA KV layout metadata for non-GPU context mode. - - Args: - payload: Struct containing all registration fields - (instance_id, model_name, world_size, block_size, - num_layers, hidden_dim_size, dtype_str, use_mla). - - Raises: - ValueError: If ``payload.dtype_str`` is not a valid torch dtype name. - """ - if payload.instance_id in self.contexts: - logger.warning( - "Instance %s's KV cache is already registered, " - "skipping the new registration", - payload.instance_id, - ) - return - - dtype = getattr(torch, payload.dtype_str, None) - if dtype is None or not isinstance(dtype, torch.dtype): - raise ValueError( - f"Invalid dtype_str '{payload.dtype_str}': must be a valid torch dtype " - "attribute name (e.g. 'float16' for torch.float16, " - "'bfloat16' for torch.bfloat16, 'float32' for torch.float32)." - ) - - shape = ( - torch.Size([payload.num_layers, self.chunk_size, payload.hidden_dim_size]) - if payload.use_mla - else torch.Size( - [2, payload.num_layers, self.chunk_size, payload.hidden_dim_size] - ) - ) - layout_desc = MemoryLayoutDesc(shapes=[shape], dtypes=[dtype]) - self.contexts[payload.instance_id] = RegisteredContext( - model_name=payload.model_name, - world_size=payload.world_size, - non_cuda_metadata=NonGpuContextMetadata( - layout_desc=layout_desc, - block_size=payload.block_size, - use_mla=payload.use_mla, - ), - ) - - def _resolve_obj_keys(self, key: IPCCacheEngineKey) -> list[ObjectKey]: - """Resolve object keys from an IPC cache key. - - Args: - key: IPC cache key describing model/session/token range. - - Returns: - Resolved object keys for the requested token range. - - Raises: - ValueError: If ``key.worker_id`` is ``None``. - """ - session = self.session_manager.get_or_create(key.request_id) - session.set_tokens(list(key.token_ids)) - chunk_hashes = [ - TokenHasher.hash_to_bytes(h) for h in session.get_hashes(key.start, key.end) - ] - if key.worker_id is None: - raise ValueError("Must resolve keys with worker_id != None") - return ipc_key_to_object_keys(key, chunk_hashes) - - @_lmcache_nvtx_annotate - def prepare_store( - self, - key: IPCCacheEngineKey, - instance_id: int, - ) -> PrepareStoreResponse: - """Prepare a store operation. For pickle mode, returns empty slots. - - Args: - key: Cache key for the token range to store. - instance_id: Worker instance identifier. - - Returns: - PrepareStoreResponse with empty slots for pickle mode. - """ - - return PrepareStoreResponse(context={}) - - @_lmcache_nvtx_annotate - def commit_store( - self, - key: IPCCacheEngineKey, - instance_id: int, - cpu_data: bytes, - ) -> bool: - """Commit serialized CPU chunks to storage. - - Args: - key: Cache key for the token range to store. - instance_id: Worker instance identifier. - cpu_data: Pickled list of CPU tensors produced by the worker. - - Returns: - ``True`` when all reserved objects are written, otherwise ``False``. - """ - obj_keys = self._resolve_obj_keys(key) - - context = self.contexts.get(instance_id) - if context is None or context.non_cuda_metadata is None: - raise ValueError( - f"non-CUDA context not registered for instance ID {instance_id}" - ) - ctx = context.non_cuda_metadata - chunks: list[torch.Tensor] = pickle.loads(cpu_data) - reserved_dict = self.storage_manager.reserve_write( - obj_keys, ctx.layout_desc, "new" - ) - written_keys: list[ObjectKey] = [] - try: - for idx, obj_key in enumerate(obj_keys): - if obj_key not in reserved_dict: - continue - if idx >= len(chunks): - continue - memory_obj = reserved_dict[obj_key] - if memory_obj.tensor is None: - continue - chunk_cpu = chunks[idx] - if chunk_cpu.shape != memory_obj.tensor.shape: - continue - memory_obj.tensor.copy_(chunk_cpu) - written_keys.append(obj_key) - finally: - if written_keys: - self.storage_manager.finish_write(written_keys) - - return len(written_keys) == len(reserved_dict) - - @_lmcache_nvtx_annotate - def prepare_retrieve( - self, - key: IPCCacheEngineKey, - instance_id: int, - ) -> PrepareRetrieveResponse: - """Retrieve prefetched chunks and return serialized CPU tensors. - - Args: - key: Cache key for the token range to retrieve. - instance_id: Worker instance identifier. - - Returns: - PrepareRetrieveResponse with serialized data on hit. - """ - - obj_keys = self._resolve_obj_keys(key) - - context = self.contexts.get(instance_id) - if context is None or context.non_cuda_metadata is None: - raise ValueError( - f"non-CUDA context not registered for instance ID {instance_id}" - ) - - prefetched_keys: list[ObjectKey] = [] - try: - with self.storage_manager.read_prefetched_results(obj_keys) as memory_objs: - if not memory_objs or len(memory_objs) != len(obj_keys): - return PrepareRetrieveResponse(success=False, data=b"", context={}) - prefetched_keys = obj_keys[: len(memory_objs)] - chunks = [] - for memory_obj in memory_objs: - if memory_obj.tensor is None: - return PrepareRetrieveResponse( - success=False, data=b"", context={} - ) - chunks.append(memory_obj.tensor.cpu().clone()) - return PrepareRetrieveResponse( - success=True, data=pickle.dumps(chunks), context={} - ) - finally: - if prefetched_keys: - self.storage_manager.finish_read_prefetched(prefetched_keys) - - @_lmcache_nvtx_annotate - def commit_retrieve( - self, - key: IPCCacheEngineKey, - instance_id: int, - ) -> bool: - """Finalize a retrieve operation. No-op for pickle mode. - - Args: - key: Cache key (unused for pickle). - instance_id: Worker instance identifier (unused for pickle). - - Returns: - Always ``True``. - """ - return True - - @_lmcache_nvtx_annotate - def store( - self, - key: IPCCacheEngineKey, - instance_id: int, - gpu_block_ids: list[int], - event_ipc_handle: bytes, - ) -> tuple[bytes, bool]: - """ - Stores the GPU KV cache blocks to CPU. - - Args: - key (IPCCacheEngineKey): The IPC key for the KV cache blocks. - Must have worker_id != None (worker store operation). - instance_id (int): The GPU instance ID (such as PID). - gpu_block_ids (list[int]): The GPU block IDs to store. - event_ipc_handle (bytes): The IPC handle of the event to wait on. - - Returns: - tuple[bytes, bool]: The first element is the IPC handle of the event - that signals the completion of the store operation. The second - element indicates whether the store operation was successful. - """ - st = time.perf_counter() - obj_keys = self._resolve_obj_keys(key) - - context = self.contexts.get(instance_id) - assert context is not None, ( - f"No context registered for instance ID {instance_id}" - ) - assert context.gpu_context is not None, ( - f"GPU context not registered for instance ID {instance_id}" - ) - gpu_context = context.gpu_context - model_name = context.model_name - - # ``blocks_per_chunk`` is counted in inference-engine-side - # blocks (each block addresses - # ``inference_engine_logical_block_size`` *logical* tokens). - # For compressed groups the per-group physical slot count - # differs, but the block-id indexing is shared with the engine - # and therefore uses the engine logical block size here. - blocks_per_chunk = ( - self.chunk_size - // gpu_context.kv_layer_groups_manager.inference_engine_logical_block_size - ) - - with ( - torch_dev.device(gpu_context.device), - torch_dev.stream(gpu_context.stream), - ): - # Not all backends support interprocess Events (CUDA IPC specific) - check_interprocess_event_support() - event = torch_dev.Event(interprocess=True) - - # Stage all block_ids to GPU once before the loop - all_block_ids_gpu = gpu_context.stage_block_ids(gpu_block_ids) - - # Wait for vLLM to finish - # Not all backends support IPC event handles (CUDA IPC specific) - if not hasattr(torch_dev.Event, "from_ipc_handle"): - raise RuntimeError( - f"Backend '{torch_device_type}' does not support IPC event " - "handles (Event.from_ipc_handle not available). " - "Multiprocess IPC requires CUDA." - ) - vllm_event = torch_dev.Event.from_ipc_handle( - gpu_context.device, event_ipc_handle - ) - vllm_event.wait(stream=gpu_context.stream) - - # CPU-synchronous sentinel: a GPU store is about to be enqueued. - # Must be published via publish() (not publish_on_stream) so the - # drain thread sees it before MP_REQUEST_END can race MP_STORE_END. - self._event_bus.publish( - Event( - event_type=EventType.MP_STORE_SUBMITTED, - session_id=key.request_id, - metadata={"device": str(gpu_context.device)}, - ) - ) - - self._event_bus.publish_on_stream( - gpu_context.cupy_stream, - Event( - event_type=EventType.MP_STORE_START, - session_id=key.request_id, - metadata={ - "device": str(gpu_context.device), - "engine_id": instance_id, - "model_name": model_name, - }, - ), - ) - - reserved_dict: dict = {} - try: - layout_desc = get_layout_desc(gpu_context, self.chunk_size) - reserved_dict = self.storage_manager.reserve_write( - obj_keys, layout_desc, "new" - ) - - # NOTE: Store is not batched because some obj_keys may be - # skipped (not in reserved_dict), making block_ids - # non-contiguous. Batching would require torch.cat to - # reassemble block_ids, negating the benefit. - num_groups = gpu_context.kv_layer_groups_manager.num_groups - for idx, obj_key in enumerate(obj_keys): - if obj_key in reserved_dict: - memory_obj = reserved_dict[obj_key] - else: - continue - - chunk_block_ids_gpu = all_block_ids_gpu[ - idx * blocks_per_chunk : (idx + 1) * blocks_per_chunk - ] - - # Copy from GPU paged buffer to tmp buffer, then to CPU — per group - for group_idx in range(num_groups): - tmp_buffer = gpu_context.get_tmp_chunk_gpu_buffer(group_idx) - group_kv_pointers = gpu_context.get_group_kv_pointers(group_idx) - # Kernel contract: ``group_lmcache_chunk_size`` here is the - # number of *physical* slots per chunk for this group - # (= logical chunk_size // compress_ratio). - group_lmcache_chunk_size = gpu_context.get_physical_chunk_size( - group_idx - ) - lmc_ops.multi_layer_block_kv_transfer( - group_kv_pointers, - [tmp_buffer.data_ptr()], - chunk_block_ids_gpu, - gpu_context.device, - lmc_ops.TransferDirection.D2H, - gpu_context.get_shape_desc(group_idx), - group_lmcache_chunk_size, - gpu_context.gpu_kv_format_, - 0, - ) - # Store is not batched, so we always use chunk_idx=0 (single slot) - lmcache_memcpy_async_d2h( - gpu_context.get_tmp_gpu_buffer_flat(chunk_idx=0), memory_obj - ) - except Exception: - logger.exception("Cannot store keys due to exception") - finally: - event.record() - if reserved_dict: - submit_callback_to_stream( - gpu_context.cupy_stream, - "finish_write", - list(reserved_dict.keys()), - ) - # All reserved MemoryObjs share one layout_desc, so per-object - # size is identical — avoid summing N identical values. - total_bytes = ( - next(iter(reserved_dict.values())).get_size() * len(reserved_dict) - if reserved_dict - else 0 - ) - self._event_bus.publish_on_stream( - gpu_context.cupy_stream, - Event( - event_type=EventType.MP_STORE_END, - session_id=key.request_id, - metadata={ - "stored_count": len(reserved_dict), - "device": str(gpu_context.device), - "engine_id": instance_id, - "model_name": model_name, - "total_bytes": total_bytes, - }, - ), - ) - - ed = time.perf_counter() - if length := len(reserved_dict): - logger.info( - "Stored %d tokens in %.3f seconds", - length * self.chunk_size, - ed - st, - ) - return event.ipc_handle(), True - - @_lmcache_nvtx_annotate - def retrieve( - self, - key: IPCCacheEngineKey, - instance_id: int, - gpu_block_ids: list[int], - event_ipc_handle: bytes, - skip_first_n_tokens: int = 0, - ) -> tuple[bytes, bool]: - """ - Retrieves the CPU KV cache and put into GPU blocks. - - Args: - key (IPCCacheEngineKey): The IPC key for the KV cache blocks. - Must have worker_id != None (worker retrieve operation). - instance_id (int): The GPU instance ID (such as PID). - gpu_block_ids (list[int]): The GPU block IDs to retrieve into. - event_ipc_handle (bytes): The IPC handle of the event to wait on. - skip_first_n_tokens (int): Number of tokens to skip writing at - the start of the retrieve range. This avoids overwriting - APC-shared GPU blocks that may be read concurrently by other - requests. - - Returns: - tuple[bytes, bool]: The first element is the IPC handle of the event - that signals the completion of the retrieve operation. The second - element indicates whether the key was successfully retrieved. - """ - st = time.perf_counter() - obj_keys = self._resolve_obj_keys(key) - - context = self.contexts.get(instance_id) - assert context is not None, ( - f"No context registered for instance ID {instance_id}" - ) - assert context.gpu_context is not None, ( - f"GPU context not registered for instance ID {instance_id}" - ) - gpu_context = context.gpu_context - model_name = context.model_name - - # CPU-synchronous sentinel: a GPU retrieve is about to be enqueued. - # Must be published via publish() (not publish_on_stream) so the - # drain thread sees it before MP_REQUEST_END can race MP_RETRIEVE_END. - self._event_bus.publish( - Event( - event_type=EventType.MP_RETRIEVE_SUBMITTED, - session_id=key.request_id, - metadata={"device": str(gpu_context.device)}, - ) - ) - - self._event_bus.publish_on_stream( - gpu_context.cupy_stream, - Event( - event_type=EventType.MP_RETRIEVE_START, - session_id=key.request_id, - metadata={ - "device": str(gpu_context.device), - "engine_id": instance_id, - "model_name": model_name, - }, - ), - ) - - # ``skip_*_in_chunk`` is expressed in engine-block units - # (logical tokens), which is what the kernel's - # ``skip_blocks_in_chunk`` argument expects regardless - # of per-group compression. - ie_logical_block_size = ( - gpu_context.kv_layer_groups_manager.inference_engine_logical_block_size - ) - blocks_per_chunk = self.chunk_size // ie_logical_block_size - - def _retrieve_loop(keys: list[ObjectKey], memory_objs: list[MemoryObj]) -> None: - _BATCH_SIZE = gpu_context.max_batch_size - num_groups = gpu_context.kv_layer_groups_manager.num_groups - for batch_idx, memory_obj_batch in enumerate( - batched_iteration(memory_objs, batch_size=_BATCH_SIZE) - ): - batch_len = len(memory_obj_batch) - chunk_start = batch_idx * self.chunk_size * _BATCH_SIZE - chunk_end = chunk_start + self.chunk_size * batch_len - - effective_start = max(chunk_start, skip_first_n_tokens) - if effective_start >= chunk_end: - # Entire batch is within APC range, skip it - continue - - skip_tokens_in_chunk = max( - 0, - min( - effective_start - chunk_start, - self.chunk_size * batch_len - 1, - ), - ) - if skip_tokens_in_chunk % ie_logical_block_size != 0: - logger.error( - "skip_first_n_tokens (%d) is not aligned to " - "inference_engine_logical_block_size (%d), " - "rounding down from %d tokens to %d blocks", - skip_first_n_tokens, - ie_logical_block_size, - skip_tokens_in_chunk, - skip_tokens_in_chunk // ie_logical_block_size, - ) - skip_blocks_in_chunk = skip_tokens_in_chunk // ie_logical_block_size - - start_chunk_id = batch_idx * _BATCH_SIZE - end_chunk_id = start_chunk_id + batch_len - chunk_block_ids_gpu = all_block_ids_gpu[ - start_chunk_id * blocks_per_chunk : end_chunk_id * blocks_per_chunk - ] - - # Copy from CPU to GPU tmp buffers, then scatter to paged KV — per group - # H2D copy: each memory_obj maps to its own batch slot - for chunk_idx, memory_obj in enumerate(memory_obj_batch): - lmcache_memcpy_async_h2d( - memory_obj, - gpu_context.get_tmp_gpu_buffer_flat(chunk_idx=chunk_idx), - ) - for group_idx in range(num_groups): - tmp_buffers = gpu_context.get_tmp_chunk_gpu_buffer_batched( - batch_len, group_idx - ) - group_kv_pointers = gpu_context.get_group_kv_pointers(group_idx) - group_lmcache_chunk_size = gpu_context.get_physical_chunk_size( - group_idx - ) - - lmc_ops.multi_layer_block_kv_transfer( - group_kv_pointers, - [tb.data_ptr() for tb in tmp_buffers], - chunk_block_ids_gpu, - gpu_context.device, - lmc_ops.TransferDirection.H2D, - gpu_context.get_shape_desc(group_idx), - group_lmcache_chunk_size, - gpu_context.gpu_kv_format_, - skip_blocks_in_chunk, - ) - - with ( - torch_dev.device(gpu_context.device), - torch_dev.stream(gpu_context.stream), - ): - # Stage all block_ids to GPU once before the loop - all_block_ids_gpu = gpu_context.stage_block_ids(gpu_block_ids) - - # Not all backends support interprocess Events (CUDA IPC specific) - check_interprocess_event_support() - event = torch_dev.Event(interprocess=True) - - prefetched_keys: list[ObjectKey] = [] - retrieve_succeeded = False - total_bytes = 0 - try: - with self.storage_manager.read_prefetched_results( - obj_keys - ) as memory_objs: - if not memory_objs or len(memory_objs) != len(obj_keys): - logger.error("Some keys not found during retrieve!") - return event.ipc_handle(), False - - prefetched_keys = obj_keys[: len(memory_objs)] - total_bytes = sum(mo.get_size() for mo in memory_objs) - _retrieve_loop(obj_keys, memory_objs) - # Only set True when with-block exits normally - retrieve_succeeded = True - except Exception: - logger.exception("Cannot retrieve keys due to exception") - return event.ipc_handle(), False - finally: - event.record() - if retrieve_succeeded: - submit_callback_to_stream( - gpu_context.cupy_stream, - "finish_read_prefetched", - prefetched_keys, - ) - self._event_bus.publish_on_stream( - gpu_context.cupy_stream, - Event( - event_type=EventType.MP_RETRIEVE_END, - session_id=key.request_id, - metadata={ - "retrieved_count": len(prefetched_keys), - "device": str(gpu_context.device), - "engine_id": instance_id, - "model_name": model_name, - "cache_salt": key.cache_salt, - "total_bytes": total_bytes, - }, - ), - ) - tokens_retrieved = len(obj_keys) * self.chunk_size - ed = time.perf_counter() - logger.info( - "Retrieved %d tokens in %.3f seconds", - tokens_retrieved, - ed - st, - ) - - return event.ipc_handle(), True - - def _find_layout_desc( - self, - model_name: str, - world_size: int, - ) -> MemoryLayoutDesc | None: - """Find layout desc from a matching GPU or CPU context. - - Returns: - The layout descriptor, or None if no context matches - ``(model_name, world_size)``. GPU contexts are checked first, - then CPU contexts. - """ - for context in self.contexts.values(): - if context.model_name == model_name and context.world_size == world_size: - return context.get_layout_desc(self.chunk_size) - return None - - def lookup( - self, - key: IPCCacheEngineKey, - tp_size: int, - ) -> None: - """Submit a prefix lookup. - - Hashes the key, submits a prefetch task to the storage manager, - and registers the job under ``key.request_id`` for later polling - via query_prefetch_status. - - Args: - key: Cache key with request_id embedded. - tp_size: Tensor-parallel size for MLA multi-reader locking. - """ - model_name, world_size = key.model_name, key.world_size - self._event_bus.publish( - Event( - event_type=EventType.MP_REQUEST_START, - session_id=key.request_id, - ) - ) - self._event_bus.publish( - Event( - event_type=EventType.MP_LOOKUP_PREFETCH_START, - session_id=key.request_id, - ) - ) - - layout_desc = self._find_layout_desc(model_name, world_size) - if layout_desc is None: - logger.error( - "No GPU context found for model %s with world size %d during lookup!", - model_name, - world_size, - ) - self._register_prefetch_job( - _PrefetchJob( - handle=PrefetchHandle( - prefetch_request_id=-1, - external_request_id=key.request_id, - l1_prefix_hit_count=0, - total_requested_keys=0, - submit_time=time.monotonic(), - ), - world_size=1, - request_id=key.request_id, - requested_tokens=0, - model_name=model_name, - cache_salt=key.cache_salt, - ) - ) - return - - extra_count = compute_extra_count(tp_size, world_size) - - # Compute chunk hashes for all full chunks - chunk_hashes = self.token_hasher.compute_chunk_hashes(list(key.token_ids)) - if not chunk_hashes: - self._register_prefetch_job( - _PrefetchJob( - handle=PrefetchHandle( - prefetch_request_id=-1, - external_request_id=key.request_id, - l1_prefix_hit_count=0, - total_requested_keys=0, - submit_time=time.monotonic(), - ), - world_size=1, - request_id=key.request_id, - requested_tokens=0, - model_name=model_name, - cache_salt=key.cache_salt, - ) - ) - return - - # Total chunk-aligned tokens submitted for lookup; surfaces as the - # denominator of the L1+L2 token-level hit-rate via the - # ``requested_tokens`` field on ``MP_LOOKUP_PREFETCH_END``. Sub-chunk - # trailing tokens are intentionally excluded — they cannot hit at - # chunk granularity. - requested_tokens = len(chunk_hashes) * self.chunk_size - - # Publish lookup event via EventBus for observability subscribers. - # Guard with has_subscribers() to avoid allocating the metadata dict - # (including dtype/shape list comprehensions) when no subscriber is - # listening (e.g. lookup hash logger is disabled). - if self._event_bus.has_subscribers(EventType.MP_LOOKUP): - self._event_bus.publish( - Event( - event_type=EventType.MP_LOOKUP, - session_id=key.request_id, - metadata={ - "request_id": key.request_id, - "chunk_hashes": chunk_hashes, - "model_name": model_name, - "chunk_size": self.chunk_size, - "seq_len": len(key.token_ids), - "dtypes": [str(d) for d in layout_desc.dtypes], - "shapes": [list(s) for s in layout_desc.shapes], - }, - ) - ) - - # set lookup ipc key, for session manager to use and generate object keys - session = self.session_manager.get_or_create(key.request_id) - session.set_tokens(list(key.token_ids)) - session.lookup_ipc_key = key - - obj_keys = ipc_key_to_object_keys(key, chunk_hashes) - - handle = self.storage_manager.submit_prefetch_task( - obj_keys, - layout_desc, - extra_count=extra_count, - external_request_id=key.request_id, - ) - self._register_prefetch_job( - _PrefetchJob( - handle=handle, - world_size=key.world_size, - request_id=key.request_id, - requested_tokens=requested_tokens, - model_name=model_name, - cache_salt=key.cache_salt, - ) - ) - - def _register_prefetch_job(self, job: _PrefetchJob) -> None: - with self._prefetch_job_lock: - self._prefetch_jobs[job.request_id] = job - - def query_prefetch_lookup_hits( - self, - request_id: str, - ) -> int | None: - """Query the number of hits for a prefetch request before it's finished. - - Returns: - The number of hits for the prefetched keys if the lookup phase is - done. None if the lookup phase is still in progress. 0 if the - request_id is unknown (already completed and consumed, or invalid). - """ - with self._prefetch_job_lock: - job = self._prefetch_jobs.get(request_id) - - if job is None: - logger.warning( - "Prefetch job for request %s not found (already completed or invalid)", - request_id, - ) - return 0 - - found_count = self.storage_manager.query_prefetch_lookup_hits(job.handle) - if found_count is None: - return None - - found_count = found_count // job.world_size - return found_count - - def query_prefetch_status( - self, - request_id: str, - ) -> int | None: - """Poll the status of a prefetch job by request_id. - - Returns the chunk count when the prefetch is complete, or None - if it is still in progress. The job entry is automatically - removed once a non-None result is returned (exactly-once - semantics). - - Args: - request_id: The external request ID passed in the lookup key. - - Returns: - Chunk count (int) when done, None if still in progress, - 0 if the request_id is unknown (already completed and consumed, - or invalid). - """ - with self._prefetch_job_lock: - job = self._prefetch_jobs.get(request_id) - if job is None: - logger.warning( - "Prefetch job for request %s not found (already completed or invalid)", - request_id, - ) - return 0 - - found_count = self.storage_manager.query_prefetch_status(job.handle) - if found_count is None: - return None - - # NOTE(Kuntai): this assumes two things: - # 1. the world size is the same between keys - # 2. the lookup sort the keys in prefix order and breaks at the - # first failure - found_count = found_count // job.world_size - - self._event_bus.publish( - Event( - event_type=EventType.MP_LOOKUP_PREFETCH_END, - session_id=job.request_id, - metadata={ - "found_count": found_count, - "requested_tokens": job.requested_tokens, - "hit_tokens": found_count * self.chunk_size, - "model_name": job.model_name, - "cache_salt": job.cache_salt, - }, - ) - ) - - with self._prefetch_job_lock: - self._prefetch_jobs.pop(request_id, None) - - return found_count - - def free_lookup_locks( - self, - key: IPCCacheEngineKey, - tp_size: int, - ) -> None: - """Release read locks acquired during lookup. - - Hashes are computed only for chunks in ``[start, end)`` to avoid - unnecessary work on tokens outside that range. - ``start`` and ``end`` must be aligned to ``chunk_size``; it is the - caller's responsibility to align the boundaries as desired. - - Computes the extra reader count from ``tp_size`` and - ``world_size`` the same way :meth:`lookup` does, so - the correct number of locks is released. - - Args: - key: Cache key whose read locks should be released. - tp_size: Tensor-parallel size for MLA - multi-reader locking. - """ - chunk_hashes = self.token_hasher.compute_chunk_hashes( - list(key.token_ids), start=key.start, end=key.end - ) - if not chunk_hashes: - return - obj_keys = ipc_key_to_object_keys(key, chunk_hashes) - - extra_count = compute_extra_count(tp_size, key.world_size) - - self.storage_manager.finish_read_prefetched(obj_keys, extra_count=extra_count) - - # ========================================================================= - # Utility methods - # ========================================================================= - - def ping(self) -> bool: - """ - Respond to a ping request. - - Returns: - bool: Always True. - """ - return True + @property + def context(self) -> MPCacheEngineContext: + """Return the shared engine context.""" + return self._context - def get_chunk_size(self) -> int: - """ - Returns the chunk size used for KV cache operations. + def report_status(self) -> dict: + """Return an aggregated status dict from all modules. Returns: - int: The chunk size. + Combined status from the storage manager, engine metadata, + and each module's ``report_status()`` output. """ - return self.chunk_size - - def end_session(self, request_id: str) -> None: - """Remove the session for a finished request. - - Args: - request_id: The request ID whose session should be removed. - """ - self._event_bus.publish( - Event( - event_type=EventType.MP_VLLM_END_SESSION, - metadata={"request_id": request_id}, - ) - ) - session = self.session_manager.remove(request_id) - self._event_bus.publish( - Event( - event_type=EventType.MP_REQUEST_END, - session_id=request_id, - ) - ) - if session is None: - logger.warning("Session %s not found, skipping touch", request_id) - return - if session.lookup_ipc_key is None: - logger.warning( - "Session %s has no lookup ipc key, skipping touch", request_id - ) - return - - chunk_hashes = [TokenHasher.hash_to_bytes(h) for h in session.get_hashes(0)] - obj_keys = ipc_key_to_object_keys(session.lookup_ipc_key, chunk_hashes) - # unified touch of all keys, which include retrieved and stored keys - # TODO(chunxiaozheng): when l2 is enabled, the prefetched keys from l2 are temp - # and will be deleted after finish_read_prefetched, when we touch all keys, - # these keys has been deleted and will not be touched. - self.storage_manager.touch_l1_keys(obj_keys) - - def report_status(self) -> dict: - """Return a status dict for the entire cache engine.""" - sm = self.storage_manager.report_status() - - gpu_context_meta: dict[str, dict] = {} - non_cuda_context_meta: dict[str, dict] = {} - registered_gpu_ids: list[int] = [] - registered_non_cuda_ids: list[int] = [] - - for instance_id, context in self.contexts.items(): - entry: dict = { - "model_name": context.model_name, - "world_size": context.world_size, - } - if context.gpu_context is not None: - registered_gpu_ids.append(instance_id) - ctx = context.gpu_context - entry["kv_cache_layout"] = { - "num_layers": ctx.num_layers, - "inference_engine_logical_block_size": ( - ctx.kv_layer_groups_manager.inference_engine_logical_block_size - ), - "group_physical_block_sizes": ctx.group_physical_block_sizes, - "group_compress_ratios": ctx.group_compress_ratios, - "hidden_dim_sizes": str(ctx.hidden_dim_sizes), - "dtype": str(ctx.dtype), - "is_mla": ctx.is_mla, - "num_blocks": ctx.num_blocks, - "gpu_kv_format": ctx.gpu_kv_format_name, - "gpu_kv_shape": ctx.gpu_kv_shape, - "gpu_kv_concrete_shape": ctx.concrete_gpu_kv_shape, - "attention_backend": ctx.attention_backend, - "cache_size_per_token": ctx.cache_size_per_token(), - } - gpu_context_meta[str(instance_id)] = entry - continue - - if context.non_cuda_metadata is not None: - registered_non_cuda_ids.append(instance_id) - non_cuda_context_meta[str(instance_id)] = { - **entry, - "block_size": context.non_cuda_metadata.block_size, - "use_mla": context.non_cuda_metadata.use_mla, - } - - return { + sm = self._context.storage_manager.report_status() + status: dict = { "is_healthy": sm["is_healthy"], "engine_type": self.__class__.__name__, - "chunk_size": self.chunk_size, - "hash_algorithm": self.token_hasher.hash_algorithm_name, - "registered_gpu_ids": registered_gpu_ids, - "gpu_context_meta": gpu_context_meta, - "registered_non_cuda_instance_ids": registered_non_cuda_ids, - "non_cuda_context_meta": non_cuda_context_meta, - "active_sessions": self.session_manager.active_count(), - "active_prefetch_jobs": self._active_prefetch_count(), + "chunk_size": self._context.chunk_size, + "hash_algorithm": self._context.token_hasher.hash_algorithm_name, + "active_sessions": self._context.session_manager.active_count(), "storage_manager": sm, } - - def report_block_allocations( - self, - instance_id: int, - model_name: str, - records: list[BlockAllocationRecord], - ) -> None: - """Publish vLLM block allocation records to the EventBus. - - Args: - instance_id: The scheduler instance ID. - model_name: The model name from the adapter. - records: List of BlockAllocationRecord with per-request - block and token allocation deltas. - """ - self._event_bus.publish( - Event( - event_type=EventType.MP_VLLM_BLOCK_ALLOCATION, - metadata={ - "instance_id": instance_id, - "model_name": model_name, - "records": records, - }, - ) - ) - - def debug(self) -> str: - return "OK" - - def clear(self) -> None: - """ - Clears all stored KV cache data from the storage manager. - """ - with self.lock: - self.storage_manager.memcheck() - self.storage_manager.clear(force=True) - self.storage_manager.memcheck() + for module in self._modules: + status.update(module.report_status()) + return status def close(self) -> None: - """ - Closes the MPCacheEngine and releases all resources. - """ - # Stop the drain thread before storage_manager.close() so any - # in-flight completions reach a live storage manager. - self._device_host_func_dispatcher.stop() - - # Close storage manager - self.storage_manager.close() + """Close all modules and release shared resources.""" + for module in self._modules: + module.close() + self._context.storage_manager.close() logger.info("MPCacheEngine closed") - # Release GPU contexts - self.contexts.clear() - - def _active_prefetch_count(self) -> int: - """Return the number of active prefetch jobs (thread-safe).""" - with self._prefetch_job_lock: - return len(self._prefetch_jobs) - - def _setup_metrics(self) -> None: - """Register OTel observable gauges for MP engine metrics.""" - _gauge = partial(register_gauge, "lmcache.mp_engine") - _gauge( - "lmcache_mp.active_prefetch_jobs", - "Number of active prefetch jobs", - self._active_prefetch_count, - ) - def add_handler_helper( server: MessageQueueServer, request_type: RequestType, handler_function ): + """Register a handler with the message queue server. + + Args: + server: The message queue server. + request_type: The request type to handle. + handler_function: The handler callable. + """ payload_classes = get_payload_classes(request_type) handler_type = get_handler_type(request_type) server.add_handler( @@ -1403,124 +121,119 @@ def add_handler_helper( ) +def _build_modules( + ctx: MPCacheEngineContext, + mp_config: MPServerConfig, +) -> list[EngineModule]: + """Assemble the list of engine modules based on configuration. + + Args: + ctx: The shared engine context. + mp_config: Server configuration determining which modules to load. + + Returns: + List of initialized engine modules. + + Raises: + ValueError: If blend engine is requested with non-GPU transfer mode. + """ + modules: list[EngineModule] = [ + LookupModule(ctx), + ManagementModule(ctx), + ] + + if mp_config.transfer_mode == "gpu": + modules.append(GPUTransferModule(ctx)) + else: + modules.append(NonGPUTransferModule(ctx)) + + if mp_config.engine_type == "blend": + if mp_config.transfer_mode != "gpu": + raise ValueError( + "Blend engine requires transfer_mode='gpu', " + f"got '{mp_config.transfer_mode}'" + ) + # First Party + from lmcache.v1.multiprocess.modules.blend import BlendModule + + modules.append(BlendModule(ctx)) + + return modules + + def run_cache_server( mp_config: MPServerConfig, storage_manager_config: StorageManagerConfig, obs_config: ObservabilityConfig, return_engine: bool = False, start_prometheus_http_server: bool = True, -): - """ - Run the LMCache cache server with ZMQ message queue. +) -> tuple[MessageQueueServer, MPCacheEngine] | None: + """Run the LMCache cache server with ZMQ message queue. Args: - mp_config: Configuration for the ZMQ multiprocess server - storage_manager_config: Configuration for the storage manager - obs_config: Configuration for the observability stack + mp_config: Configuration for the ZMQ multiprocess server. + storage_manager_config: Configuration for the storage manager. + obs_config: Configuration for the observability stack. return_engine: If True, return (server, engine) after starting; - if False, run blocking loop to keep server alive + if False, run blocking loop to keep server alive. start_prometheus_http_server: Whether to start a standalone Prometheus HTTP server in a background thread. Set to ``False`` when an external HTTP framework already serves ``/metrics`` to avoid port conflicts or redundant servers. Returns: - If return_engine is True: tuple of (MessageQueueServer, MPCacheEngine) - If return_engine is False: None (blocks until interrupted) + If return_engine is True: tuple of (MessageQueueServer, MPCacheEngine). + If return_engine is False: None (blocks until interrupted). """ event_bus = init_observability( obs_config, start_prometheus_http_server=start_prometheus_http_server ) - # Wire up the trace recorder (no-op when --trace-level is unset). - # Registered before the engine handlers are added so any - # storage-manager calls during engine init are captured too. maybe_initialize_trace_recorder(event_bus, obs_config, storage_manager_config) - # Initialize the engine (loggers self-register with the global controller) - engine = MPCacheEngine( + ctx = MPCacheEngineContext( storage_manager_config=storage_manager_config, chunk_size=mp_config.chunk_size, hash_algorithm=mp_config.hash_algorithm, ) - # Initialize the message queue server - context = zmq.Context.instance() + modules = _build_modules(ctx, mp_config) + engine = MPCacheEngine(ctx, modules) + + zmq_context = zmq.Context.instance() server = MessageQueueServer( bind_url=f"tcp://{mp_config.host}:{mp_config.port}", - context=context, + context=zmq_context, ) - # Add handlers - add_handler_helper(server, RequestType.REGISTER_KV_CACHE, engine.register_kv_cache) - add_handler_helper( - server, RequestType.UNREGISTER_KV_CACHE, engine.unregister_kv_cache - ) - add_handler_helper(server, RequestType.STORE, engine.store) - add_handler_helper( - server, - RequestType.REGISTER_KV_CACHE_NON_GPU_CONTEXT, - engine.register_kv_cache_non_gpu_context, - ) - add_handler_helper(server, RequestType.PREPARE_STORE, engine.prepare_store) - add_handler_helper(server, RequestType.LOOKUP, engine.lookup) - add_handler_helper( - server, RequestType.QUERY_PREFETCH_STATUS, engine.query_prefetch_status - ) - add_handler_helper( - server, - RequestType.QUERY_PREFETCH_LOOKUP_HITS, - engine.query_prefetch_lookup_hits, - ) - add_handler_helper(server, RequestType.FREE_LOOKUP_LOCKS, engine.free_lookup_locks) - add_handler_helper(server, RequestType.RETRIEVE, engine.retrieve) - add_handler_helper(server, RequestType.COMMIT_STORE, engine.commit_store) - add_handler_helper(server, RequestType.PREPARE_RETRIEVE, engine.prepare_retrieve) - add_handler_helper(server, RequestType.COMMIT_RETRIEVE, engine.commit_retrieve) - add_handler_helper(server, RequestType.CLEAR, engine.clear) - add_handler_helper(server, RequestType.GET_CHUNK_SIZE, engine.get_chunk_size) - add_handler_helper(server, RequestType.PING, engine.ping) - add_handler_helper(server, RequestType.END_SESSION, engine.end_session) - add_handler_helper(server, RequestType.NOOP, engine.debug) - add_handler_helper( - server, - RequestType.REPORT_BLOCK_ALLOCATION, - engine.report_block_allocations, - ) + all_specs: list[HandlerSpec] = [] + for module in modules: + all_specs.extend(module.get_handlers()) - # Assign thread pools - server.add_affinity_thread_pool( - [ - RequestType.STORE, - RequestType.RETRIEVE, - RequestType.PREPARE_STORE, - RequestType.COMMIT_STORE, - RequestType.PREPARE_RETRIEVE, - RequestType.COMMIT_RETRIEVE, - ], - max_workers=mp_config.max_gpu_workers, - ) - server.add_normal_thread_pool( - [ - RequestType.LOOKUP, - RequestType.QUERY_PREFETCH_STATUS, - RequestType.QUERY_PREFETCH_LOOKUP_HITS, - RequestType.FREE_LOOKUP_LOCKS, - RequestType.END_SESSION, - RequestType.CLEAR, - RequestType.PING, - RequestType.REPORT_BLOCK_ALLOCATION, - ], - max_workers=mp_config.max_cpu_workers, - ) + for spec in all_specs: + add_handler_helper(server, spec.request_type, spec.handler) + + affinity_types = [ + s.request_type for s in all_specs if s.pool == ThreadPoolType.AFFINITY + ] + normal_types = [ + s.request_type for s in all_specs if s.pool == ThreadPoolType.NORMAL + ] + if affinity_types: + server.add_affinity_thread_pool( + affinity_types, max_workers=mp_config.max_gpu_workers + ) + if normal_types: + server.add_normal_thread_pool( + normal_types, max_workers=mp_config.max_cpu_workers + ) logger.info( "LMCache ZMQ cache server is running on tcp://%s:%d", mp_config.host, mp_config.port, ) - # Start the ZMQ server - # Not all backends expose init(); some auto-initialize on first use + if not hasattr(torch_dev, "init"): logger.warning( "Backend '%s' does not support init(), skipping device init", @@ -1532,11 +245,9 @@ def run_cache_server( logger.info("LMCache cache server is running...") - # Return server and engine if requested (for HTTP server integration) if return_engine: return server, engine - # Dummy loop to keep the server running try: while True: time.sleep(1) @@ -1545,9 +256,15 @@ def run_cache_server( event_bus.stop() server.close() engine.close() + return None def parse_args(): + """Parse command line arguments for the cache server. + + Returns: + Parsed arguments namespace. + """ parser = argparse.ArgumentParser( description="LMCache ZMQ Cache Server (without HTTP)" ) diff --git a/tests/v1/multiprocess/test_blend_server_v2.py b/tests/v1/multiprocess/test_blend_server_v2.py index bdc91e68e17..715711df3f8 100644 --- a/tests/v1/multiprocess/test_blend_server_v2.py +++ b/tests/v1/multiprocess/test_blend_server_v2.py @@ -1,13 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 """ -Unit and integration tests for BlendTokenRangeMatcher and BlendEngineV2. +Unit and integration tests for BlendTokenRangeMatcher and BlendModule. Structure --------- Part 1 – BlendTokenRangeMatcher (pure unit tests, no GPU/server needed) Tests the rolling-hash sub-sequence matching logic in isolation. -Part 2 – BlendEngineV2 integration tests (two-process ZMQ architecture) +Part 2 – Blend integration tests (two-process ZMQ architecture) Uses CB_LOOKUP_PRE_COMPUTED_V2 / CB_RETRIEVE_PRE_COMPUTED_V2, which return/accept list[CBMatchResult] instead of list[tuple[int, int]]. @@ -42,17 +42,17 @@ StorageManagerConfig, ) from lmcache.v1.mp_observability.config import DEFAULT_OBSERVABILITY_CONFIG -from lmcache.v1.multiprocess.blend_server_v2 import ( - BlendEngineV2, - BlendTokenRangeMatcher, - _unique_token_coverage, -) from lmcache.v1.multiprocess.custom_types import ( CBMatchResult, CudaIPCWrapper, IPCCacheEngineKey, KVCache, ) +from lmcache.v1.multiprocess.modules.blend import ( + BlendModule, + BlendTokenRangeMatcher, + _unique_token_coverage, +) from lmcache.v1.multiprocess.mq import MessageQueueClient from lmcache.v1.multiprocess.protocol import ( RequestType, @@ -692,17 +692,17 @@ def expected_full_chunks(num_tokens: int, chunk_size: int = CHUNK_SIZE) -> int: # ============================================================================= -# Server Process Runner (BlendEngineV2) +# Server Process Runner (Blend Mode) # ============================================================================= def server_process_runner_v2( host: str, port: int, chunk_size: int, cpu_buffer_size: float ): - """Entry point for the server process running BlendEngineV2.""" + """Entry point for the server process running blend mode.""" # First Party - from lmcache.v1.multiprocess.blend_server_v2 import run_cache_server from lmcache.v1.multiprocess.config import MPServerConfig + from lmcache.v1.multiprocess.server import run_cache_server mp_config = MPServerConfig( host=host, @@ -733,7 +733,7 @@ def server_process_runner_v2( @pytest.fixture(scope="module") def server_process() -> Generator[mp.Process, None, None]: - """Start the BlendEngineV2 server in a separate process for the module.""" + """Start the blend mode server in a separate process for the module.""" mp.set_start_method("spawn", force=True) process = mp.Process( target=server_process_runner_v2, @@ -855,7 +855,7 @@ def registered_instance( # ============================================================================= -# Part 2: BlendEngineV2 Integration Tests +# Part 2: Blend Integration Tests # ============================================================================= # --------------------------------------------------------------------------- @@ -865,7 +865,7 @@ def registered_instance( def test_server_running_v2(server_process: mp.Process): """Server process should be alive.""" - assert server_process.is_alive(), "BlendEngineV2 server process should be running" + assert server_process.is_alive(), "Blend server process should be running" @pytest.mark.skipif( @@ -2001,19 +2001,28 @@ def test_cb_store_final_v2_visible_to_cb_lookup_v2( # 8. report_status() – CB-extended fields # --------------------------------------------------------------------------- # -# These tests construct a BlendEngineV2 directly in the test process because -# report_status() is invoked in-process by the FastAPI /api/status handler, -# not over ZMQ. CUDA IPC unwrapping is bypassed via a monkeypatched -# unwrap_kv_cache_tensors so the public cb_register_kv_cache path can run -# without a cross-process producer. +# These tests construct an MPCacheEngine with a BlendModule directly in the +# test process because report_status() is invoked in-process by the FastAPI +# /api/status handler, not over ZMQ. CUDA IPC unwrapping is bypassed via a +# monkeypatched unwrap_kv_cache_tensors so the public cb_register_kv_cache +# path can run without a cross-process producer. @pytest.fixture(scope="function") -def in_process_blend_engine() -> Generator[BlendEngineV2, None, None]: - """Build a BlendEngineV2 in-process for direct method-call tests.""" +def in_process_blend_engine() -> Generator[tuple, None, None]: + """Build an MPCacheEngine with BlendModule in-process for direct tests. + + Yields: + (engine, blend_module) tuple. + """ if not torch.cuda.is_available(): pytest.skip("CUDA is not available") + # First Party + from lmcache.v1.multiprocess.engine_context import MPCacheEngineContext + from lmcache.v1.multiprocess.modules.gpu_transfer import GPUTransferModule + from lmcache.v1.multiprocess.server import MPCacheEngine + config = StorageManagerConfig( l1_manager_config=L1ManagerConfig( memory_config=L1MemoryManagerConfig( @@ -2023,12 +2032,15 @@ def in_process_blend_engine() -> Generator[BlendEngineV2, None, None]: ), eviction_config=EvictionConfig(eviction_policy="LRU"), ) - engine = BlendEngineV2( + ctx = MPCacheEngineContext( storage_manager_config=config, chunk_size=CHUNK_SIZE, ) + gpu_module = GPUTransferModule(ctx) + blend_module = BlendModule(ctx) + engine = MPCacheEngine(ctx, [gpu_module, blend_module]) try: - yield engine + yield engine, blend_module finally: engine.close() @@ -2055,10 +2067,11 @@ def _local_unwrap(_kv_caches): not torch.cuda.is_available(), reason="report_status CB tests require CUDA" ) def test_report_status_no_cb_registrations( - in_process_blend_engine: BlendEngineV2, + in_process_blend_engine: tuple, ): """Without any CB registration, the new fields are present and empty.""" - status = in_process_blend_engine.report_status() + engine, _blend_module = in_process_blend_engine + status = engine.report_status() assert "registered_cb_gpu_ids" in status assert "cb_gpu_context_meta" in status @@ -2070,15 +2083,15 @@ def test_report_status_no_cb_registrations( not torch.cuda.is_available(), reason="report_status CB tests require CUDA" ) def test_report_status_surfaces_cb_registration( - in_process_blend_engine: BlendEngineV2, + in_process_blend_engine: tuple, cb_client_context: CBClientContext, local_kv_cache_unwrap, ): """A CB-registered instance shows up in both new fields with correct shape.""" - engine = in_process_blend_engine + engine, blend_module = in_process_blend_engine instance_id = 4242 - engine.cb_register_kv_cache( + blend_module.cb_register_kv_cache( instance_id, cb_client_context.get_kv_cache(), "testmodel", @@ -2110,21 +2123,21 @@ def test_report_status_surfaces_cb_registration( not torch.cuda.is_available(), reason="report_status CB tests require CUDA" ) def test_report_status_unregister_clears_cb_fields( - in_process_blend_engine: BlendEngineV2, + in_process_blend_engine: tuple, cb_client_context: CBClientContext, local_kv_cache_unwrap, ): """Unregistering removes the entry from both new fields.""" - engine = in_process_blend_engine + engine, blend_module = in_process_blend_engine instance_id = 4243 - engine.cb_register_kv_cache( + blend_module.cb_register_kv_cache( instance_id, cb_client_context.get_kv_cache(), "testmodel", 1, ) - engine.cb_unregister_kv_cache(instance_id) + blend_module.cb_unregister_kv_cache(instance_id) status = engine.report_status() assert status["registered_cb_gpu_ids"] == [] diff --git a/tests/v1/multiprocess/test_cache_server.py b/tests/v1/multiprocess/test_cache_server.py index 61818b89536..62c84b94d92 100644 --- a/tests/v1/multiprocess/test_cache_server.py +++ b/tests/v1/multiprocess/test_cache_server.py @@ -38,7 +38,7 @@ SERVER_URL = f"tcp://{SERVER_HOST}:{SERVER_PORT}" CHUNK_SIZE = 256 CPU_BUFFER_SIZE = 5.0 -DEFAULT_TIMEOUT = 5.0 +DEFAULT_TIMEOUT = 10.0 def _has_working_new_shared_cuda() -> bool: diff --git a/tests/v1/multiprocess/test_free_locks.py b/tests/v1/multiprocess/test_free_locks.py index 4ba1936df45..e08a7911513 100644 --- a/tests/v1/multiprocess/test_free_locks.py +++ b/tests/v1/multiprocess/test_free_locks.py @@ -88,42 +88,42 @@ def test_mq_free_locks(): def test_server_free_lookup_locks_calls_finish_read_prefetched(): - """MPCacheEngine.free_lookup_locks should resolve hash keys and call + """LookupModule.free_lookup_locks should resolve hash keys and call finish_read_prefetched on the storage manager.""" # First Party - from lmcache.v1.multiprocess.server import MPCacheEngine + from lmcache.v1.multiprocess.modules.lookup import LookupModule - engine = MagicMock() - engine.token_hasher = MagicMock() - engine.token_hasher.chunk_size = 256 - engine.token_hasher.compute_chunk_hashes.return_value = [b"hash0"] + ctx = MagicMock() + ctx.token_hasher.chunk_size = 256 + ctx.token_hasher.compute_chunk_hashes.return_value = [b"hash0"] + + module = LookupModule(ctx) # Build a key key = create_cache_key(0).no_worker_id_version() sentinel_obj_keys = [MagicMock()] with patch( - "lmcache.v1.multiprocess.server.ipc_key_to_object_keys", + "lmcache.v1.multiprocess.modules.lookup.ipc_key_to_object_keys", return_value=sentinel_obj_keys, ): - # Call the real method on the mock - MPCacheEngine.free_lookup_locks(engine, key, 1) + module.free_lookup_locks(key, 1) - engine.storage_manager.finish_read_prefetched.assert_called_once_with( + module.context.storage_manager.finish_read_prefetched.assert_called_once_with( sentinel_obj_keys, extra_count=0 ) def test_server_free_lookup_locks_no_matching_chunks(): - """MPCacheEngine.free_lookup_locks with no chunks in range should be a no-op.""" + """LookupModule.free_lookup_locks with no chunks in range should be a no-op.""" # First Party - from lmcache.v1.multiprocess.server import MPCacheEngine + from lmcache.v1.multiprocess.modules.lookup import LookupModule + + ctx = MagicMock() + ctx.token_hasher.chunk_size = 256 + ctx.token_hasher.compute_chunk_hashes.return_value = [] - engine = MagicMock() - engine.token_hasher = MagicMock() - engine.token_hasher.chunk_size = 256 - # start=end=0 is passed to compute_chunk_hashes, which returns no hashes - engine.token_hasher.compute_chunk_hashes.return_value = [] + module = LookupModule(ctx) # Key with start == end means no chunks to free key = IPCCacheEngineKey( @@ -136,19 +136,18 @@ def test_server_free_lookup_locks_no_matching_chunks(): request_id="req-empty", ) - MPCacheEngine.free_lookup_locks(engine, key, 1) + module.free_lookup_locks(key, 1) - engine.storage_manager.finish_read_prefetched.assert_not_called() + module.context.storage_manager.finish_read_prefetched.assert_not_called() def test_server_handler_registered(): - """run_cache_server should register a FREE_LOOKUP_LOCKS handler.""" + """LookupModule should have a free_lookup_locks method.""" # First Party - from lmcache.v1.multiprocess.server import MPCacheEngine + from lmcache.v1.multiprocess.modules.lookup import LookupModule - engine = MPCacheEngine.__new__(MPCacheEngine) - assert hasattr(engine, "free_lookup_locks") - assert callable(engine.free_lookup_locks) + assert hasattr(LookupModule, "free_lookup_locks") + assert callable(LookupModule.free_lookup_locks) # ============================================================================ diff --git a/tests/v1/multiprocess/test_non_cuda_data_transfer.py b/tests/v1/multiprocess/test_non_cuda_data_transfer.py index f8a281a8785..a9b246a1dc6 100644 --- a/tests/v1/multiprocess/test_non_cuda_data_transfer.py +++ b/tests/v1/multiprocess/test_non_cuda_data_transfer.py @@ -357,16 +357,18 @@ def test_server_register_and_find_non_cuda_context_layout( """Ensure non-CUDA registration stores metadata and lookup finds layout.""" # First Party from lmcache.v1.multiprocess.custom_types import RegisterNonGpuContextPayload - from lmcache.v1.multiprocess.server import MPCacheEngine + from lmcache.v1.multiprocess.engine_context import MPCacheEngineContext + from lmcache.v1.multiprocess.modules.non_gpu_transfer import NonGPUTransferModule with ( - patch("lmcache.v1.multiprocess.server.StorageManager"), - patch("lmcache.v1.multiprocess.server.TokenHasher"), - patch("lmcache.v1.multiprocess.server.SessionManager"), - patch("lmcache.v1.multiprocess.server.get_event_bus"), + patch("lmcache.v1.multiprocess.engine_context.StorageManager"), + patch("lmcache.v1.multiprocess.engine_context.TokenHasher"), + patch("lmcache.v1.multiprocess.engine_context.SessionManager"), + patch("lmcache.v1.multiprocess.engine_context.get_event_bus"), ): - engine = MPCacheEngine(storage_manager_config=MagicMock(), chunk_size=16) - engine.register_kv_cache_non_gpu_context( + ctx = MPCacheEngineContext(storage_manager_config=MagicMock(), chunk_size=16) + module = NonGPUTransferModule(ctx) + module.register_kv_cache_non_gpu_context( RegisterNonGpuContextPayload( instance_id=1, model_name="m", @@ -379,7 +381,7 @@ def test_server_register_and_find_non_cuda_context_layout( ) ) - layout = engine._find_layout_desc("m", 1) + layout = ctx.layout_desc_registry.find("m", 1) assert layout is not None assert layout.shapes[0] == torch.Size([2, 2, 16, 16]) @@ -391,7 +393,8 @@ def test_server_store_and_retrieve_cpu_chunks(stub_native_storage_ops: Any) -> N IPCCacheEngineKey, RegisterNonGpuContextPayload, ) - from lmcache.v1.multiprocess.server import MPCacheEngine + from lmcache.v1.multiprocess.engine_context import MPCacheEngineContext + from lmcache.v1.multiprocess.modules.non_gpu_transfer import NonGPUTransferModule mock_storage = MagicMock() target_tensor = torch.zeros(2, 2, 8, 16) @@ -408,21 +411,22 @@ def _read_prefetched_results(_keys: Any) -> Any: mock_session.get_hashes.return_value = [b"h"] with ( patch( - "lmcache.v1.multiprocess.server.StorageManager", + "lmcache.v1.multiprocess.engine_context.StorageManager", return_value=mock_storage, ), - patch("lmcache.v1.multiprocess.server.TokenHasher"), - patch("lmcache.v1.multiprocess.server.SessionManager") as session_cls, - patch("lmcache.v1.multiprocess.server.get_event_bus"), + patch("lmcache.v1.multiprocess.engine_context.TokenHasher"), + patch("lmcache.v1.multiprocess.engine_context.SessionManager") as session_cls, + patch("lmcache.v1.multiprocess.engine_context.get_event_bus"), patch( - "lmcache.v1.multiprocess.server.ipc_key_to_object_keys", + "lmcache.v1.multiprocess.engine_context.ipc_key_to_object_keys", return_value=["obj"], ), ): session_cls.return_value.get_or_create.return_value = mock_session - engine = MPCacheEngine(storage_manager_config=MagicMock(), chunk_size=8) + ctx = MPCacheEngineContext(storage_manager_config=MagicMock(), chunk_size=8) - engine.register_kv_cache_non_gpu_context( + module = NonGPUTransferModule(ctx) + module.register_kv_cache_non_gpu_context( RegisterNonGpuContextPayload( instance_id=2, model_name="m", @@ -445,11 +449,11 @@ def _read_prefetched_results(_keys: Any) -> Any: request_id="req", ) with patch( - "lmcache.v1.multiprocess.server.ipc_key_to_object_keys", + "lmcache.v1.multiprocess.engine_context.ipc_key_to_object_keys", return_value=["obj"], ): - store_ok = engine.commit_store(key, 2, pickle.dumps([payload])) - response = engine.prepare_retrieve(key, 2) + store_ok = module.commit_store(key, 2, pickle.dumps([payload])) + response = module.prepare_retrieve(key, 2) success = response.success cpu_data = response.data assert isinstance(store_ok, bool) diff --git a/tests/v1/multiprocess/test_query_lookup_hits.py b/tests/v1/multiprocess/test_query_lookup_hits.py index 53a1d85cbc2..387f1b5061b 100644 --- a/tests/v1/multiprocess/test_query_lookup_hits.py +++ b/tests/v1/multiprocess/test_query_lookup_hits.py @@ -6,11 +6,11 @@ # Standard from unittest.mock import MagicMock -import threading import time # First Party from lmcache.v1.distributed.storage_manager import PrefetchHandle +from lmcache.v1.multiprocess.modules.lookup import LookupModule, _PrefetchJob from lmcache.v1.multiprocess.protocol import ( RequestType, get_handler_type, @@ -18,7 +18,6 @@ get_response_class, ) from lmcache.v1.multiprocess.protocols.base import HandlerType -from lmcache.v1.multiprocess.server import MPCacheEngine, _PrefetchJob # Test helpers from tests.v1.multiprocess.test_mq import ( @@ -107,16 +106,17 @@ def test_mq_query_prefetch_lookup_hits_none_response(): # ============================================================================ -def _make_engine_with_job( +def _make_module_with_job( world_size: int, storage_return: int | None -) -> tuple[MagicMock, str]: - """Create a mock MPCacheEngine with a single prefetch job. +) -> tuple[LookupModule, str]: + """Create a LookupModule with a mock context and a single prefetch job. Returns: - (engine_mock, request_id) + (module, request_id) """ - engine = MagicMock() - engine._prefetch_job_lock = threading.Lock() + ctx = MagicMock() + ctx.token_hasher.chunk_size = 256 + module = LookupModule(ctx) handle = PrefetchHandle( prefetch_request_id=0, @@ -132,50 +132,50 @@ def _make_engine_with_job( request_id=request_id, requested_tokens=0, ) - engine._prefetch_jobs = {request_id: job} - engine.storage_manager.query_prefetch_lookup_hits.return_value = storage_return + module._prefetch_jobs[request_id] = job + ctx.storage_manager.query_prefetch_lookup_hits.return_value = storage_return - return engine, request_id + return module, request_id def test_server_lookup_hits_returns_count(): """query_prefetch_lookup_hits returns chunk count when lookup is done.""" - engine, request_id = _make_engine_with_job(world_size=1, storage_return=5) + module, request_id = _make_module_with_job(world_size=1, storage_return=5) - result = MPCacheEngine.query_prefetch_lookup_hits(engine, request_id) + result = module.query_prefetch_lookup_hits(request_id) assert result == 5 - engine.storage_manager.query_prefetch_lookup_hits.assert_called_once() + module.context.storage_manager.query_prefetch_lookup_hits.assert_called_once() def test_server_lookup_hits_divides_by_world_size(): """Result should be divided by world_size for tensor parallelism.""" - engine, request_id = _make_engine_with_job(world_size=2, storage_return=10) + module, request_id = _make_module_with_job(world_size=2, storage_return=10) - result = MPCacheEngine.query_prefetch_lookup_hits(engine, request_id) + result = module.query_prefetch_lookup_hits(request_id) assert result == 5 # 10 // 2 def test_server_lookup_hits_returns_none_when_in_progress(): """Returns None when storage manager lookup is still in progress.""" - engine, request_id = _make_engine_with_job(world_size=1, storage_return=None) + module, request_id = _make_module_with_job(world_size=1, storage_return=None) - result = MPCacheEngine.query_prefetch_lookup_hits(engine, request_id) + result = module.query_prefetch_lookup_hits(request_id) assert result is None def test_server_lookup_hits_returns_zero_for_invalid_request(): """Returns 0 for a request_id that doesn't exist (prevents infinite spin).""" - engine = MagicMock() - engine._prefetch_job_lock = threading.Lock() - engine._prefetch_jobs = {} + ctx = MagicMock() + ctx.token_hasher.chunk_size = 256 + module = LookupModule(ctx) - result = MPCacheEngine.query_prefetch_lookup_hits(engine, "nonexistent-req") + result = module.query_prefetch_lookup_hits("nonexistent-req") assert result == 0 - engine.storage_manager.query_prefetch_lookup_hits.assert_not_called() + ctx.storage_manager.query_prefetch_lookup_hits.assert_not_called() def test_server_lookup_hits_returns_zero_after_prefetch_consumed(): @@ -183,28 +183,27 @@ def test_server_lookup_hits_returns_zero_after_prefetch_consumed(): This prevents the caller from spinning forever on a completed request. """ - engine, request_id = _make_engine_with_job(world_size=1, storage_return=5) + module, request_id = _make_module_with_job(world_size=1, storage_return=5) # Simulate query_prefetch_status consuming the job - del engine._prefetch_jobs[request_id] + module._prefetch_jobs.pop(request_id) - result = MPCacheEngine.query_prefetch_lookup_hits(engine, request_id) + result = module.query_prefetch_lookup_hits(request_id) assert result == 0 def test_server_lookup_hits_zero_count(): """Returns 0 when no keys matched (not None).""" - engine, request_id = _make_engine_with_job(world_size=1, storage_return=0) + module, request_id = _make_module_with_job(world_size=1, storage_return=0) - result = MPCacheEngine.query_prefetch_lookup_hits(engine, request_id) + result = module.query_prefetch_lookup_hits(request_id) assert result == 0 assert result is not None def test_server_handler_registered(): - """MPCacheEngine should have a query_prefetch_lookup_hits method.""" - engine = MPCacheEngine.__new__(MPCacheEngine) - assert hasattr(engine, "query_prefetch_lookup_hits") - assert callable(engine.query_prefetch_lookup_hits) + """LookupModule should have a query_prefetch_lookup_hits method.""" + assert hasattr(LookupModule, "query_prefetch_lookup_hits") + assert callable(LookupModule.query_prefetch_lookup_hits) From 3dd3668181462e522d1229a491ad8446f89fecc5 Mon Sep 17 00:00:00 2001 From: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Date: Wed, 27 May 2026 03:05:14 +0800 Subject: [PATCH 66/69] fix: support HND formats in MP KV transfer (#3282) Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> --- csrc/mp_mem_kernels.cu | 18 +++++++++++++- tests/v1/test_mp_mem_kernels.py | 42 +++++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/csrc/mp_mem_kernels.cu b/csrc/mp_mem_kernels.cu index ccdc9d62bbc..8d861e87385 100644 --- a/csrc/mp_mem_kernels.cu +++ b/csrc/mp_mem_kernels.cu @@ -37,10 +37,18 @@ __device__ inline size_t calculate_engine_global_offset( // Normal: L tensors [2, NB, BS, NH, HS] return engine_block_idx * scalars_per_block + k_or_v * shape_desc.nb * scalars_per_block; + } else if constexpr (format == GPUKVFormat::NL_X_TWO_NB_NH_BS_HS) { + // Normal HND: L tensors [2, NB, NH, BS, HS] + return engine_block_idx * scalars_per_block + + k_or_v * shape_desc.nb * scalars_per_block; } else if constexpr (format == GPUKVFormat::NL_X_NB_TWO_BS_NH_HS) { // Flash Infer: L tensors [NB, 2, BS, NH, HS] return engine_block_idx * shape_desc.kv_size * scalars_per_block + k_or_v * scalars_per_block; + } else if constexpr (format == GPUKVFormat::NL_X_NB_TWO_NH_BS_HS) { + // Flash Infer HND: L tensors [NB, 2, NH, BS, HS] + return engine_block_idx * shape_desc.kv_size * scalars_per_block + + k_or_v * scalars_per_block; } else if constexpr (format == GPUKVFormat::NL_X_NB_BS_HS) { // MLA: L tensors [NB, BS, HS] return engine_block_idx * scalars_per_block; @@ -70,7 +78,9 @@ __device__ inline size_t calculate_engine_local_offset( const PageBufferShapeDesc shape_desc) { size_t scalars_per_head = shape_desc.scalars_per_head(); size_t scalars_per_token = shape_desc.scalars_per_token(); - if constexpr (format == GPUKVFormat::NB_NL_TWO_NH_BS_HS) { + if constexpr (format == GPUKVFormat::NB_NL_TWO_NH_BS_HS || + format == GPUKVFormat::NL_X_TWO_NB_NH_BS_HS || + format == GPUKVFormat::NL_X_NB_TWO_NH_BS_HS) { // HND: [NH, BS, HS] — heads are outermost within a block size_t scalars_per_head_block = shape_desc.bs * scalars_per_head; // BS * HS @@ -248,9 +258,15 @@ __global__ void multi_layer_block_transfer_kernel( case GPUKVFormat::NL_X_TWO_NB_BS_NH_HS: \ LAUNCH_KERNEL(DIRECTION, GPUKVFormat::NL_X_TWO_NB_BS_NH_HS); \ break; \ + case GPUKVFormat::NL_X_TWO_NB_NH_BS_HS: \ + LAUNCH_KERNEL(DIRECTION, GPUKVFormat::NL_X_TWO_NB_NH_BS_HS); \ + break; \ case GPUKVFormat::NL_X_NB_TWO_BS_NH_HS: \ LAUNCH_KERNEL(DIRECTION, GPUKVFormat::NL_X_NB_TWO_BS_NH_HS); \ break; \ + case GPUKVFormat::NL_X_NB_TWO_NH_BS_HS: \ + LAUNCH_KERNEL(DIRECTION, GPUKVFormat::NL_X_NB_TWO_NH_BS_HS); \ + break; \ case GPUKVFormat::NL_X_NB_BS_HS: \ LAUNCH_KERNEL(DIRECTION, GPUKVFormat::NL_X_NB_BS_HS); \ break; \ diff --git a/tests/v1/test_mp_mem_kernels.py b/tests/v1/test_mp_mem_kernels.py index 37a6db72536..6e4320f03ef 100644 --- a/tests/v1/test_mp_mem_kernels.py +++ b/tests/v1/test_mp_mem_kernels.py @@ -49,6 +49,8 @@ def _create_zero_tensor( FMT_MLA = lmc_ops.GPUKVFormat.NL_X_NB_BS_HS FMT_SGLANG_MHA = lmc_ops.GPUKVFormat.TWO_X_NL_X_NBBS_NH_HS FMT_SGLANG_MLA = lmc_ops.GPUKVFormat.NL_X_NBBS_ONE_HS +FMT_NORMAL_HND = lmc_ops.GPUKVFormat.NL_X_TWO_NB_NH_BS_HS +FMT_FLASH_INFER_HND = lmc_ops.GPUKVFormat.NL_X_NB_TWO_NH_BS_HS # Format parameters: (gpu_kv_format, num_layers, num_heads, head_size, is_mla) # Use small layer counts to keep GPU memory usage low in CI @@ -59,6 +61,8 @@ def _create_zero_tensor( (FMT_MLA, 4, 1, 576, True), (FMT_SGLANG_MHA, 4, 8, 128, False), (FMT_SGLANG_MLA, 4, 1, 576, True), + (FMT_NORMAL_HND, 4, 8, 128, False), + (FMT_FLASH_INFER_HND, 4, 8, 128, False), ] @@ -76,12 +80,18 @@ def create_vllm_tensors( if gpu_kv_format == FMT_NORMAL: shape = [2, nb, bs, nh, hs] return [_create_random_tensor(shape, dtype, device) for _ in range(nl)] + elif gpu_kv_format == FMT_NORMAL_HND: + shape = [2, nb, nh, bs, hs] + return [_create_random_tensor(shape, dtype, device) for _ in range(nl)] elif gpu_kv_format == FMT_CROSS_LAYER: shape = [nb, nl, 2, bs, nh, hs] return [_create_random_tensor(shape, dtype, device)] elif gpu_kv_format == FMT_FLASH_INFER: shape = [nb, 2, bs, nh, hs] return [_create_random_tensor(shape, dtype, device) for _ in range(nl)] + elif gpu_kv_format == FMT_FLASH_INFER_HND: + shape = [nb, 2, nh, bs, hs] + return [_create_random_tensor(shape, dtype, device) for _ in range(nl)] elif gpu_kv_format == FMT_MLA: shape = [nb, bs, hs] return [_create_random_tensor(shape, dtype, device) for _ in range(nl)] @@ -108,12 +118,18 @@ def create_zero_vllm_tensors( if gpu_kv_format == FMT_NORMAL: shape = [2, nb, bs, nh, hs] return [_create_zero_tensor(shape, dtype, device) for _ in range(nl)] + elif gpu_kv_format == FMT_NORMAL_HND: + shape = [2, nb, nh, bs, hs] + return [_create_zero_tensor(shape, dtype, device) for _ in range(nl)] elif gpu_kv_format == FMT_CROSS_LAYER: shape = [nb, nl, 2, bs, nh, hs] return [_create_zero_tensor(shape, dtype, device)] elif gpu_kv_format == FMT_FLASH_INFER: shape = [nb, 2, bs, nh, hs] return [_create_zero_tensor(shape, dtype, device) for _ in range(nl)] + elif gpu_kv_format == FMT_FLASH_INFER_HND: + shape = [nb, 2, nh, bs, hs] + return [_create_zero_tensor(shape, dtype, device) for _ in range(nl)] elif gpu_kv_format == FMT_MLA: shape = [nb, bs, hs] return [_create_zero_tensor(shape, dtype, device) for _ in range(nl)] @@ -162,10 +178,14 @@ def get_block_data( for layer_idx in range(nl): if gpu_kv_format == FMT_NORMAL: results.append(vllm_tensors[layer_idx][:, block_idx, :, :, :].clone()) + elif gpu_kv_format == FMT_NORMAL_HND: + results.append(vllm_tensors[layer_idx][:, block_idx, :, :, :].clone()) elif gpu_kv_format == FMT_CROSS_LAYER: results.append(vllm_tensors[0][block_idx, layer_idx, :, :, :, :].clone()) elif gpu_kv_format == FMT_FLASH_INFER: results.append(vllm_tensors[layer_idx][block_idx, :, :, :, :].clone()) + elif gpu_kv_format == FMT_FLASH_INFER_HND: + results.append(vllm_tensors[layer_idx][block_idx, :, :, :, :].clone()) elif gpu_kv_format == FMT_MLA: results.append(vllm_tensors[layer_idx][block_idx, :, :].clone()) elif gpu_kv_format == FMT_SGLANG_MHA: @@ -243,7 +263,16 @@ def call_block_kernel( @pytest.mark.parametrize( "gpu_kv_format,nl,nh,hs,is_mla", FORMAT_PARAMS, - ids=["normal", "cross_layer", "flash_infer", "mla", "sglang_mha", "sglang_mla"], + ids=[ + "normal", + "cross_layer", + "flash_infer", + "mla", + "sglang_mha", + "sglang_mla", + "normal_hnd", + "flash_infer_hnd", + ], ) @pytest.mark.parametrize( "dtype", [torch.bfloat16, torch.float8_e4m3fn], ids=["bf16", "fp8"] @@ -333,7 +362,16 @@ def test_block_transfer_roundtrip(gpu_kv_format, nl, nh, hs, is_mla, dtype, mem_ @pytest.mark.parametrize( "gpu_kv_format,nl,nh,hs,is_mla", FORMAT_PARAMS, - ids=["normal", "cross_layer", "flash_infer", "mla", "sglang_mha", "sglang_mla"], + ids=[ + "normal", + "cross_layer", + "flash_infer", + "mla", + "sglang_mha", + "sglang_mla", + "normal_hnd", + "flash_infer_hnd", + ], ) @pytest.mark.parametrize("dtype", [torch.bfloat16], ids=["bf16"]) def test_block_transfer_skip_prefix(gpu_kv_format, nl, nh, hs, is_mla, dtype): From 1966340e5fc7e45112f83f0796804b962daa066d Mon Sep 17 00:00:00 2001 From: Se7en Date: Wed, 27 May 2026 03:10:24 +0800 Subject: [PATCH 67/69] [Fix][Observability] PrometheusLogger instance already created with different metadata (#3147) Signed-off-by: cr7258 --- lmcache/integration/base_service_factory.py | 8 +- lmcache/integration/vllm/vllm_v1_adapter.py | 12 +- lmcache/observability.py | 258 +++++++++++------- lmcache/v1/cache_engine.py | 2 +- .../chunk_statistics_lookup_client.py | 31 ++- lmcache/v1/lookup_client/factory.py | 1 + lmcache/v1/pin_monitor.py | 34 ++- .../storage_backend/batched_message_sender.py | 18 +- .../v1/storage_backend/local_cpu_backend.py | 23 +- lmcache/v1/storage_backend/remote_backend.py | 26 +- lmcache/v1/storage_backend/storage_manager.py | 11 +- tests/test_observability.py | 88 +++++- 12 files changed, 346 insertions(+), 166 deletions(-) diff --git a/lmcache/integration/base_service_factory.py b/lmcache/integration/base_service_factory.py index b8c88567d0c..8fcb050e9f7 100644 --- a/lmcache/integration/base_service_factory.py +++ b/lmcache/integration/base_service_factory.py @@ -125,8 +125,12 @@ def _create_health_monitor( health_monitor.start() logger.info("Health monitor initialized and started") - prometheus_logger = PrometheusLogger.GetInstanceOrNone() - if prometheus_logger is not None: + metadata = lmcache_manager.lmcache_engine_metadata + if metadata is not None: + prometheus_logger = PrometheusLogger.GetOrCreate( + metadata, + config=config, + ) prometheus_logger.lmcache_is_healthy.set_function( lambda: 1 if lmcache_manager.is_healthy() else 0 ) diff --git a/lmcache/integration/vllm/vllm_v1_adapter.py b/lmcache/integration/vllm/vllm_v1_adapter.py index 4fbb6a65fa7..27c29c8d089 100644 --- a/lmcache/integration/vllm/vllm_v1_adapter.py +++ b/lmcache/integration/vllm/vllm_v1_adapter.py @@ -618,15 +618,19 @@ def lookup_server(self): """Get the lookup server from manager.""" return self._manager.lookup_server - def _setup_metrics(self): + def _setup_metrics(self) -> None: """Setup metrics for monitoring data structures in the connector.""" - prometheus_logger = PrometheusLogger.GetInstanceOrNone() - if prometheus_logger is None: + metadata = self._manager.lmcache_engine_metadata + if metadata is None: logger.warning( - "PrometheusLogger is not initialized, " + "LMCache metadata is not initialized, " "connector metrics will not be collected" ) return + prometheus_logger = PrometheusLogger.GetOrCreate( + metadata, + config=self.config, + ) # Set up metrics for scheduler-specific and general data structures metrics_map = { diff --git a/lmcache/observability.py b/lmcache/observability.py index 8a9a3a1545d..4c422ed19cc 100644 --- a/lmcache/observability.py +++ b/lmcache/observability.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Standard from contextlib import contextmanager +from copy import copy from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, @@ -848,6 +849,11 @@ def unregister_all_metrics(): class PrometheusLogger: + lmcache_is_healthy: prometheus_client.Gauge + periodic_threads_total_count: prometheus_client.Gauge + periodic_threads_running_count: prometheus_client.Gauge + periodic_threads_active_count: prometheus_client.Gauge + _gauge_cls = prometheus_client.Gauge _counter_cls = prometheus_client.Counter _histogram_cls = prometheus_client.Histogram @@ -900,17 +906,21 @@ def _create_histogram( self._histograms.append(histogram) return histogram + @staticmethod + def _ensure_multiprocess_dir() -> None: + multiprocess_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR") + if not multiprocess_dir: + multiprocess_dir = "/tmp/lmcache_prometheus" + os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiprocess_dir + os.makedirs(multiprocess_dir, exist_ok=True) + def __init__( self, metadata: LMCacheMetadata, config: Optional["LMCacheEngineConfig"] = None, ): # Ensure PROMETHEUS_MULTIPROC_DIR is set before any metric registration - if "PROMETHEUS_MULTIPROC_DIR" not in os.environ: - default_dir = "/tmp/lmcache_prometheus" - os.environ["PROMETHEUS_MULTIPROC_DIR"] = default_dir - if not os.path.exists(default_dir): - os.makedirs(default_dir, exist_ok=True) + PrometheusLogger._ensure_multiprocess_dir() self.metadata = metadata self.config = config @@ -1454,125 +1464,153 @@ def __init__( ) self._dynamic_metrics(labelnames) - def _dynamic_metrics(self, labelnames): + def _create_dynamic_gauge( + self, + name: str, + documentation: str, + labelnames: List[str], + multiprocess_mode: str, + ) -> None: + metric_attr = name.removeprefix("lmcache:") + if metric_attr == name or not metric_attr.isidentifier(): + raise ValueError(f"Invalid dynamic metric name: {name}") + + gauge = self._gauge_cls( + name=name, + documentation=documentation, + labelnames=labelnames, + multiprocess_mode=multiprocess_mode, + ) + # Store the shared collector separately from the labeled child. + # Shallow-copied label views reuse the collector, then call labels() + # with their own labels so set_function callbacks publish to that view. + self._dynamic_gauge_collectors[metric_attr] = gauge + setattr(self, metric_attr, gauge.labels(**self.labels)) + + def _bind_dynamic_metric_children(self) -> None: + PrometheusLogger._ensure_multiprocess_dir() + for metric_attr, gauge in self._dynamic_gauge_collectors.items(): + setattr(self, metric_attr, gauge.labels(**self.labels)) + + def _dynamic_metrics(self, labelnames: List[str]) -> None: """ Dynamically get value by lambda function while capture """ - self.local_cpu_hot_cache_count = self._gauge_cls( + self._dynamic_gauge_collectors: Dict[str, prometheus_client.Gauge] = {} + self._create_dynamic_gauge( name="lmcache:local_cpu_hot_cache_count", documentation="The size of the hot_cache", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.local_cpu_keys_in_request_count = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:local_cpu_keys_in_request_count", documentation="The size of the keys_in_request", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.kv_msg_queue_size = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:kv_msg_queue_size", documentation="The size of the KV message queue in BatchedMessageSender", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.remote_put_task_num = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:remote_put_task_num", documentation="The number of remote put tasks", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.pin_monitor_pinned_objects_count = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:pin_monitor_pinned_objects_count", documentation="The number of pinned objects in PinMonitor", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.lmcache_is_healthy = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:lmcache_is_healthy", documentation="The health status of LMCache (1=healthy, 0=unhealthy)", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.get_blocking_failed_count = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:get_blocking_failed_count", documentation="The number of get blocking failed", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.put_failed_count = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:put_failed_count", documentation="The number of put failed", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) + ) event_statuses = ["ongoing", "done", "not_found"] for status in event_statuses: metric_name = f"storage_events_{status}_count" - gauge = self._gauge_cls( + self._create_dynamic_gauge( name=f"lmcache:{metric_name}", documentation=f"The number of {status.replace('_', ' ')} events", labelnames=labelnames, multiprocess_mode="sum", - ).labels(**self.labels) - setattr(self, metric_name, gauge) + ) # Chunk statistics metrics (dynamic) - self.chunk_statistics_enabled = self._gauge_cls( + self._create_dynamic_gauge( name="lmcache:chunk_statistics_enabled", documentation="Whether chunk statistics collection is enabled", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.chunk_statistics_total_requests = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:chunk_statistics_total_requests", documentation="Total number of requests processed by chunk statistics", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.chunk_statistics_total_chunks = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:chunk_statistics_total_chunks", documentation="Total number of chunks processed", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.chunk_statistics_unique_chunks = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:chunk_statistics_unique_chunks", documentation="Number of unique chunks (estimated)", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.chunk_statistics_reuse_rate = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:chunk_statistics_reuse_rate", documentation="Chunk reuse rate (0.0 to 1.0)", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.chunk_statistics_bloom_filter_size_mb = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:chunk_statistics_bloom_filter_size_mb", documentation="Bloom Filter memory usage in MB", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.chunk_statistics_bloom_filter_fill_rate = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:chunk_statistics_bloom_filter_fill_rate", documentation="Bloom Filter fill rate (0.0 to 1.0)", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.chunk_statistics_file_count = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:chunk_statistics_file_count", documentation="Number of files created for file_hash strategy", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.chunk_statistics_current_file_size = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:chunk_statistics_current_file_size", documentation="Current file size in bytes for file_hash strategy", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) + ) # Connector metrics connector_metrics = [ @@ -1586,59 +1624,55 @@ def _dynamic_metrics(self, labelnames): ] for metric_name in connector_metrics: - gauge = self._gauge_cls( + self._create_dynamic_gauge( name=f"lmcache:{metric_name}", documentation=f"The count of {metric_name.replace('_', ' ')}", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - setattr(self, metric_name, gauge) + ) # PeriodicThread metrics - self.periodic_threads_total_count = self._gauge_cls( + self._create_dynamic_gauge( name="lmcache:periodic_threads_total_count", documentation="Total number of registered periodic threads", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.periodic_threads_running_count = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:periodic_threads_running_count", documentation="Number of running periodic threads", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - self.periodic_threads_active_count = self._gauge_cls( + ) + self._create_dynamic_gauge( name="lmcache:periodic_threads_active_count", documentation="Number of active periodic threads (recently executed)", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) + ) # Per-level metrics for periodic threads for level_name in ["critical", "high", "medium", "low"]: - total_gauge = self._gauge_cls( + self._create_dynamic_gauge( name=f"lmcache:periodic_threads_{level_name}_total", documentation=f"Total number of {level_name} level periodic threads", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - setattr(self, f"periodic_threads_{level_name}_total", total_gauge) + ) - running_gauge = self._gauge_cls( + self._create_dynamic_gauge( name=f"lmcache:periodic_threads_{level_name}_running", documentation=f"Number of running {level_name} level periodic threads", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - setattr(self, f"periodic_threads_{level_name}_running", running_gauge) + ) - active_gauge = self._gauge_cls( + self._create_dynamic_gauge( name=f"lmcache:periodic_threads_{level_name}_active", documentation=f"Number of active {level_name} level periodic threads", labelnames=labelnames, multiprocess_mode="livemostrecent", - ).labels(**self.labels) - setattr(self, f"periodic_threads_{level_name}_active", active_gauge) + ) def _log_gauge(self, gauge, data: Union[int, float]) -> None: # Convenience function for logging to gauge. @@ -1810,50 +1844,36 @@ def log_prometheus(self, stats: LMCacheStats): self.gauge_pinned_memory_objs_count, stats.pinned_memory_objs_count ) - @staticmethod - def _metadata_to_labels(metadata: LMCacheMetadata): - labels = { - "model_name": metadata.model_name, - "worker_id": metadata.worker_id, - "role": metadata.role, - } - if metadata.served_model_name: - labels["served_model_name"] = metadata.served_model_name - return labels - - _instance = None + _instances: Dict[tuple[tuple[str, str], ...], "PrometheusLogger"] = {} @staticmethod + @thread_safe def GetOrCreate( metadata: LMCacheMetadata, config: Optional["LMCacheEngineConfig"] = None, ) -> "PrometheusLogger": - if PrometheusLogger._instance is None: - PrometheusLogger._instance = PrometheusLogger(metadata, config=config) - # assert PrometheusLogger._instance.metadata == metadata, \ - # "PrometheusLogger instance already created with different metadata" - if PrometheusLogger._instance.metadata != metadata: - logger.error( - "PrometheusLogger instance already created with " - "different metadata. This should not happen except " - "in test" + metadata_key = PrometheusLogger._metadata_to_key(metadata) + if metadata_key in PrometheusLogger._instances: + return PrometheusLogger._instances[metadata_key] + + base_logger = PrometheusLogger._get_base_logger() + if base_logger is None: + logger_instance = PrometheusLogger(metadata, config=config) + else: + logger_instance = PrometheusLogger._create_label_view( + base_logger, + metadata, ) - return PrometheusLogger._instance - @staticmethod - def GetInstance() -> "PrometheusLogger": - assert PrometheusLogger._instance is not None, ( - "PrometheusLogger instance not created yet" - ) - return PrometheusLogger._instance + PrometheusLogger._instances[metadata_key] = logger_instance + return logger_instance @staticmethod - def GetInstanceOrNone() -> Optional["PrometheusLogger"]: + def _get_base_logger() -> Optional["PrometheusLogger"]: """ - Returns the singleton instance of PrometheusLogger if it exists, - otherwise returns None. + Return an existing logger to reuse registered Prometheus collectors. """ - return PrometheusLogger._instance + return next(iter(PrometheusLogger._instances.values()), None) @thread_safe def reset_counters(self) -> None: @@ -1861,10 +1881,12 @@ def reset_counters(self) -> None: Reset all Prometheus Counter metrics by calling clear(). After clearing, re-initialize with labels so metrics remain visible. """ + label_views = self._reset_label_views() for counter in self._counters: counter.clear() - # Re-initialize with labels to make metric visible again - counter.labels(**self.labels) + # Re-initialize all known label views to keep each series visible. + for label_view in label_views: + counter.labels(**label_view.labels) @thread_safe def reset_histograms(self) -> None: @@ -1872,10 +1894,50 @@ def reset_histograms(self) -> None: Reset all Prometheus Histogram metrics by calling clear(). After clearing, re-initialize with labels so metrics remain visible. """ + label_views = self._reset_label_views() for histogram in self._histograms: histogram.clear() - # Re-initialize with labels to make metric visible again - histogram.labels(**self.labels) + # Re-initialize all known label views to keep each series visible. + for label_view in label_views: + histogram.labels(**label_view.labels) + + @staticmethod + def _metadata_to_labels(metadata: LMCacheMetadata) -> Dict[str, Any]: + labels = { + "model_name": metadata.model_name, + "worker_id": metadata.worker_id, + "role": metadata.role, + "served_model_name": metadata.served_model_name or "", + } + return labels + + @staticmethod + def _metadata_to_key(metadata: LMCacheMetadata) -> tuple[tuple[str, str], ...]: + labels = PrometheusLogger._metadata_to_labels(metadata) + return tuple(sorted((name, str(value)) for name, value in labels.items())) + + @staticmethod + def _create_label_view( + base_logger: "PrometheusLogger", + metadata: LMCacheMetadata, + ) -> "PrometheusLogger": + """Reuse registered collectors with a different metadata/label view.""" + label_view = copy(base_logger) + label_view.metadata = metadata + label_view.labels = PrometheusLogger._metadata_to_labels(metadata) + label_view._bind_dynamic_metric_children() + return label_view + + def _reset_label_views(self) -> List["PrometheusLogger"]: + """Return label views whose children must be restored after clear(). + + Counter.clear() and Histogram.clear() remove every child from the + shared collector, so reset must recreate children for all known labels. + """ + views = list(PrometheusLogger._instances.values()) + if self not in views: + views.append(self) + return views def reset_observability_metrics() -> None: @@ -1883,7 +1945,7 @@ def reset_observability_metrics() -> None: Reset observability metrics to their initial state. """ - prometheus_logger = PrometheusLogger.GetInstanceOrNone() + prometheus_logger = PrometheusLogger._get_base_logger() if prometheus_logger is not None: prometheus_logger.reset_counters() prometheus_logger.reset_histograms() diff --git a/lmcache/v1/cache_engine.py b/lmcache/v1/cache_engine.py index 5e90100a326..b0d988ba529 100644 --- a/lmcache/v1/cache_engine.py +++ b/lmcache/v1/cache_engine.py @@ -209,7 +209,7 @@ def __init__( InitializeUsageContext(config, metadata) self.stats_monitor = LMCStatsMonitor.GetOrCreate() # Initialize PinMonitor singleton with config - PinMonitor.GetOrCreate(config) + PinMonitor.GetOrCreate(config, metadata) self.post_inited = False diff --git a/lmcache/v1/lookup_client/chunk_statistics_lookup_client.py b/lmcache/v1/lookup_client/chunk_statistics_lookup_client.py index 4df7ea00dda..4879e4556fc 100644 --- a/lmcache/v1/lookup_client/chunk_statistics_lookup_client.py +++ b/lmcache/v1/lookup_client/chunk_statistics_lookup_client.py @@ -18,6 +18,7 @@ RecordStrategy, create_record_strategy, ) +from lmcache.v1.metadata import LMCacheMetadata logger = init_logger(__name__) @@ -29,8 +30,11 @@ def __init__( self, actual_lookup_client: LookupClientInterface, config: LMCacheEngineConfig, - ): + metadata: Optional[LMCacheMetadata] = None, + ) -> None: self.actual_lookup_client = actual_lookup_client + self.config = config + self.metadata = metadata self.lock = threading.RLock() self.chunk_size = config.chunk_size self.enabled = False @@ -188,13 +192,18 @@ def _trigger_stop(self, reason: str) -> None: if self.enabled: self.stop_statistics() - def _setup_metrics(self): - prometheus_logger = PrometheusLogger.GetInstanceOrNone() - if prometheus_logger is not None: - prometheus_logger.chunk_statistics_enabled.set_function( - lambda: 1.0 if self.enabled else 0.0 - ) - prometheus_logger.chunk_statistics_total_requests.set_function( - lambda: len(self.request_seen) - ) - self.recorder.strategy.setup_metrics(prometheus_logger) + def _setup_metrics(self) -> None: + if self.metadata is None: + return + + prometheus_logger = PrometheusLogger.GetOrCreate( + self.metadata, + config=self.config, + ) + prometheus_logger.chunk_statistics_enabled.set_function( + lambda: 1.0 if self.enabled else 0.0 + ) + prometheus_logger.chunk_statistics_total_requests.set_function( + lambda: len(self.request_seen) + ) + self.recorder.strategy.setup_metrics(prometheus_logger) diff --git a/lmcache/v1/lookup_client/factory.py b/lmcache/v1/lookup_client/factory.py index 3a2e8d886bf..81e35a7b9a6 100644 --- a/lmcache/v1/lookup_client/factory.py +++ b/lmcache/v1/lookup_client/factory.py @@ -94,6 +94,7 @@ def create_lookup_client( client = ChunkStatisticsLookupClient( client, config, + metadata, ) return client diff --git a/lmcache/v1/pin_monitor.py b/lmcache/v1/pin_monitor.py index 60a70ffb0f3..863f35cb9c4 100644 --- a/lmcache/v1/pin_monitor.py +++ b/lmcache/v1/pin_monitor.py @@ -19,6 +19,7 @@ # First Party from lmcache.v1.config import LMCacheEngineConfig from lmcache.v1.memory_management import MemoryObj + from lmcache.v1.metadata import LMCacheMetadata logger = init_logger(__name__) @@ -35,7 +36,7 @@ class PinMonitor(PeriodicThread): _instance = None _instance_lock = threading.Lock() # Class-level lock for singleton pattern - def __init__(self, config: "LMCacheEngineConfig"): + def __init__(self, config: "LMCacheEngineConfig") -> None: # Initialize PeriodicThread base class super().__init__( name="PinMonitor-thread", @@ -59,12 +60,17 @@ def __init__(self, config: "LMCacheEngineConfig"): self.start_monitoring() @staticmethod - def GetOrCreate(config: Optional["LMCacheEngineConfig"] = None) -> "PinMonitor": + def GetOrCreate( + config: Optional["LMCacheEngineConfig"] = None, + metadata: Optional["LMCacheMetadata"] = None, + ) -> "PinMonitor": """Get or create the singleton instance. Args: config: Required for first-time initialization. Optional for subsequent calls. + metadata: Metadata for the label view that should publish + PinMonitor metrics. Raises: ValueError: If config is None when creating the instance @@ -75,7 +81,12 @@ def GetOrCreate(config: Optional["LMCacheEngineConfig"] = None) -> "PinMonitor": if PinMonitor._instance is None: assert config is not None, "config is required" PinMonitor._instance = PinMonitor(config) - return PinMonitor._instance + instance = PinMonitor._instance + assert instance is not None + if metadata is not None: + assert config is not None, "config is required to set up metrics" + instance._setup_metrics(config, metadata) + return instance def on_pin(self, memory_obj: "MemoryObj"): """Register a pinned memory object for timeout monitoring. @@ -202,7 +213,7 @@ def _execute(self) -> ThreadRunSummary: }, ) - def start_monitoring(self): + def start_monitoring(self) -> None: """Start the background monitoring thread.""" if self.is_running: return @@ -211,12 +222,15 @@ def start_monitoring(self): self.start() logger.info("PinMonitor started") - # Setup metrics callback - prometheus_logger = PrometheusLogger.GetInstanceOrNone() - if prometheus_logger is not None: - prometheus_logger.pin_monitor_pinned_objects_count.set_function( - lambda: len(self._pinned_objects) - ) + def _setup_metrics( + self, + config: "LMCacheEngineConfig", + metadata: "LMCacheMetadata", + ) -> None: + prometheus_logger = PrometheusLogger.GetOrCreate(metadata, config=config) + prometheus_logger.pin_monitor_pinned_objects_count.set_function( + lambda: len(self._pinned_objects) + ) def stop_monitoring(self): """Stop the background monitoring thread.""" diff --git a/lmcache/v1/storage_backend/batched_message_sender.py b/lmcache/v1/storage_backend/batched_message_sender.py index 6874f8f8bb7..c0b1bc089f8 100644 --- a/lmcache/v1/storage_backend/batched_message_sender.py +++ b/lmcache/v1/storage_backend/batched_message_sender.py @@ -55,10 +55,12 @@ def __init__( config: LMCacheEngineConfig, location: str, lmcache_worker: "LMCacheWorker", - ): + ) -> None: self.batch_size = config.get_extra_config_value("kv_msg_batch_size", 50) self.batch_timeout = config.get_extra_config_value("kv_msg_batch_timeout", 0.01) self.lmcache_worker = lmcache_worker + self.metadata = metadata + self.config = config # Common fields shared by all operations in the batch self.instance_id = config.lmcache_instance_id @@ -79,13 +81,15 @@ def __init__( self._setup_metrics() - def _setup_metrics(self): + def _setup_metrics(self) -> None: """Setup metrics for monitoring queue size.""" - prometheus_logger = PrometheusLogger.GetInstanceOrNone() - if prometheus_logger is not None: - prometheus_logger.kv_msg_queue_size.set_function( - lambda: self.message_queue.qsize() - ) + prometheus_logger = PrometheusLogger.GetOrCreate( + self.metadata, + config=self.config, + ) + prometheus_logger.kv_msg_queue_size.set_function( + lambda: self.message_queue.qsize() + ) def _start_background_thread(self): """Start background thread for periodic flushing.""" diff --git a/lmcache/v1/storage_backend/local_cpu_backend.py b/lmcache/v1/storage_backend/local_cpu_backend.py index fbc2e461e98..d35f3682bfb 100644 --- a/lmcache/v1/storage_backend/local_cpu_backend.py +++ b/lmcache/v1/storage_backend/local_cpu_backend.py @@ -103,15 +103,20 @@ def __init__( self._setup_metrics() - def _setup_metrics(self): - prometheus_logger = PrometheusLogger.GetInstanceOrNone() - if prometheus_logger is not None: - prometheus_logger.local_cpu_hot_cache_count.set_function( - lambda: len(self.hot_cache) - ) - prometheus_logger.local_cpu_keys_in_request_count.set_function( - lambda: len(self.keys_in_request) - ) + def _setup_metrics(self) -> None: + if self.metadata is None: + return + + prometheus_logger = PrometheusLogger.GetOrCreate( + self.metadata, + config=self.config, + ) + prometheus_logger.local_cpu_hot_cache_count.set_function( + lambda: len(self.hot_cache) + ) + prometheus_logger.local_cpu_keys_in_request_count.set_function( + lambda: len(self.keys_in_request) + ) def __str__(self): return self.__class__.__name__ diff --git a/lmcache/v1/storage_backend/remote_backend.py b/lmcache/v1/storage_backend/remote_backend.py index f6267227485..3532bbd02d7 100644 --- a/lmcache/v1/storage_backend/remote_backend.py +++ b/lmcache/v1/storage_backend/remote_backend.py @@ -96,23 +96,21 @@ def __init__( # health monitoring. The HealthMonitor in LMCacheEngine will # register RemoteBackendHealthCheck for each RemoteBackend. - self._setup_metrics() - self._get_blocking_failed_count = 0 self._put_failed_count = 0 - def _setup_metrics(self): - prometheus_logger = PrometheusLogger.GetInstanceOrNone() - if prometheus_logger is not None: - prometheus_logger.remote_put_task_num.set_function( - lambda: len(self.put_tasks) - ) - prometheus_logger.get_blocking_failed_count.set_function( - lambda: self._get_blocking_failed_count - ) - prometheus_logger.put_failed_count.set_function( - lambda: self._put_failed_count - ) + self._setup_metrics() + + def _setup_metrics(self) -> None: + prometheus_logger = PrometheusLogger.GetOrCreate( + self.metadata, + config=self.config, + ) + prometheus_logger.remote_put_task_num.set_function(lambda: len(self.put_tasks)) + prometheus_logger.get_blocking_failed_count.set_function( + lambda: self._get_blocking_failed_count + ) + prometheus_logger.put_failed_count.set_function(lambda: self._put_failed_count) def __str__(self): return self.__class__.__name__ diff --git a/lmcache/v1/storage_backend/storage_manager.py b/lmcache/v1/storage_backend/storage_manager.py index 156ba97094f..f74e76d070a 100644 --- a/lmcache/v1/storage_backend/storage_manager.py +++ b/lmcache/v1/storage_backend/storage_manager.py @@ -289,13 +289,10 @@ def __init__( self._setup_metrics() def _setup_metrics(self) -> None: - prometheus_logger = PrometheusLogger.GetInstanceOrNone() - if prometheus_logger is None: - logger.warning( - "PrometheusLogger is not initialized, " - "event metrics will not be collected" - ) - return + prometheus_logger = PrometheusLogger.GetOrCreate( + self.metadata, + config=self.config, + ) metric_map = { "storage_events_ongoing_count": EventStatus.ONGOING, diff --git a/tests/test_observability.py b/tests/test_observability.py index 6f41ae88ced..4e1f7eaed99 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock # Third Party +from prometheus_client import REGISTRY import pytest # First Party @@ -281,12 +282,12 @@ def test_zero_division_protection(stats_monitor): @pytest.fixture(scope="function") def _cleanup_prometheus_logger(): - """Reset PrometheusLogger singleton and metrics between tests.""" + """Reset PrometheusLogger instances and metrics between tests.""" LMCStatsMonitor.unregister_all_metrics() - PrometheusLogger._instance = None + PrometheusLogger._instances = {} yield LMCStatsMonitor.unregister_all_metrics() - PrometheusLogger._instance = None + PrometheusLogger._instances = {} def _make_metadata(): @@ -309,6 +310,11 @@ def _make_config(extra_config=None): return cfg +def _sample_labels(prom: PrometheusLogger) -> dict[str, str]: + """Return labels as Prometheus exposes them in collected samples.""" + return {name: str(value) for name, value in prom.labels.items()} + + def test_prometheus_logger_default_buckets(_cleanup_prometheus_logger): """Histogram should use default buckets when no config.""" meta = _make_metadata() @@ -382,3 +388,79 @@ def test_prometheus_logger_get_or_create_with_config( upper_bounds = list(hist._upper_bounds) for bucket_val in custom_buckets: assert bucket_val in upper_bounds + + +def test_prometheus_logger_get_or_create_allows_multiple_roles( + _cleanup_prometheus_logger, +): + """Different roles should not be treated as global metadata conflicts.""" + worker_meta = _make_metadata() + scheduler_meta = _make_metadata() + scheduler_meta.role = "scheduler" + + worker_prom = PrometheusLogger.GetOrCreate(worker_meta) + scheduler_prom = PrometheusLogger.GetOrCreate(scheduler_meta) + + assert worker_prom.labels["role"] == "worker" + assert scheduler_prom.labels["role"] == "scheduler" + + +def test_prometheus_logger_label_view_rebinds_dynamic_metrics( + _cleanup_prometheus_logger, +): + """Dynamic gauge callbacks should use each label view's labels.""" + worker_meta = _make_metadata() + scheduler_meta = _make_metadata() + scheduler_meta.role = "scheduler" + + worker_prom = PrometheusLogger.GetOrCreate(worker_meta) + scheduler_prom = PrometheusLogger.GetOrCreate(scheduler_meta) + + worker_prom.lmcache_is_healthy.set_function(lambda: 1) + scheduler_prom.lmcache_is_healthy.set_function(lambda: 2) + + assert ( + REGISTRY.get_sample_value( + "lmcache:lmcache_is_healthy", + _sample_labels(worker_prom), + ) + == 1 + ) + assert ( + REGISTRY.get_sample_value( + "lmcache:lmcache_is_healthy", + _sample_labels(scheduler_prom), + ) + == 2 + ) + + +def test_prometheus_logger_reset_reinitializes_all_label_views( + _cleanup_prometheus_logger, +): + """Resetting one label view should keep metric children visible for all views.""" + worker_meta = _make_metadata() + scheduler_meta = _make_metadata() + scheduler_meta.role = "scheduler" + + worker_prom = PrometheusLogger.GetOrCreate(worker_meta) + scheduler_prom = PrometheusLogger.GetOrCreate(scheduler_meta) + + worker_prom.counter_num_retrieve_requests.labels(**worker_prom.labels).inc() + scheduler_prom.counter_num_retrieve_requests.labels(**scheduler_prom.labels).inc() + scheduler_prom.reset_counters() + + assert ( + REGISTRY.get_sample_value( + "lmcache:num_retrieve_requests_total", + _sample_labels(worker_prom), + ) + == 0 + ) + assert ( + REGISTRY.get_sample_value( + "lmcache:num_retrieve_requests_total", + _sample_labels(scheduler_prom), + ) + == 0 + ) From ec4dbe127ea4a33a431f5866b96a4bf8ff51558e Mon Sep 17 00:00:00 2001 From: Yihua Cheng Date: Tue, 26 May 2026 14:55:21 -0500 Subject: [PATCH 68/69] =?UTF-8?q?docs:=20daily=20drift=20check=20=E2=80=94?= =?UTF-8?q?=20multi-process=20mode=20(2026-05-21)=20(#3361)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs(mp): note non-CUDA auto-disable of --l1-use-lazy PR #3259 (feat(mp): Non-GPU Context by pickle) added a post-init guard in lmcache/v1/distributed/config.py:L1MemoryManagerConfig that auto-disables use_lazy on backends where lmcache.torch_dev has no cudart attribute, logging a warning ("LazyMemoryAllocator requires cudart which is not available on the current backend. Disabling l1-use-lazy."). The user-facing reference in docs/source/mp/configuration.rst still listed the --l1-use-lazy default as plain "True" without mentioning that non-CUDA backends silently downgrade to eager allocation. Extend the description so the documented behavior matches the actual code path. Doc-only change; no code is modified. Signed-off-by: ApostaC --- docs/source/mp/configuration.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/mp/configuration.rst b/docs/source/mp/configuration.rst index d3777156346..cdd8641be2a 100644 --- a/docs/source/mp/configuration.rst +++ b/docs/source/mp/configuration.rst @@ -139,6 +139,10 @@ Source: ``lmcache/v1/distributed/config.py`` - Enable or disable lazy allocation for L1 memory. Pass ``--l1-use-lazy`` to enable (default) or ``--no-l1-use-lazy`` to explicitly disable. + Lazy allocation relies on ``cudart`` host-pinned memory, so on + non-CUDA backends (where ``lmcache.torch_dev`` exposes no + ``cudart`` attribute) it is automatically downgraded to eager + allocation with a logged warning, regardless of the flag value. * - ``--l1-init-size-gb`` - ``20`` - Initial allocation size (GB) when using lazy allocation. From 4bdcb5ccd7acb22cf94dd209410fe02d66e63d06 Mon Sep 17 00:00:00 2001 From: Zhengfei He <157287166+zhengfeihe@users.noreply.github.com> Date: Wed, 27 May 2026 05:06:34 +0900 Subject: [PATCH 69/69] [Fix] Change pinned pointer allocations in non-CUDA equivalents to be page-aligned for O_DIRECT (#3191) * [Fix] non_cuda_equivalents: page-align pinned allocations for O_DIRECT Signed-off-by: zhengfei.he Co-authored-by: Yihua Cheng --- lmcache/python_ops_fallback.py | 69 +++++++++++-------- .../test_rust_raw_block_backend.py | 25 +++++-- tests/v1/test_python_ops_fallback.py | 37 ++++++++++ 3 files changed, 97 insertions(+), 34 deletions(-) diff --git a/lmcache/python_ops_fallback.py b/lmcache/python_ops_fallback.py index 5c19d96f8b4..2da5e86d3e5 100644 --- a/lmcache/python_ops_fallback.py +++ b/lmcache/python_ops_fallback.py @@ -10,6 +10,7 @@ from typing import Optional, Tuple import ctypes import ctypes.util +import os import warnings # Third Party @@ -322,25 +323,48 @@ def __init__(self) -> None: self.block_stride_elems: int = 0 -def alloc_pinned_numa_ptr(size: int, numa_id: int = 0) -> int: - """Non-CUDA equivalent of allocating pinned memory with NUMA awareness. - On XPU, uses pin_memory=True (SYCL USM host allocation) for fast transfers. - Note: NUMA node selection is not supported on non-CUDA.""" +# Cuda path goes through func cudaHostAlloc, which is +# already page aligned by CUDA spec. This fallback shim mirrors that +# guarantee so consumers that require page-aligned host buffers, in +# particular the Rust raw-block backend when O_DIRECT is enabled, which +# requires page-aligned buffer pointer +try: + _PAGE_SIZE = os.sysconf("SC_PAGESIZE") +except (AttributeError, ValueError, OSError): + _PAGE_SIZE = 4096 - # Create a 1D uint8 CPU tensor, as uint8 == 1 byte - tensor = torch.empty(size, dtype=torch.uint8, pin_memory=False) - # First-touch initialization (forces physical allocation) - tensor.fill_(0) +def _alloc_page_aligned_pinned_view(size: int) -> Tuple[torch.Tensor, int]: + """ + Allocate a pinned CPU buffer whose first usable byte is page-aligned, + and return a torch view of ``size`` bytes plus its base pointer. - # Get a pointer to the start of the tensor object as this is what is - # returned by the CUDA equivalent function - ptr = tensor.data_ptr() + Internally over-allocates one extra page on a backing tensor, then + slices the aligned region out. The slice shares storage with the + backing tensor, so keeping the slice alive keeps the underlying + memory alive (no need to track the backing tensor separately). + """ + backing = torch.empty(size + _PAGE_SIZE, dtype=torch.uint8, pin_memory=False) + # First-touch initialization on the entire backing region + backing.fill_(0) + base = backing.data_ptr() + # Distance from `base` to the next page boundary (0..PAGE_SIZE-1). + offset = (-base) % _PAGE_SIZE + aligned_view = backing[offset : offset + size] + return aligned_view, aligned_view.data_ptr() - # Store the tensor so it can be accessed outide this function scope - _tensor_registry[ptr] = tensor - return ptr +def alloc_pinned_numa_ptr(size: int, numa_id: int = 0) -> int: + """Non-CUDA equivalent of allocating pinned memory with NUMA awareness. + On XPU, uses pin_memory=True (SYCL USM host allocation) for fast transfers. + Note: NUMA node selection is not supported on non-CUDA.""" + + view, aligned_ptr = _alloc_page_aligned_pinned_view(size) + # view shares storage with its over-allocated backing tensor; + # holding the view in the registry transitively keeps the underlying + # memory alive. + _tensor_registry[aligned_ptr] = view + return aligned_ptr def free_pinned_numa_ptr(ptr: int, size: int | None = None) -> None: @@ -355,20 +379,9 @@ def alloc_pinned_ptr(size: int, device_id: int = 0) -> int: to it. On XPU, uses pin_memory=True (SYCL USM host allocation) for fast DMA transfers. On other non-CUDA platforms, pinning is not supported.""" - # Create a 1D uint8 CPU tensor, as uint8 == 1 byte - tensor = torch.empty(size, dtype=torch.uint8, pin_memory=False) - - # First-touch initialization (forces physical allocation) - tensor.fill_(0) - - # Get a pointer to the start of the tensor object as this is what is - # returned by the CUDA equivalent function - ptr = tensor.data_ptr() - - # Store the tensor so it can be accessed outide this function scope - _tensor_registry[ptr] = tensor - - return ptr + view, aligned_ptr = _alloc_page_aligned_pinned_view(size) + _tensor_registry[aligned_ptr] = view + return aligned_ptr def free_pinned_ptr(ptr: int) -> None: diff --git a/tests/v1/storage_backend/test_rust_raw_block_backend.py b/tests/v1/storage_backend/test_rust_raw_block_backend.py index 6b39e75d58f..48d5308e9fa 100644 --- a/tests/v1/storage_backend/test_rust_raw_block_backend.py +++ b/tests/v1/storage_backend/test_rust_raw_block_backend.py @@ -2077,11 +2077,15 @@ def test_rust_raw_block_backend_uring_put_get_roundtrip( out = backend.get_blocking(key) assert out is not None, f"Failed to read key {i}" actual_data = bytes(out.byte_array) - assert actual_data == expected_data[i], ( - f"Data mismatch for key {i}: " - f"expected first bytes {expected_data[i][:16]}, " - f"got {actual_data[:16]}" - ) + # Use `raise AssertionError` instead of `assert ==` so pytest + # doesn't try to render a difflib diff between two large + # byte arrays on failure (effectively a hang on _fancy_replace). + if actual_data != expected_data[i]: + raise AssertionError( + f"Data mismatch for key {i}: " + f"expected first bytes {expected_data[i][:16]!r}, " + f"got {actual_data[:16]!r}" + ) finally: backend.close() @@ -2136,7 +2140,16 @@ def test_rust_raw_block_backend_close_with_inflight(memory_allocator, loop_in_th for i, expected in enumerate(buffers): actual = bytearray(payload_len) reader.pread_into(i * payload_len, actual, payload_len, payload_len) - assert actual == expected + # Use `raise AssertionError` instead of `assert ==` so pytest + # doesn't try to render a difflib-based diff between two + # large byte arrays on failure, which is O(n^2) on + # `_fancy_replace` and effectively never returns. + if actual != expected: + raise AssertionError( + f"Data mismatch at offset {i * payload_len}: " + f"expected first bytes {bytes(expected[:16])!r}, " + f"got {bytes(actual[:16])!r}" + ) finally: reader.close() diff --git a/tests/v1/test_python_ops_fallback.py b/tests/v1/test_python_ops_fallback.py index a071ef6e17d..723aa6459d7 100644 --- a/tests/v1/test_python_ops_fallback.py +++ b/tests/v1/test_python_ops_fallback.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # Standard from typing import Any, Union +import ctypes +import os import unittest.mock # Third Party @@ -1830,3 +1832,38 @@ def test_2_compare(self, name: str) -> None: pytest.fail( f"{name}/{key}: '{bid}'={val} != '{base_bid}'={base_val}" ) + + +# ========================================== +# Allocation page alignment +# ========================================== +# +# Rust raw-block backend with O_DIRECT requires page-aligned buffer +# pointers; CUDA path gets this for free via cudaHostAlloc, and the +# non-CUDA fallback in lmcache.non_cuda_equivalents shall mirror the same +# guarantee. + +_PINNED_ALLOC_SIZES = [1, 4095, 4096, 8192, 1024 * 1024] + + +@pytest.mark.parametrize("size", _PINNED_ALLOC_SIZES) +def test_alloc_pinned_ptr_is_page_aligned(size: int) -> None: + page_size = os.sysconf("SC_PAGESIZE") + ptr = _py_ops.alloc_pinned_ptr(size) + try: + assert ptr != 0 + if ptr % page_size != 0: + raise AssertionError( + f"alloc_pinned_ptr({size}) returned non-page-aligned ptr " + f"{hex(ptr)} (page size {page_size})" + ) + # Touch every byte in the requested region through the raw pointer + # to confirm the registered view covers the full requested size + # (an undersized view would corrupt adjacent memory or segfault). + buf = (ctypes.c_uint8 * size).from_address(ptr) + for i in range(size): + buf[i] = (i & 0xFF) ^ 0xA5 + for i in range(size): + assert buf[i] == ((i & 0xFF) ^ 0xA5) + finally: + _py_ops.free_pinned_ptr(ptr)