Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions studio/backend/core/inference/llama_cpp.py
Original file line number Diff line number Diff line change
Expand Up @@ -15034,9 +15034,22 @@ def _subset_model_size(n_gpus: int) -> int:
# Saved KV encodes chat content; keep it from other local users.
with contextlib.suppress(OSError):
os.chmod(slot_dir, 0o700)
cmd.extend(["--slot-save-path", str(slot_dir)])
self._slot_save_dir = str(slot_dir)
self._slot_save_binary = (binary, Path(binary).stat().st_mtime_ns)
slot_path = str(slot_dir)
# llama.cpp's fs_is_directory() constructs a narrow
# std::filesystem::path on Windows, so an existing UTF-8
# path such as C:\Users\Егор is rejected as missing.
# Slot persistence is optional; keep model loading available
# until the managed runtime carries the upstream UTF-8 fix.
if sys.platform == "win32" and not slot_path.isascii():
logger.warning(
"Disabling llama.cpp slot persistence because its "
"Windows path contains non-ASCII characters: %s",
slot_path,
)
else:
cmd.extend(["--slot-save-path", slot_path])
self._slot_save_dir = slot_path
self._slot_save_binary = (binary, Path(binary).stat().st_mtime_ns)
except OSError:
self._slot_save_dir = None
self._slot_save_binary = None
Expand Down
42 changes: 42 additions & 0 deletions studio/backend/tests/test_llama_extra_args_platforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,48 @@ def _stable(cmd: list[str]) -> list[str]:
return masked


def _launch_with_slot_dir(tmp_path, monkeypatch, platform, slot_dir: Path):
"""Capture the real launch command with llama.cpp slot persistence advertised."""
_apply_platform(monkeypatch, platform)
from utils.paths import storage_roots

monkeypatch.setattr(storage_roots, "llama_slot_cache_root", lambda: slot_dir)
backend, gguf = _backend(tmp_path, vulkan = False, memory = [])
backend.probe_server_capabilities = lambda _binary = None: {"supports_slot_save": True}
return backend, _launch(backend, gguf)["cmd"]


def _without_flag_value(cmd: list[str], flag: str) -> list[str]:
index = cmd.index(flag)
return [*cmd[:index], *cmd[index + 2 :]]


def test_windows_unicode_slot_path_only_drops_optional_persistence(tmp_path, monkeypatch):
"""A/B: all launch arguments survive; only the broken optional pair is absent."""
windows = PLATFORMS[2]
ascii_dir = tmp_path / "Egor" / "llama-slots"
unicode_dir = tmp_path / "Егор" / "llama-slots"

_ascii_backend, ascii_cmd = _launch_with_slot_dir(tmp_path, monkeypatch, windows, ascii_dir)
unicode_backend, unicode_cmd = _launch_with_slot_dir(
tmp_path, monkeypatch, windows, unicode_dir
)

assert ascii_cmd[ascii_cmd.index("--slot-save-path") + 1] == str(ascii_dir)
assert _stable(unicode_cmd) == _without_flag_value(_stable(ascii_cmd), "--slot-save-path")
assert unicode_backend._slot_save_dir is None
assert unicode_backend._slot_save_binary is None


@pytest.mark.parametrize("platform", [PLATFORMS[0], PLATFORMS[1], PLATFORMS[3]])
def test_unicode_slot_path_is_unchanged_off_native_windows(tmp_path, monkeypatch, platform):
slot_dir = tmp_path / "Егор" / "llama-slots"

_backend_instance, cmd = _launch_with_slot_dir(tmp_path, monkeypatch, platform, slot_dir)

assert cmd[cmd.index("--slot-save-path") + 1] == str(slot_dir)


@pytest.mark.parametrize("platform,accelerator", MATRIX)
def test_an_empty_box_changes_nothing_anywhere(tmp_path, monkeypatch, platform, accelerator):
# The acceptance bar, on every combination. None (inherit) and [] (explicitly
Expand Down
Loading