From 84aae356eadc7ef5bdb2b6a824370593772a70e8 Mon Sep 17 00:00:00 2001 From: Bru Date: Thu, 27 Aug 2026 02:07:18 +0200 Subject: [PATCH 1/9] Add portable persistent Raw preload cache --- doc/changes/dev/14216.newfeature.rst | 2 + mne/channels/channels.py | 1 + mne/io/_preload_cache.py | 628 +++++++++++++++++++++++++ mne/io/base.py | 55 ++- mne/io/brainvision/brainvision.py | 6 + mne/io/edf/edf.py | 34 ++ mne/io/fiff/raw.py | 25 + mne/io/fiff/tests/test_raw_fiff.py | 23 + mne/io/tests/test_preload_cache.py | 679 +++++++++++++++++++++++++++ mne/io/tests/test_raw.py | 62 ++- mne/utils/config.py | 21 +- mne/utils/docs.py | 14 +- 12 files changed, 1523 insertions(+), 27 deletions(-) create mode 100644 doc/changes/dev/14216.newfeature.rst create mode 100644 mne/io/_preload_cache.py create mode 100644 mne/io/tests/test_preload_cache.py diff --git a/doc/changes/dev/14216.newfeature.rst b/doc/changes/dev/14216.newfeature.rst new file mode 100644 index 00000000000..ebfb87f9fdd --- /dev/null +++ b/doc/changes/dev/14216.newfeature.rst @@ -0,0 +1,2 @@ +Speed up repeated preloading of uncompressed FIF, EDF/BDF, and BrainVision +recordings with a persistent copy-on-write decoded-data cache, by `Bruno Aristimunha`_. diff --git a/mne/channels/channels.py b/mne/channels/channels.py index d0e9ae73d6e..6262208b1ba 100644 --- a/mne/channels/channels.py +++ b/mne/channels/channels.py @@ -748,6 +748,7 @@ def add_channels(self, add_list, force_update_info=False): # Now update the attributes if ( isinstance(self._data, np.memmap) + and self._data.mode != "c" and con_axis == 0 and sys.platform != "darwin" ): # resizing not available--no mremap diff --git a/mne/io/_preload_cache.py b/mne/io/_preload_cache.py new file mode 100644 index 00000000000..e9e62656931 --- /dev/null +++ b/mne/io/_preload_cache.py @@ -0,0 +1,628 @@ +"""Persistent decoded-data cache for Raw readers.""" + +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +import ctypes +import errno +import hashlib +import json +import os +import pickle +import stat +import time +from ctypes import wintypes +from pathlib import Path + +import numpy as np + +from .. import __version__ as MNE_VERSION # ty: ignore[unresolved-import] +from ..utils import get_config, logger + +_RAW_PRELOAD_CACHE_VERSION = 1 +_RAW_PRELOAD_LOCK_TIMEOUT = 300.0 +_IS_WINDOWS = os.name == "nt" + +if _IS_WINDOWS: + import msvcrt + +_FILE_ATTRIBUTE_DIRECTORY = 0x00000010 +_FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400 +_FILE_ATTRIBUTE_TAG_INFO = 9 +_FILE_BASIC_INFO = 0 +_FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 +_FILE_ID_INFO = 18 +_FILE_SHARE_ALL = 0x00000007 +_GENERIC_READ = 0x80000000 +_OPEN_EXISTING = 3 +_PROCESS_SYNCHRONIZE = 0x00100000 +_ERROR_INVALID_PARAMETER = 87 +_WAIT_OBJECT_0 = 0 + + +class _FileBasicInfo(ctypes.Structure): + _fields_ = ( + ("creation_time", ctypes.c_longlong), + ("last_access_time", ctypes.c_longlong), + ("last_write_time", ctypes.c_longlong), + ("change_time", ctypes.c_longlong), + ("file_attributes", wintypes.DWORD), + ) + + +class _FileId128(ctypes.Structure): + _fields_ = (("identifier", ctypes.c_ubyte * 16),) + + +class _FileIdInfo(ctypes.Structure): + _fields_ = ( + ("volume_serial_number", ctypes.c_ulonglong), + ("file_id", _FileId128), + ) + + +def _raw_preload_windows_fstat(descriptor, result): + """Return Windows change time and file ID from an open handle.""" + kernel32 = getattr(ctypes, "WinDLL")("kernel32", use_last_error=True) + handle = getattr(msvcrt, "get_osfhandle")(descriptor) + basic = _FileBasicInfo() + get_ex = kernel32.GetFileInformationByHandleEx + get_ex.argtypes = ( + wintypes.HANDLE, + ctypes.c_int, + wintypes.LPVOID, + wintypes.DWORD, + ) + get_ex.restype = wintypes.BOOL + if not get_ex(handle, _FILE_BASIC_INFO, ctypes.byref(basic), ctypes.sizeof(basic)): + raise getattr(ctypes, "WinError")(getattr(ctypes, "get_last_error")()) + file_info = _FileIdInfo() + if not get_ex( + handle, _FILE_ID_INFO, ctypes.byref(file_info), ctypes.sizeof(file_info) + ): + raise getattr(ctypes, "WinError")(getattr(ctypes, "get_last_error")()) + return dict( + size=int(result.st_size), + mtime_ns=int(basic.last_write_time) * 100, + change_ns=int(basic.change_time) * 100, + device=int(file_info.volume_serial_number), + inode=int.from_bytes(bytes(file_info.file_id.identifier), "little"), + ) + + +def _raw_preload_fstat(descriptor): + """Return an identity token for an already-open regular file.""" + result = os.fstat(descriptor) + if not stat.S_ISREG(result.st_mode): + raise OSError("Decoded data cache entries must be regular files") + if _IS_WINDOWS: + return _raw_preload_windows_fstat(descriptor, result) + return dict( + size=int(result.st_size), + mtime_ns=int(result.st_mtime_ns), + change_ns=int(result.st_ctime_ns), + device=int(result.st_dev), + inode=int(result.st_ino), + ) + + +def _raw_preload_open_windows(path): + """Open a Windows file handle without traversing a reparse point.""" + kernel32 = getattr(ctypes, "WinDLL")("kernel32", use_last_error=True) + create_file = kernel32.CreateFileW + create_file.argtypes = ( + wintypes.LPCWSTR, + wintypes.DWORD, + wintypes.DWORD, + wintypes.LPVOID, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, + ) + create_file.restype = wintypes.HANDLE + handle = create_file( + str(path), + _GENERIC_READ, + _FILE_SHARE_ALL, + None, + _OPEN_EXISTING, + _FILE_FLAG_OPEN_REPARSE_POINT, + None, + ) + invalid_handle = ctypes.c_void_p(-1).value + if handle == invalid_handle: + raise getattr(ctypes, "WinError")(getattr(ctypes, "get_last_error")()) + close_handle = kernel32.CloseHandle + close_handle.argtypes = (wintypes.HANDLE,) + close_handle.restype = wintypes.BOOL + try: + attributes = wintypes.DWORD() + get_attributes = kernel32.GetFileInformationByHandleEx + get_attributes.argtypes = ( + wintypes.HANDLE, + ctypes.c_int, + wintypes.LPVOID, + wintypes.DWORD, + ) + get_attributes.restype = wintypes.BOOL + # FileAttributeTagInfo is 9; its first DWORD contains the attributes. + attribute_tag_info = (wintypes.DWORD * 2)() + if not get_attributes( + handle, + _FILE_ATTRIBUTE_TAG_INFO, + ctypes.byref(attribute_tag_info), + ctypes.sizeof(attribute_tag_info), + ): + raise getattr(ctypes, "WinError")(getattr(ctypes, "get_last_error")()) + attributes.value = attribute_tag_info[0] + if attributes.value & _FILE_ATTRIBUTE_REPARSE_POINT: + raise OSError( + f"Decoded data cache entries cannot be reparse points: {path}" + ) + if attributes.value & _FILE_ATTRIBUTE_DIRECTORY: + raise OSError(f"Decoded data cache entries must be regular files: {path}") + descriptor = getattr(msvcrt, "open_osfhandle")( + handle, os.O_RDONLY | getattr(os, "O_BINARY") + ) + except Exception: + close_handle(handle) + raise + return os.fdopen(descriptor, "rb") + + +def _raw_preload_open_regular(path): + """Open a regular cache file without following a final-component link.""" + if _IS_WINDOWS: + return _raw_preload_open_windows(path) + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + try: + flags |= os.O_NOFOLLOW + except AttributeError as error: + raise PermissionError( + "Automatic Raw preload caching requires no-follow file opens" + ) from error + descriptor = os.open(path, flags) + try: + _raw_preload_fstat(descriptor) + except Exception: + os.close(descriptor) + raise + return os.fdopen(descriptor, "rb") + + +def _raw_preload_path_stat(path): + """Return the identity token for a path using one validated handle.""" + with _raw_preload_open_regular(path) as file: + return _raw_preload_fstat(file.fileno()) + + +def _raw_preload_source_signature(raw): + """Return filesystem identities for the source data files.""" + sources = [] + for filename in raw.filenames: + if filename is None: + raise ValueError( + 'preload="auto" requires stable source files; use preload=True ' + "or an explicit memory-map path" + ) + path = Path(filename).resolve(strict=True) + sources.append(dict(path=str(path), **_raw_preload_path_stat(path))) + return sources + + +def _raw_preload_cache_dir(cache_root=None): + """Resolve and validate the managed cache directory.""" + if cache_root is None: + cache_root = get_config("MNE_CACHE_DIR", None) + if cache_root is None: + raise ValueError( + 'preload="auto" requires a configured cache directory; use ' + "mne.set_cache_dir(...) first" + ) + cache_root = Path(cache_root).expanduser().resolve() + cache_dir = cache_root / f"raw-preload-v{_RAW_PRELOAD_CACHE_VERSION}" + cache_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + if cache_dir.is_symlink() or not cache_dir.is_dir(): + raise OSError(f"Decoded data cache must be a regular directory: {cache_dir}") + if not _IS_WINDOWS: + _raw_preload_validate_directory(cache_dir) + cache_dir.chmod(0o700) + if cache_dir.stat().st_mode & 0o077: + raise PermissionError(f"Decoded data cache is not private: {cache_dir}") + return cache_dir + + +def _raw_preload_validate_directory(cache_dir): + """Reject cache paths that another local user could replace.""" + user_id = os.geteuid() + child_stat = cache_dir.lstat() + if stat.S_ISLNK(child_stat.st_mode) or not stat.S_ISDIR(child_stat.st_mode): + raise OSError(f"Decoded data cache must be a regular directory: {cache_dir}") + if child_stat.st_uid != user_id: + raise PermissionError( + f"Decoded data cache must be owned by the current user: {cache_dir}" + ) + if child_stat.st_mode & 0o077: + raise PermissionError( + f"Decoded data cache must already be private: {cache_dir}" + ) + for parent in cache_dir.parents: + parent_stat = parent.lstat() + if stat.S_ISLNK(parent_stat.st_mode) or not stat.S_ISDIR(parent_stat.st_mode): + raise PermissionError( + f"Decoded data cache cannot have a symlink ancestor: {parent}" + ) + if parent_stat.st_uid not in (0, user_id): + raise PermissionError( + "Decoded data cache cannot have an untrusted owner in its " + f"physical ancestry: {parent}" + ) + if parent_stat.st_mode & 0o022 and not parent_stat.st_mode & stat.S_ISVTX: + raise PermissionError( + "Decoded data cache cannot have an untrusted writable ancestor: " + f"{parent}" + ) + + +def _raw_preload_cache_info(raw): + """Return the managed cache location and expected array description.""" + cache_identity = raw._decoded_cache_identity() + if cache_identity is None: + raise ValueError( + f'preload="auto" is not supported for {type(raw).__name__}; use ' + "preload=True or an explicit memory-map path" + ) + decoder_abi, decoder_state = cache_identity + cache_dir = _raw_preload_cache_dir() + sources = _raw_preload_source_signature(raw) + dtype = np.dtype(raw._dtype) + shape = (int(raw.info["nchan"]), int(raw.n_times)) + identity = dict( + version=_RAW_PRELOAD_CACHE_VERSION, + mne_version=MNE_VERSION, + reader=(type(raw).__module__, type(raw).__qualname__), + decoder_abi=decoder_abi, + sources=sources, + decoder_state=decoder_state, + read_picks=raw._read_picks, + cals=raw._cals, + projector=raw._projector, + compensator=raw._comp, + first_samps=raw._first_samps, + last_samps=raw._last_samps, + dtype=dtype.str, + shape=shape, + ) + try: + serialized = pickle.dumps(identity, protocol=5) + except Exception as exc: + raise ValueError( + f'preload="auto" cannot identify this {type(raw).__name__} source' + ) from exc + key = hashlib.sha256(serialized).hexdigest() + return cache_dir, key, sources, shape, dtype + + +def _raw_preload_generation_name(key, token): + """Return a unique immutable generation basename.""" + return f"{key}.{token}.data" + + +def _raw_preload_generation_valid(name, key): + """Check that a manifest generation is a managed basename.""" + prefix = f"{key}." + suffix = ".data" + if ( + not isinstance(name, str) + or not name.startswith(prefix) + or not name.endswith(suffix) + ): + return False + token = name[len(prefix) : -len(suffix)] + return len(token) == 32 and all(char in "0123456789abcdef" for char in token) + + +def _raw_preload_read_manifest(cache_dir, key): + """Read one manifest through its validated handle.""" + path = cache_dir / f"{key}.json" + with _raw_preload_open_regular(path) as file: + file_stat = _raw_preload_fstat(file.fileno()) + if file_stat["size"] > 4096: + raise ValueError("Oversized Raw preload manifest") + return json.loads(file.read().decode("utf-8")) + + +def _raw_preload_cache_read(raw, cache_dir, key, sources, shape, dtype): + """Read and validate one managed decoded-data cache entry.""" + try: + manifest = _raw_preload_read_manifest(cache_dir, key) + if set(manifest) != { + "version", + "generation", + "generation_stat", + }: + return None + nbytes = int(np.prod(shape, dtype=np.int64)) * dtype.itemsize + if manifest[ + "version" + ] != _RAW_PRELOAD_CACHE_VERSION or not _raw_preload_generation_valid( + manifest["generation"], key + ): + return None + generation = cache_dir / manifest["generation"] + with _raw_preload_open_regular(generation) as file: + generation_stat = _raw_preload_fstat(file.fileno()) + if ( + generation_stat != manifest["generation_stat"] + or generation_stat["size"] != nbytes + ): + return None + data = np.memmap(file, mode="c", dtype=dtype, shape=shape) + data.filename = str(generation) # ty: ignore[invalid-assignment] + if _raw_preload_source_signature(raw) != sources: + data._mmap.close() # ty: ignore[unresolved-attribute] # memmap private + return None + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return None + logger.info(f"Reusing decoded data from {generation}") + return data + + +def _raw_preload_fsync_directory(path): + """Sync publication metadata where directory fsync is supported.""" + if _IS_WINDOWS: + return + descriptor = os.open(path, os.O_RDONLY) + try: + try: + os.fsync(descriptor) + except OSError as error: + unsupported = {errno.EINVAL, getattr(errno, "ENOTSUP", errno.EINVAL)} + if error.errno not in unsupported: + raise + finally: + os.close(descriptor) + + +def _raw_preload_protect_generation(descriptor): + """Make a generation owner-read-only where file modes support it.""" + if not _IS_WINDOWS: + os.fchmod(descriptor, 0o400) + + +def _raw_preload_replace_manifest(source, destination): + """Atomically replace a manifest, tolerating transient Windows locks.""" + for attempt in range(5): + try: + os.replace(source, destination) + except PermissionError: + if attempt == 4: + raise + time.sleep(0.01 * (attempt + 1)) + else: + return + + +def _raw_preload_process_alive(process_id): + """Return whether a local process still owns a cache publication lock.""" + if process_id <= 0: + return False + if _IS_WINDOWS: + kernel32 = getattr(ctypes, "WinDLL")("kernel32", use_last_error=True) + open_process = kernel32.OpenProcess + open_process.argtypes = (wintypes.DWORD, wintypes.BOOL, wintypes.DWORD) + open_process.restype = wintypes.HANDLE + wait = kernel32.WaitForSingleObject + wait.argtypes = (wintypes.HANDLE, wintypes.DWORD) + wait.restype = wintypes.DWORD + close = kernel32.CloseHandle + close.argtypes = (wintypes.HANDLE,) + close.restype = wintypes.BOOL + handle = open_process(_PROCESS_SYNCHRONIZE, False, process_id) + if not handle: + # Access denied is known-alive; unknown failures are also treated + # conservatively. ERROR_INVALID_PARAMETER is the invalid-PID case. + return getattr(ctypes, "get_last_error")() != _ERROR_INVALID_PARAMETER + try: + result = wait(handle, 0) + if result == _WAIT_OBJECT_0: + return False + return True # WAIT_TIMEOUT, WAIT_FAILED, or an unknown result + finally: + close(handle) + try: + os.kill(process_id, 0) + except ProcessLookupError: + return False + except (OSError, PermissionError): + return True + return True + + +def _raw_preload_try_lock(lock_path, owner): + """Try to claim a cache lock without blocking.""" + try: + descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + except FileExistsError: + return False + try: + with os.fdopen(descriptor, "w", encoding="ascii") as file: + file.write(owner) + file.flush() + os.fsync(file.fileno()) + except Exception: + lock_path.unlink(missing_ok=True) + raise + return True + + +def _raw_preload_remove_stale_lock(lock_path): + """Remove a dead or abandoned cache lock without touching its target.""" + try: + lock_stat = lock_path.lstat() + if lock_path.is_symlink(): + stale = True + else: + age = time.time() - lock_stat.st_mtime + try: + content = lock_path.read_text(encoding="ascii") + if len(content) > 256: + raise ValueError + process_id = int(content.split()[0]) + alive = _raw_preload_process_alive(process_id) + except (IndexError, OSError, OverflowError, ValueError): + stale = age > 5.0 + else: + stale = not alive + current = lock_path.lstat() + unchanged = (current.st_dev, current.st_ino, current.st_mtime_ns) == ( + lock_stat.st_dev, + lock_stat.st_ino, + lock_stat.st_mtime_ns, + ) + if stale and unchanged: + lock_path.unlink() + return True + except FileNotFoundError: + return True + return False + + +def _raw_preload_release_lock(lock_path, owner): + """Release a cache lock only when it is still owned by this process.""" + try: + if ( + not lock_path.is_symlink() + and lock_path.read_text(encoding="ascii") == owner + ): + lock_path.unlink() + except FileNotFoundError: + pass + + +def _raw_preload_scavenge_key(cache_dir, key): + """Remove abandoned temporary and unreferenced same-key generations.""" + referenced = None + try: + manifest = _raw_preload_read_manifest(cache_dir, key) + candidate = manifest.get("generation") + if _raw_preload_generation_valid(candidate, key): + referenced = candidate + except (OSError, TypeError, ValueError, json.JSONDecodeError): + pass + patterns = (f".{key}.*.tmp", f"{key}.*.data") + for pattern in patterns: + for path in cache_dir.glob(pattern): + if path.name == referenced: + continue + try: + path.unlink() + except OSError: + logger.debug( + f"Could not remove abandoned Raw preload cache file {path}" + ) + + +def _raw_preload_cache_create(raw, cache_dir, key, sources, shape, dtype): + """Decode, durably publish, and reopen an immutable cache generation.""" + token = os.urandom(16).hex() + generation_name = _raw_preload_generation_name(key, token) + generation = cache_dir / generation_name + temporary = cache_dir / f".{generation_name}.tmp" + manifest_temporary = None + manifest_published = False + nbytes = int(np.prod(shape, dtype=np.int64)) * dtype.itemsize + descriptor = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) + try: + with os.fdopen(descriptor, "r+b") as file: + file.truncate(nbytes) + data_buffer = np.memmap(file, mode="r+", dtype=dtype, shape=shape) + data = raw._read_segment(data_buffer=data_buffer) + try: + data.flush() + except BaseException: + try: + data._mmap.close() # memmap private + except Exception: + pass + raise + else: + data._mmap.close() # memmap private + del data, data_buffer + _raw_preload_protect_generation(file.fileno()) + os.fsync(file.fileno()) + if _raw_preload_source_signature(raw) != sources: + raise RuntimeError( + "Source data changed while decoded cache was created; retry" + ) + os.replace(temporary, generation) + _raw_preload_fsync_directory(cache_dir) + manifest = dict( + version=_RAW_PRELOAD_CACHE_VERSION, + generation=generation_name, + generation_stat=_raw_preload_path_stat(generation), + ) + manifest_temporary = cache_dir / f".{key}.{os.urandom(16).hex()}.json.tmp" + descriptor = os.open( + manifest_temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600 + ) + with os.fdopen(descriptor, "w", encoding="utf-8") as file: + json.dump(manifest, file, sort_keys=True, separators=(",", ":")) + file.flush() + os.fsync(file.fileno()) + _raw_preload_replace_manifest(manifest_temporary, cache_dir / f"{key}.json") + manifest_published = True + _raw_preload_fsync_directory(cache_dir) + with _raw_preload_open_regular(generation) as file: + if _raw_preload_fstat(file.fileno()) != manifest["generation_stat"]: + raise RuntimeError( + "Decoded cache generation changed during publication" + ) + result = np.memmap(file, mode="c", dtype=dtype, shape=shape) + result.filename = str(generation) # ty: ignore[invalid-assignment] + return result + finally: + for path in (temporary, manifest_temporary): + if path is not None: + try: + path.unlink(missing_ok=True) + except OSError: + pass + if not manifest_published: + try: + generation.unlink(missing_ok=True) + except OSError: + pass + + +def _raw_preload_auto(raw): + """Reuse or create an automatic decoded-data cache entry.""" + cache_dir, key, sources, shape, dtype = _raw_preload_cache_info(raw) + key_lock = cache_dir / f"{key}.lock" + owner = f"{os.getpid()} {os.urandom(16).hex()}" + deadline = time.monotonic() + _RAW_PRELOAD_LOCK_TIMEOUT + while True: + data = _raw_preload_cache_read(raw, cache_dir, key, sources, shape, dtype) + if data is not None: + return data + if _raw_preload_try_lock(key_lock, owner): + try: + _raw_preload_scavenge_key(cache_dir, key) + data = _raw_preload_cache_read( + raw, cache_dir, key, sources, shape, dtype + ) + if data is None: + logger.info(f"Creating decoded data cache in {cache_dir}") + data = _raw_preload_cache_create( + raw, cache_dir, key, sources, shape, dtype + ) + _raw_preload_scavenge_key(cache_dir, key) + return data + finally: + _raw_preload_release_lock(key_lock, owner) + elif _raw_preload_remove_stale_lock(key_lock): + continue + if time.monotonic() >= deadline: + raise TimeoutError(f"Timed out waiting for decoded cache lock {key_lock}") + time.sleep(0.025) diff --git a/mne/io/base.py b/mne/io/base.py index 7d41bb20be6..f6a4bdad993 100644 --- a/mne/io/base.py +++ b/mne/io/base.py @@ -103,6 +103,7 @@ warn, ) from ..utils._typing import Color, Self +from ._preload_cache import _raw_preload_auto if TYPE_CHECKING: # Heavy/optional deps kept out of the runtime import path (see @@ -143,8 +144,13 @@ class BaseRaw( freshly created memory-mapped file used to store the data on the hard drive (slower, requires less memory). An existing file is overwritten. The caller owns the file and is responsible for removing it after the - Raw object is no longer in use. If preload is an ndarray, the data are - taken from that array. If False, data are not read until save. + Raw object is no longer in use. For supported file readers, the exact + string ``"auto"`` instead stores and reuses decoded data in the directory + configured by :func:`mne.set_cache_dir`. Cached data persist in a + versioned ``raw-preload`` directory and are mapped copy-on-write. Use + ``Path("auto")`` or ``"./auto"`` for a literal filename. If preload is + an ndarray, the data are taken from that array. If False, data are not + read until save. first_samps : sequence Sequence of the first sample number from each raw file. For unsplit raw files this should be a length-one list or tuple. @@ -201,6 +207,10 @@ class BaseRaw( _filenames: list[Path | None] _data: np.ndarray | None + def _decoded_cache_identity(self): + """Return ``(ABI, state)`` for numeric decoding, or ``None``.""" + return None + @verbose def __init__( self, @@ -594,18 +604,22 @@ def _check_bad_segment( def load_data( self, *, - memmap: Path | str | None = None, + memmap: Path | Literal["auto"] | str | None = None, verbose: bool | str | int | None = None, ) -> Self: """Load raw data. Parameters ---------- - memmap : path-like | None - If not ``None``, preload data into a freshly created memory-mapped file - at this path. An existing file is overwritten. The caller owns the file - and is responsible for removing it after the Raw object is no longer in - use. If ``None`` (default), preload data into RAM. + memmap : path-like | "auto" | None + If a path, preload data into a freshly created memory-mapped file at + this path. An existing file is overwritten. The caller owns the file + and is responsible for removing it after the Raw object is no longer + in use. For supported file readers, ``"auto"`` instead reuses the + persistent decoded-data cache configured by :func:`mne.set_cache_dir`. + Cache entries for superseded source or decoder identities remain in + a versioned ``raw-preload`` directory below the configured path. + If ``None`` (default), preload data into RAM. .. versionadded:: 1.13 %(verbose)s @@ -630,12 +644,22 @@ def load_data( def _preload_data(self, preload): """Actually preload the data.""" + if isinstance(preload, str) and preload == "auto": + self._data = _raw_preload_auto(self) + assert len(self._data) == self.info["nchan"] + self.preload = True + self._comp = None + self.close() + return data_buffer = preload if isinstance(preload, bool | np.bool_) and not preload: data_buffer = None - t = self.times + # Avoid materializing ``self.times``; that scales with the recording + # length and can dominate a decoded-cache hit. + n_times = self.n_times + last_time = (n_times - 1) / self.info["sfreq"] logger.info( - f"Reading 0 ... {len(t) - 1} = {0.0:9.3f} ... {t[-1]:9.3f} secs..." + f"Reading 0 ... {n_times - 1} = {0.0:9.3f} ... {last_time:9.3f} secs..." ) self._data = self._read_segment(data_buffer=data_buffer) assert len(self._data) == self.info["nchan"] @@ -793,17 +817,18 @@ def set_annotations( "of the raw object." ) - delta = 1.0 / self.info["sfreq"] + # This is algebraically ``self.times[-1] + 1 / sfreq`` without + # allocating the full time vector for large file-backed recordings. + sfreq = self.info["sfreq"] + annotation_end = (self.n_times - 1) / sfreq + 1.0 / sfreq new_annotations = annotations.copy() new_annotations._prune_ch_names(self.info, on_missing) if annotations.orig_time is None: - new_annotations.crop( - 0, self.times[-1] + delta, emit_warning=emit_warning - ) + new_annotations.crop(0, annotation_end, emit_warning=emit_warning) new_annotations.onset += self._first_time else: tmin = meas_date + timedelta(0, self._first_time) - tmax = tmin + timedelta(seconds=self.times[-1] + delta) + tmax = tmin + timedelta(seconds=annotation_end) new_annotations.crop(tmin=tmin, tmax=tmax, emit_warning=emit_warning) new_annotations.onset -= ( meas_date - new_annotations.orig_time diff --git a/mne/io/brainvision/brainvision.py b/mne/io/brainvision/brainvision.py index 9c9d978a861..bca37afa9cc 100644 --- a/mne/io/brainvision/brainvision.py +++ b/mne/io/brainvision/brainvision.py @@ -95,6 +95,12 @@ class RawBrainVision(BaseRaw): _extra_attributes = ("impedances",) + def _decoded_cache_identity(self): + """Return identity that determines numeric BrainVision decoding.""" + keys = ("offsets", "fmt", "order", "n_samples", "orig_nchan") + state = [{key: extra[key] for key in keys} for extra in self._raw_extras] + return (1, state) + @verbose def __init__( self, diff --git a/mne/io/edf/edf.py b/mne/io/edf/edf.py index 390289ae6bb..e26774370d1 100644 --- a/mne/io/edf/edf.py +++ b/mne/io/edf/edf.py @@ -59,6 +59,34 @@ class FileType(Enum): } +def _edf_decoded_cache_identity(raw): + """Return identity that determines numeric EDF/BDF decoding.""" + if any(extra["blob"] is not None for extra in raw._raw_extras): + raise ValueError('preload="auto" does not support file-like EDF/BDF inputs') + # Keep this in the same order as the values consumed by + # ``_read_segment_file`` below. Header metadata that cannot affect decoded + # samples (for example channel types and filter descriptions) stays live. + state = [ + ( + extra["n_samps"], + extra["max_samp"], + extra["dtype_np"], + extra["dtype_byte"], + extra["data_offset"], + extra["stim_channel_idxs"], + extra["sel"], + extra["tal_idx"], + extra["subtype"], + extra["cal"], + extra["offsets"], + extra["units"], + extra["nsamples"], + ) + for extra in raw._raw_extras + ] + return (1, state) + + @fill_doc class RawEDF(BaseRaw): """Raw object from EDF, EDF+ file. @@ -154,6 +182,9 @@ class RawEDF(BaseRaw): encoded in such analog stim channels. """ + def _decoded_cache_identity(self): + return _edf_decoded_cache_identity(self) + @verbose def __init__( self, @@ -366,6 +397,9 @@ class RawBDF(BaseRaw): encoded in such analog stim channels. """ + def _decoded_cache_identity(self): + return _edf_decoded_cache_identity(self) + @verbose def __init__( self, diff --git a/mne/io/fiff/raw.py b/mne/io/fiff/raw.py index 95c6db5dbec..779005dda72 100644 --- a/mne/io/fiff/raw.py +++ b/mne/io/fiff/raw.py @@ -83,6 +83,20 @@ class Raw(BaseRaw): Indicates whether raw data are in memory. """ + def _decoded_cache_identity(self): + """Return identity that determines numeric FIF decoding.""" + if any( + filename is not None and filename.suffix == ".gz" + for filename in self.filenames + ): + raise ValueError( + 'preload="auto" supports only uncompressed FIF files; use ' + "preload=True for gzip-compressed FIF" + ) + keys = ("ent", "bounds", "orig_nchan") + state = [{key: extra[key] for key in keys} for extra in self._raw_extras] + return (1, state) + _extra_attributes = ( "fix_mag_coil_types", "acqparser", @@ -98,6 +112,17 @@ def __init__( on_split_missing: str = "raise", verbose: bool | str | int | None = None, ): + if isinstance(preload, str) and preload == "auto": + if _file_like(fname): + raise ValueError( + 'preload="auto" requires stable source files and is not ' + "supported for file-like FIF inputs" + ) + if isinstance(fname, Path | str) and Path(fname).suffix == ".gz": + raise ValueError( + 'preload="auto" supports only uncompressed FIF files; use ' + "preload=True for gzip-compressed FIF" + ) raws = [] do_check_ext = not _file_like(fname) next_fname = fname diff --git a/mne/io/fiff/tests/test_raw_fiff.py b/mne/io/fiff/tests/test_raw_fiff.py index c996f187015..93298307978 100644 --- a/mne/io/fiff/tests/test_raw_fiff.py +++ b/mne/io/fiff/tests/test_raw_fiff.py @@ -2120,6 +2120,29 @@ def test_file_like(kind, preload, split, tmp_path): assert file_fid.closed +def test_file_like_auto_preload_rejected(tmp_path, monkeypatch): + """Test that automatic caching cannot misidentify a named stream.""" + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + monkeypatch.setenv("MNE_CACHE_DIR", str(cache_dir)) + stream = BytesIO(test_fif_fname.read_bytes()) + stream.name = str(test_fif_fname) + with pytest.raises(ValueError, match="stable source files"): + read_raw_fif(stream, preload="auto") + + +def test_compressed_auto_preload_rejected(tmp_path, monkeypatch): + """Test that gzip FIF does not advertise ineffective decoded caching.""" + cache_dir = tmp_path / "cache" + cache_dir.mkdir() + monkeypatch.setenv("MNE_CACHE_DIR", str(cache_dir)) + with pytest.raises(ValueError, match="uncompressed FIF"): + read_raw_fif(test_fif_gz_fname, preload="auto") + raw = read_raw_fif(test_fif_gz_fname, preload=False) + with pytest.raises(ValueError, match="uncompressed FIF"): + raw.load_data(memmap="auto") + + def test_str_like(): """Test handling with str-like objects.""" fname = pathlib.Path(test_fif_fname) diff --git a/mne/io/tests/test_preload_cache.py b/mne/io/tests/test_preload_cache.py new file mode 100644 index 00000000000..7e7bc7b012d --- /dev/null +++ b/mne/io/tests/test_preload_cache.py @@ -0,0 +1,679 @@ +"""Tests for persistent Raw preload-cache infrastructure.""" + +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +import errno +import gc +import hashlib +import json +import multiprocessing +import os +import shutil +import threading +import time +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor +from contextlib import chdir, nullcontext +from pathlib import Path + +import numpy as np +import pytest +from numpy.testing import assert_array_equal + +import mne +from mne._fiff.pick import pick_info +from mne.io import RawArray, _preload_cache +from mne.io.tests.test_raw import ( + _RawArange, + _read_raw_arange, +) + +_ORIGINAL_CACHE_REPLACE = None +_IO_DATA_DIR = Path(mne.io.__file__).parent + + +def _auto_preload_process(reader_name, source, cache_dir): + """Read one automatic cache entry in an isolated process.""" + os.environ["MNE_CACHE_DIR"] = cache_dir + raw = getattr(mne.io, reader_name)(source, preload="auto", verbose="error") + digest = hashlib.sha256(raw.get_data().tobytes()).hexdigest() + return raw._data.mode, str(raw._data.filename), digest + + +def _replace_cache_generation_then_exit(source, destination): + """Publish one generation and simulate an immediate process crash.""" + _ORIGINAL_CACHE_REPLACE(source, destination) + if str(destination).endswith(".data"): + os._exit(91) + + +def _auto_preload_crash_process(source, cache_dir): + """Crash a cache writer after its generation becomes durable.""" + global _ORIGINAL_CACHE_REPLACE + + os.environ["MNE_CACHE_DIR"] = cache_dir + _ORIGINAL_CACHE_REPLACE = _preload_cache.os.replace + _preload_cache.os.replace = _replace_cache_generation_then_exit + mne.io.read_raw_edf(source, preload="auto", verbose="error") + + +class _RawArangeBarrier(_RawArange): + _barrier = None + + def _read_segment(self, *args, **kwargs): + self._barrier.wait(timeout=2.0) + return super()._read_segment(*args, **kwargs) + + +class _RawArangeRecording(_RawArange): + _mappings = None + + def _read_segment(self, *args, **kwargs): + data = super()._read_segment(*args, **kwargs) + self._mappings.append(data) + return data + + +@pytest.fixture +def cache_root(tmp_path, monkeypatch): + """Configure and return an isolated preload cache.""" + cache_root = tmp_path / "cache" + cache_root.mkdir() + monkeypatch.setenv("MNE_CACHE_DIR", str(cache_root)) + return cache_root + + +@pytest.fixture +def auto_cache(tmp_path, cache_root): + """Create one stable source for the isolated preload cache.""" + source = tmp_path / "source.bin" + source.write_bytes(b"source identity") + return source, cache_root + + +def _fail_manifest_replace(source, destination): + raise OSError("injected manifest failure") + + +def _fail_memmap_flush(self): + raise OSError("injected flush failure") + + +def _raise_unsupported_fsync(descriptor): + raise OSError(errno.EINVAL, "unsupported") + + +def _raise_fsync_io_error(descriptor): + raise OSError(errno.EIO, "I/O failure") + + +def test_live_writer_lock_is_never_stale(tmp_path): + """Test that age alone cannot evict a positively live publisher.""" + lock = tmp_path / "entry.lock" + lock.write_text(f"{os.getpid()} token", encoding="ascii") + old = time.time() - 7200.0 + os.utime(lock, (old, old)) + assert not _preload_cache._raw_preload_remove_stale_lock(lock) + assert lock.is_file() + + +@pytest.mark.parametrize( + ("replacement", "error"), + ( + (_raise_unsupported_fsync, None), + (_raise_fsync_io_error, "I/O failure"), + ), +) +@pytest.mark.skipif(os.name == "nt", reason="directory fsync is POSIX-only") +def test_directory_fsync_unsupported_only(replacement, error, tmp_path, monkeypatch): + """Test narrow handling of filesystems without directory fsync.""" + monkeypatch.setattr(_preload_cache.os, "fsync", replacement) + context = nullcontext() if error is None else pytest.raises(OSError, match=error) + with context: + _preload_cache._raw_preload_fsync_directory(tmp_path) + + +def test_windows_identity_uses_change_time_and_file_id(tmp_path, monkeypatch): + """Test that Windows identity comes from the validated file handle.""" + source = tmp_path / "source.data" + source.write_bytes(b"source") + expected = dict( + size=len(b"source"), + mtime_ns=10, + change_ns=20, + device=30, + inode=40, + ) + descriptor = os.open(source, os.O_RDONLY) + try: + monkeypatch.setattr(_preload_cache, "_IS_WINDOWS", True) + monkeypatch.setattr( + _preload_cache, + "_raw_preload_windows_fstat", + lambda descriptor, result: expected, + raising=False, + ) + identity = _preload_cache._raw_preload_fstat(descriptor) + finally: + os.close(descriptor) + assert identity == expected + + +def test_windows_open_uses_reparse_safe_handle(tmp_path, monkeypatch): + """Test that Windows cache opens use the no-reparse helper.""" + source = tmp_path / "source.data" + source.write_bytes(b"source") + opened = [] + + def _open_windows(path): + opened.append(path) + return path.open("rb") + + monkeypatch.setattr(_preload_cache, "_IS_WINDOWS", True) + monkeypatch.setattr( + _preload_cache, + "_raw_preload_open_windows", + _open_windows, + raising=False, + ) + with _preload_cache._raw_preload_open_regular(source) as file: + assert file.read() == b"source" + assert opened == [source] + + +def test_windows_uses_configured_cache_directory(tmp_path, monkeypatch): + """Test that Windows can create the managed cache directory.""" + cache_root = tmp_path / "cache" + cache_root.mkdir() + path_class = _preload_cache.Path + monkeypatch.setattr(_preload_cache, "_IS_WINDOWS", True) + monkeypatch.setattr( + _preload_cache, + "Path", + lambda value: value if isinstance(value, path_class) else path_class(value), + ) + managed = _preload_cache._raw_preload_cache_dir(cache_root) + assert managed == cache_root / "raw-preload-v1" + assert managed.is_dir() + + +def test_windows_does_not_use_posix_fchmod(monkeypatch): + """Test that Windows publication avoids unavailable POSIX permissions.""" + monkeypatch.setattr(_preload_cache, "_IS_WINDOWS", True) + monkeypatch.setattr( + _preload_cache.os, + "fchmod", + lambda *args: pytest.fail("os.fchmod was called"), + ) + _preload_cache._raw_preload_protect_generation(1) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX directory permissions") +def test_cache_rejects_untrusted_ancestor(tmp_path): + """Test that another local user cannot replace the managed directory.""" + shared = tmp_path / "shared" + shared.mkdir(mode=0o777) + shared.chmod(0o777) + cache_root = shared / "cache" + cache_root.mkdir(mode=0o700) + with pytest.raises(PermissionError, match="untrusted writable ancestor"): + _preload_cache._raw_preload_cache_dir(cache_root) + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX directory permissions") +def test_cache_rejects_existing_public_managed_directory(tmp_path): + """Test that making an already exposed cache private is insufficient.""" + managed = tmp_path / "raw-preload-v1" + managed.mkdir(mode=0o777) + managed.chmod(0o777) + with pytest.raises(PermissionError, match="already be private"): + _preload_cache._raw_preload_cache_dir(tmp_path) + assert managed.stat().st_mode & 0o777 == 0o777 + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics") +def test_cache_canonicalizes_symlink_ancestor(tmp_path): + """Test that a configured symlink is fixed to one physical location.""" + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + try: + link.symlink_to(real, target_is_directory=True) + except OSError: + pytest.skip("symlink creation is unavailable") + managed = _preload_cache._raw_preload_cache_dir(link) + assert managed == real / "raw-preload-v1" + + +def test_distinct_keys_publish_concurrently(tmp_path, cache_root, monkeypatch): + """Test that unrelated first-time decodes do not share a long-held lock.""" + sources = [tmp_path / f"source-{index}.bin" for index in range(2)] + for index, source in enumerate(sources): + source.write_bytes(bytes([index])) + raws = [_RawArangeBarrier(preload=False, filename=source) for source in sources] + barrier = threading.Barrier(2) + monkeypatch.setattr(_RawArangeBarrier, "_barrier", barrier) + with ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(lambda raw: raw.load_data(memmap="auto"), raws)) + assert len({Path(raw._data.filename) for raw in raws}) == 2 + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX replacement semantics") +def test_generation_mapping_uses_validated_handle(tmp_path): + """Test that pathname replacement cannot change the mapped generation.""" + generation = tmp_path / "generation.data" + replacement = tmp_path / "replacement.data" + np.arange(4.0).tofile(generation) + np.full(4, 99.0).tofile(replacement) + with _preload_cache._raw_preload_open_regular(generation) as file: + os.replace(replacement, generation) + data = np.memmap(file, mode="c", dtype=np.float64, shape=(4,)) + np.testing.assert_array_equal(data, np.arange(4.0)) + + +def test_auto_preload_first_miss_is_copy_on_write(auto_cache, tmp_path): + """Test that an automatic cache miss publishes immutable data.""" + source, _ = auto_cache + with chdir(tmp_path): + raw = _RawArange(preload="auto", filename=source) + + assert isinstance(raw._data, np.memmap) + assert raw._data.mode == "c" + generation = Path(raw._data.filename) + expected = raw.get_data() + raw._data[0, 0] = 99.0 + del raw + gc.collect() + + with chdir(tmp_path): + other = _RawArange(preload="auto", filename=source) + assert other._data.mode == "c" + assert Path(other._data.filename) == generation + assert_array_equal(other.get_data(), expected) + assert not (tmp_path / "auto").exists() + + +def test_auto_preload_scavenges_same_key(auto_cache): + """Test that a retry removes abandoned files for its cache key.""" + source, _ = auto_cache + raw = _RawArange(preload="auto", filename=source) + cache_dir = Path(raw._data.filename).parent + manifest_path = next(cache_dir.glob("*.json")) + key = manifest_path.stem + orphan = cache_dir / f"{key}.{'0' * 32}.data" + temporary = cache_dir / f".{key}.abandoned.tmp" + orphan.write_bytes(b"orphan") + temporary.write_bytes(b"temporary") + manifest_path.unlink() + + other = _RawArange(preload="auto", filename=source) + assert_array_equal(other.get_data(), raw.get_data()) + assert len(list(cache_dir.glob(f"{key}.*.data"))) == 1 + assert not temporary.exists() + + +def test_auto_preload_cleans_failed_publication(auto_cache, monkeypatch): + """Test cleanup and retry after manifest publication fails.""" + source, cache_root = auto_cache + + with monkeypatch.context() as context: + context.setattr( + _preload_cache, "_raw_preload_replace_manifest", _fail_manifest_replace + ) + with pytest.raises(OSError, match="injected manifest failure"): + _RawArange(preload="auto", filename=source) + + cache_dir = cache_root / "raw-preload-v1" + assert list(cache_dir.iterdir()) == [] + raw = _RawArange(preload="auto", filename=source) + assert_array_equal(raw.get_data()[:, 0], np.arange(1, 9)) + + +def test_auto_preload_closes_failed_flush(auto_cache, monkeypatch): + """Test that a flush failure closes its temporary mapping.""" + source, cache_root = auto_cache + mappings = [] + monkeypatch.setattr(_RawArangeRecording, "_mappings", mappings) + monkeypatch.setattr(np.memmap, "flush", _fail_memmap_flush) + with pytest.raises(OSError, match="injected flush failure"): + _RawArangeRecording(preload="auto", filename=source) + + assert mappings[0]._mmap.closed + assert list((cache_root / "raw-preload-v1").iterdir()) == [] + + +def test_auto_preload_invalidates_mne_version(auto_cache, monkeypatch): + """Test that decoded cache data do not cross MNE version boundaries.""" + source, _ = auto_cache + raw = _RawArange(preload="auto", filename=source) + first = Path(raw._data.filename) + + monkeypatch.setattr(_preload_cache, "MNE_VERSION", "next-version") + other = _RawArange(preload="auto", filename=source) + assert Path(other._data.filename) != first + assert_array_equal(other.get_data(), raw.get_data()) + + +def test_auto_preload_api_contract(tmp_path, monkeypatch): + """Test automatic preload errors and the literal-path escape.""" + source = tmp_path / "source.bin" + source.write_bytes(b"source identity") + monkeypatch.setattr(_preload_cache, "get_config", lambda *args, **kwargs: None) + with pytest.raises(ValueError, match="set_cache_dir"): + _RawArange(preload="auto", filename=source) + raw = _RawArange(preload=False, filename=source) + with pytest.raises(ValueError, match="set_cache_dir"): + raw.load_data(memmap="auto") + identity_method = _RawArange._decoded_cache_identity + monkeypatch.setattr(_RawArange, "_decoded_cache_identity", lambda self: None) + monkeypatch.setattr( + _preload_cache, "get_config", lambda *args, **kwargs: str(tmp_path) + ) + with pytest.raises(ValueError, match="is not supported"): + _RawArange(preload="auto", filename=source) + + with chdir(tmp_path): + literal = _RawArange(preload=Path("auto"), filename=source) + assert literal._data.mode == "w+" + assert (tmp_path / "auto").is_file() + + monkeypatch.setattr(_RawArange, "_decoded_cache_identity", identity_method) + lazy = _RawArange(preload=False, filename=source) + lazy.load_data(memmap="auto") + assert lazy._data.mode == "c" + + +@pytest.mark.skipif(os.name == "nt", reason="POSIX file permissions") +def test_auto_preload_private_storage(auto_cache): + """Test private permissions for managed decoded data.""" + source, cache_root = auto_cache + raw = _RawArange(preload="auto", filename=source) + cache_dir = next(cache_root.iterdir()) + manifest = next(cache_dir.glob("*.json")) + generation = Path(raw._data.filename) + assert cache_dir.stat().st_mode & 0o077 == 0 + assert manifest.stat().st_mode & 0o077 == 0 + assert generation.stat().st_mode & 0o077 == 0 + assert generation.stat().st_mode & 0o222 == 0 + + +def test_auto_preload_rejects_cache_symlink(tmp_path, cache_root): + """Test that the private managed directory cannot be redirected.""" + source = tmp_path / "source.bin" + source.write_bytes(b"source identity") + outside = tmp_path / "outside" + outside.mkdir() + try: + os.symlink(outside, cache_root / "raw-preload-v1") + except OSError: + pytest.skip("symlink creation is unavailable") + with pytest.raises(OSError, match="regular directory"): + _RawArange(preload="auto", filename=source) + assert not list(outside.iterdir()) + + +@pytest.mark.parametrize( + ("reader_name", "relative_path"), + ( + ("read_raw_fif", "tests/data/test_raw.fif"), + ("read_raw_edf", "edf/tests/data/test.edf"), + ("read_raw_bdf", "edf/tests/data/test.bdf"), + ("read_raw_brainvision", "brainvision/tests/data/test.vhdr"), + ), +) +def test_auto_preload_cache_formats(reader_name, relative_path, cache_root): + """Test exact automatic preload reuse across supported formats.""" + source = _IO_DATA_DIR / relative_path + reader = getattr(mne.io, reader_name) + reference = reader(source, preload=True, verbose="error").get_data() + + raw = reader(source, preload="auto", verbose="error") + assert raw._data.mode == "c" + generation = Path(raw._data.filename) + assert_array_equal(raw.get_data(), reference) + raw._data[0, 0] += 1.0 + del raw + gc.collect() + + other = reader(source, preload="auto", verbose="error") + assert other._data.mode == "c" + assert Path(other._data.filename) == generation + assert_array_equal(other.get_data(), reference) + + +@pytest.mark.parametrize( + "corruption", + ( + "truncated_json", + "oversized_manifest", + "symlink_manifest", + "missing_field", + "unknown_field", + "version", + "traversal_generation", + "missing_generation", + "wrong_size_generation", + "same_size_generation", + "symlink_generation", + ), +) +def test_auto_preload_cache_corruption(corruption, auto_cache, tmp_path): + """Test that corrupt cache entries always become safe misses.""" + source, cache_root = auto_cache + raw = _RawArange(preload="auto", filename=source) + expected = raw.get_data() + del raw + cache_dir = next(cache_root.iterdir()) + manifest_path = next(cache_dir.glob("*.json")) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + generation = cache_dir / manifest["generation"] + + if corruption == "truncated_json": + manifest_path.write_text('{"version":', encoding="utf-8") + elif corruption == "oversized_manifest": + manifest_path.write_text(" " * 4097, encoding="utf-8") + elif corruption == "symlink_manifest": + outside = tmp_path / "outside.json" + outside.write_text(json.dumps(manifest), encoding="utf-8") + manifest_path.unlink() + try: + os.symlink(outside, manifest_path) + except OSError: + pytest.skip("symlink creation is unavailable") + elif corruption == "missing_field": + manifest.pop("version") + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + elif corruption == "unknown_field": + manifest["unknown"] = True + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + elif corruption == "version": + manifest[corruption] = -1 + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + elif corruption == "traversal_generation": + manifest["generation"] = f"../{manifest['generation']}" + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + elif corruption == "missing_generation": + generation.unlink() + elif corruption == "wrong_size_generation": + generation.chmod(0o600) + generation.write_bytes(b"short") + elif corruption == "same_size_generation": + generation.chmod(0o600) + stat = generation.stat() + with generation.open("r+b") as file: + file.write(b"\x00" * 8) + os.utime(generation, ns=(stat.st_atime_ns, stat.st_mtime_ns)) + else: + outside = tmp_path / "outside.dat" + outside.write_bytes(b"outside") + generation.unlink() + try: + os.symlink(outside, generation) + except OSError: + pytest.skip("symlink creation is unavailable") + + other = _RawArange(preload="auto", filename=source) + assert other._data.mode == "c" + assert_array_equal(other.get_data(), expected) + if corruption == "symlink_generation": + assert outside.read_bytes() == b"outside" + elif corruption == "symlink_manifest": + assert json.loads(outside.read_text(encoding="utf-8")) == manifest + + +def test_auto_preload_concurrent_misses(cache_root): + """Test that concurrent misses publish one exact generation.""" + source = _IO_DATA_DIR / "edf/tests/data/test.edf" + args = ("read_raw_edf", str(source), str(cache_root)) + with ProcessPoolExecutor(max_workers=4) as pool: + results = list(pool.map(_auto_preload_process, *zip(*(args,) * 4))) + + assert {result[0] for result in results} == {"c"} + assert len({result[1] for result in results}) == 1 + assert len({result[2] for result in results}) == 1 + cache_dir = next(cache_root.iterdir()) + assert len(list(cache_dir.glob("*.data"))) == 1 + assert not list(cache_dir.glob("*.tmp")) + + +def test_auto_preload_recovers_crashed_publisher(cache_root): + """Test recovery when a writer dies before manifest publication.""" + source = _IO_DATA_DIR / "edf/tests/data/test.edf" + context = multiprocessing.get_context("spawn") + process = context.Process( + target=_auto_preload_crash_process, args=(str(source), str(cache_root)) + ) + process.start() + process.join(timeout=15) + assert process.exitcode == 91 + cache_dir = next(cache_root.iterdir()) + assert len(list(cache_dir.glob("*.data"))) == 1 + assert len(list(cache_dir.glob("*.lock"))) == 1 + assert not list(cache_dir.glob("*.json")) + + raw = mne.io.read_raw_edf(source, preload="auto", verbose="error") + reference = mne.io.read_raw_edf(source, preload=True, verbose="error") + assert raw._data.mode == "c" + assert_array_equal(raw.get_data(), reference.get_data()) + assert not list(cache_dir.glob("*.lock")) + assert not list(cache_dir.glob("*.tmp")) + assert len(list(cache_dir.glob("*.data"))) == 1 + + +def test_auto_preload_identity_ignores_edf_channel_type(cache_root): + """Test that live metadata does not invalidate decoded samples.""" + source = _IO_DATA_DIR / "edf/tests/data/test.edf" + raw = mne.io.read_raw_edf(source, preload="auto", verbose="error") + generation = Path(raw._data.filename) + + other = mne.io.read_raw_edf(source, eog=[0], preload="auto", verbose="error") + assert Path(other._data.filename) == generation + assert other.get_channel_types()[0] == "eog" + assert_array_equal(other.get_data(), raw.get_data()) + + +def test_auto_preload_brainvision_live_markers(tmp_path, cache_root): + """Test that markers remain live while decoded samples are reused.""" + data_dir = _IO_DATA_DIR / "brainvision/tests/data" + for name in ("test.vhdr", "test.vmrk", "test.eeg"): + shutil.copy(data_dir / name, tmp_path / name) + source = tmp_path / "test.vhdr" + raw = mne.io.read_raw_brainvision(source, preload="auto", verbose="error") + generation = Path(raw._data.filename) + annotation_count = len(raw.annotations) + with (tmp_path / "test.vmrk").open("a", encoding="utf-8") as file: + file.write("\nMk15=Stimulus,S 99,7800,1,0\n") + + other = mne.io.read_raw_brainvision(source, preload="auto", verbose="error") + assert Path(other._data.filename) == generation + assert len(other.annotations) == annotation_count + 1 + assert other.annotations.description[-1] == "Stimulus/S 99" + plain = mne.io.read_raw_brainvision( + source, ignore_marker_types=True, preload="auto", verbose="error" + ) + assert Path(plain._data.filename) == generation + assert plain.annotations.description[-1] == "S 99" + + +def test_auto_preload_numeric_invalidation(tmp_path, cache_root): + """Test numeric options and filesystem changes invalidate cached data.""" + data_dir = _IO_DATA_DIR / "brainvision/tests/data" + source = data_dir / "test.vhdr" + raw = mne.io.read_raw_brainvision(source, preload="auto", verbose="error") + scaled = mne.io.read_raw_brainvision( + source, scale=2.0, preload="auto", verbose="error" + ) + assert Path(scaled._data.filename) != Path(raw._data.filename) + assert_array_equal(scaled.get_data(), 2.0 * raw.get_data()) + + edf_source = tmp_path / "test.edf" + shutil.copy(_IO_DATA_DIR / "edf/tests/data/test.edf", edf_source) + original = mne.io.read_raw_edf(edf_source, preload="auto", verbose="error") + generation = Path(original._data.filename) + excluded = mne.io.read_raw_edf( + edf_source, + exclude=[original.ch_names[0]], + preload="auto", + verbose="error", + ) + assert Path(excluded._data.filename) != generation + assert excluded._data.shape[0] == original._data.shape[0] - 1 + stat = edf_source.stat() + with edf_source.open("r+b") as file: + file.seek(-1, os.SEEK_END) + byte = file.read(1) + file.seek(-1, os.SEEK_END) + file.write(bytes([byte[0] ^ 1])) + os.utime(edf_source, ns=(stat.st_atime_ns, stat.st_mtime_ns)) + changed = mne.io.read_raw_edf(edf_source, preload="auto", verbose="error") + assert Path(changed._data.filename) != generation + + alias = tmp_path / "alias.edf" + try: + os.symlink(edf_source, alias) + except OSError: + pytest.skip("symlink creation is unavailable") + aliased = mne.io.read_raw_edf(alias, preload="auto", verbose="error") + assert Path(aliased._data.filename) == Path(changed._data.filename) + + +@pytest.mark.parametrize("attribute", ("_projector", "_comp")) +def test_auto_preload_transform_invalidation(attribute, auto_cache): + """Test delayed projection and compensation use distinct cache entries.""" + source, _ = auto_cache + transformed = _read_raw_arange(filename=source) + setattr(transformed, attribute, 2.0 * np.eye(len(transformed.ch_names))) + transformed.load_data(memmap="auto", verbose="error") + plain = _read_raw_arange(filename=source) + plain.load_data(memmap="auto", verbose="error") + + assert Path(transformed._data.filename) != Path(plain._data.filename) + assert_array_equal(transformed.get_data(), 2.0 * plain.get_data()) + + +def test_add_channels_copy_on_write_memmap(tmp_path, monkeypatch): + """Test adding channels to a copy-on-write memmap.""" + from mne.channels import channels as channels_module + + memmap_fname = tmp_path / "raw-copy-on-write-memmap.dat" + raw = _read_raw_arange(preload=memmap_fname) + shape = raw._data.shape + expected = raw._data.copy() + raw._data._mmap.close() + raw._data = np.memmap(memmap_fname, mode="c", dtype=np.float64, shape=shape) + raw._data[0, 0] = 99.0 + + info = pick_info(raw.info, [0]) + mne.rename_channels(info, {info["ch_names"][0]: "extra"}) + extra = RawArray(np.zeros((1, raw.n_times)), info) + monkeypatch.setattr(channels_module.sys, "platform", "linux") + raw.add_channels([extra]) + + assert not isinstance(raw._data, np.memmap) + assert raw._data.shape == (shape[0] + 1, shape[1]) + expected[0, 0] = 99.0 + assert_array_equal(raw._data[:-1], expected) + stored = np.memmap(memmap_fname, mode="r", dtype=np.float64, shape=shape) + assert stored[0, 0] != 99.0 + stored._mmap.close() diff --git a/mne/io/tests/test_raw.py b/mne/io/tests/test_raw.py index 0f4b47d6e80..53ec6444040 100644 --- a/mne/io/tests/test_raw.py +++ b/mne/io/tests/test_raw.py @@ -48,6 +48,20 @@ ) +def _fail_if_times_materialized(*args, **kwargs): + pytest.fail("The full Raw.times vector was materialized") + + +class _CropRecorder: + def __init__(self): + self.args = None + self.kwargs = None + + def crop(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def assert_named_constants(info): """Assert that info['chs'] has named constants.""" # for now we just check one @@ -99,6 +113,37 @@ def test_orig_units(): BaseRaw(info, last_samps=[1], orig_units=True) +def test_preload_does_not_materialize_times(monkeypatch): + """Test preloading does not construct the full time vector.""" + monkeypatch.setattr("mne.io.base._arange_div", _fail_if_times_materialized) + raw = read_raw_fif(raw_fname, preload=True, verbose="error") + assert raw.preload + + +def test_set_annotations_does_not_materialize_times(monkeypatch): + """Test annotation bounds use the scalar recording endpoint.""" + raw = read_raw_fif(raw_fname, preload=False, verbose="error") + annotations = Annotations([0.0], [0.1], ["test"]) + monkeypatch.setattr("mne.io.base._arange_div", _fail_if_times_materialized) + raw.set_annotations(annotations) + assert len(raw.annotations) == 1 + + +def test_set_annotations_preserves_endpoint_arithmetic(monkeypatch): + """Test annotation bounds preserve the prior floating-point operations.""" + raw = RawArray(np.zeros((1, 6)), create_info(1, 100.0), verbose="error") + annotations = Annotations([0.0], [0.0], ["test"]) + recorder = _CropRecorder() + monkeypatch.setattr(Annotations, "crop", recorder.crop) + + raw.set_annotations(annotations) + + endpoint = (raw.n_times - 1) / raw.info["sfreq"] + 1.0 / raw.info["sfreq"] + assert endpoint != raw.duration + assert recorder.args == (0, endpoint) + assert recorder.kwargs == {"emit_warning": True} + + def _test_raw_reader( reader, test_preloading=True, @@ -824,9 +869,15 @@ def test_repr(sfreq): # A class that sets channel data to np.arange, for testing _test_raw_reader class _RawArange(BaseRaw): - def __init__(self, preload=False, verbose=None): + def __init__(self, preload=False, filename=None, verbose=None): info = create_info(list(str(x) for x in range(1, 9)), 1000.0, "eeg") - super().__init__(info, preload, last_samps=(999,), verbose=verbose) + super().__init__( + info, + preload, + last_samps=(999,), + filenames=(filename,), + verbose=verbose, + ) assert len(self.times) == 1000 def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): @@ -834,9 +885,12 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): one[idx] = np.arange(1, 9)[idx, np.newaxis] _mult_cal_one(data, one, idx, cals, mult) + def _decoded_cache_identity(self): + return (1, ()) + -def _read_raw_arange(preload=False, verbose=None): - return _RawArange(preload, verbose) +def _read_raw_arange(preload=False, filename=None, verbose=None): + return _RawArange(preload, filename=filename, verbose=verbose) @pytest.mark.parametrize("method", ("constructor", "load_data")) diff --git a/mne/utils/config.py b/mne/utils/config.py index ba0cb50af9d..92597d84344 100644 --- a/mne/utils/config.py +++ b/mne/utils/config.py @@ -40,17 +40,24 @@ class UnknownPlatformError(Exception): def set_cache_dir(cache_dir): - """Set the directory to be used for temporary file storage. + """Set the directory used for temporary and managed cache storage. - This directory is used by joblib to store memmapped arrays, - which reduces memory requirements and speeds up parallel - computation. + This directory is used by joblib to store temporary memmapped arrays and, + when requested by supported Raw readers, to persist decoded preload data. Parameters ---------- cache_dir : str or None - Directory to use for temporary file storage. None disables - temporary file storage. + Directory to use for cache storage. None disables cache storage. + + Notes + ----- + Persistent decoded Raw entries are not automatically size-limited. They are + stored below ``cache_dir`` in a versioned ``raw-preload`` directory. + On POSIX, physical ancestors must be owned by the current user or root, and + writable shared parents require sticky-directory semantics. On Windows, use + a private local cache directory controlled by the current account; MNE does + not verify its ACL. """ if cache_dir is not None and not op.exists(cache_dir): raise OSError(f"Directory {cache_dir} does not exist") @@ -109,7 +116,7 @@ def set_memmap_min_size(memmap_min_size): "MNE_BROWSER_USE_OPENGL": ( "bool, whether to use OpenGL for rendering in the raw browser" ), - "MNE_CACHE_DIR": "str, path to the cache directory for parallel execution", + "MNE_CACHE_DIR": "str, path to the temporary and managed cache directory", "MNE_COREG_ADVANCED_RENDERING": ( "bool, whether to use advanced OpenGL rendering in coreg" ), diff --git a/mne/utils/docs.py b/mne/utils/docs.py index 8ae9ff682ba..f9416c9bd9f 100644 --- a/mne/utils/docs.py +++ b/mne/utils/docs.py @@ -3721,7 +3721,19 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75): freshly created memory-mapped file used to store the data on the hard drive (slower, requires less memory). An existing file is overwritten. The caller owns the file and is responsible for removing it after the - Raw object is no longer in use.""" + Raw object is no longer in use. For uncompressed FIF, EDF/BDF, and + BrainVision readers, the exact string ``"auto"`` instead stores and reuses + decoded data in the directory configured by :func:`mne.set_cache_dir`. + Cached data persist in a versioned ``raw-preload`` directory below the + configured cache path and are mapped copy-on-write, so modifying the returned + Raw does not modify later reads. A cache miss performs the normal full decode. + Valid entries for historical source or decoder identities are retained without + an automatic size limit. The configured cache path is fixed to its physical + location. Use ``Path("auto")``, + ``"./auto"``, or an absolute path to create a file literally named ``auto``. + + .. versionchanged:: 1.13 + Support for the ``"auto"`` decoded-data cache was added.""" docdict["preload_concatenate"] = """ preload : bool | str | None From 53a65f9ee882007bea2e53882f75181c7be3870c Mon Sep 17 00:00:00 2001 From: Bru Date: Thu, 27 Aug 2026 10:21:52 +0200 Subject: [PATCH 2/9] Simplify portable Raw preload cache locking --- mne/io/_preload_cache.py | 457 +++-------------------------- mne/io/tests/test_preload_cache.py | 192 ++++-------- mne/utils/config.py | 6 +- pyproject.toml | 2 +- 4 files changed, 97 insertions(+), 560 deletions(-) diff --git a/mne/io/_preload_cache.py b/mne/io/_preload_cache.py index e9e62656931..88993f7641d 100644 --- a/mne/io/_preload_cache.py +++ b/mne/io/_preload_cache.py @@ -4,197 +4,32 @@ # License: BSD-3-Clause # Copyright the MNE-Python contributors. -import ctypes -import errno import hashlib import json import os import pickle import stat -import time -from ctypes import wintypes from pathlib import Path import numpy as np from .. import __version__ as MNE_VERSION # ty: ignore[unresolved-import] -from ..utils import get_config, logger +from ..utils import _soft_import, get_config, logger _RAW_PRELOAD_CACHE_VERSION = 1 _RAW_PRELOAD_LOCK_TIMEOUT = 300.0 -_IS_WINDOWS = os.name == "nt" - -if _IS_WINDOWS: - import msvcrt - -_FILE_ATTRIBUTE_DIRECTORY = 0x00000010 -_FILE_ATTRIBUTE_REPARSE_POINT = 0x00000400 -_FILE_ATTRIBUTE_TAG_INFO = 9 -_FILE_BASIC_INFO = 0 -_FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000 -_FILE_ID_INFO = 18 -_FILE_SHARE_ALL = 0x00000007 -_GENERIC_READ = 0x80000000 -_OPEN_EXISTING = 3 -_PROCESS_SYNCHRONIZE = 0x00100000 -_ERROR_INVALID_PARAMETER = 87 -_WAIT_OBJECT_0 = 0 - - -class _FileBasicInfo(ctypes.Structure): - _fields_ = ( - ("creation_time", ctypes.c_longlong), - ("last_access_time", ctypes.c_longlong), - ("last_write_time", ctypes.c_longlong), - ("change_time", ctypes.c_longlong), - ("file_attributes", wintypes.DWORD), - ) - - -class _FileId128(ctypes.Structure): - _fields_ = (("identifier", ctypes.c_ubyte * 16),) - - -class _FileIdInfo(ctypes.Structure): - _fields_ = ( - ("volume_serial_number", ctypes.c_ulonglong), - ("file_id", _FileId128), - ) - - -def _raw_preload_windows_fstat(descriptor, result): - """Return Windows change time and file ID from an open handle.""" - kernel32 = getattr(ctypes, "WinDLL")("kernel32", use_last_error=True) - handle = getattr(msvcrt, "get_osfhandle")(descriptor) - basic = _FileBasicInfo() - get_ex = kernel32.GetFileInformationByHandleEx - get_ex.argtypes = ( - wintypes.HANDLE, - ctypes.c_int, - wintypes.LPVOID, - wintypes.DWORD, - ) - get_ex.restype = wintypes.BOOL - if not get_ex(handle, _FILE_BASIC_INFO, ctypes.byref(basic), ctypes.sizeof(basic)): - raise getattr(ctypes, "WinError")(getattr(ctypes, "get_last_error")()) - file_info = _FileIdInfo() - if not get_ex( - handle, _FILE_ID_INFO, ctypes.byref(file_info), ctypes.sizeof(file_info) - ): - raise getattr(ctypes, "WinError")(getattr(ctypes, "get_last_error")()) - return dict( - size=int(result.st_size), - mtime_ns=int(basic.last_write_time) * 100, - change_ns=int(basic.change_time) * 100, - device=int(file_info.volume_serial_number), - inode=int.from_bytes(bytes(file_info.file_id.identifier), "little"), - ) - - -def _raw_preload_fstat(descriptor): - """Return an identity token for an already-open regular file.""" - result = os.fstat(descriptor) - if not stat.S_ISREG(result.st_mode): - raise OSError("Decoded data cache entries must be regular files") - if _IS_WINDOWS: - return _raw_preload_windows_fstat(descriptor, result) - return dict( - size=int(result.st_size), - mtime_ns=int(result.st_mtime_ns), - change_ns=int(result.st_ctime_ns), - device=int(result.st_dev), - inode=int(result.st_ino), - ) - - -def _raw_preload_open_windows(path): - """Open a Windows file handle without traversing a reparse point.""" - kernel32 = getattr(ctypes, "WinDLL")("kernel32", use_last_error=True) - create_file = kernel32.CreateFileW - create_file.argtypes = ( - wintypes.LPCWSTR, - wintypes.DWORD, - wintypes.DWORD, - wintypes.LPVOID, - wintypes.DWORD, - wintypes.DWORD, - wintypes.HANDLE, - ) - create_file.restype = wintypes.HANDLE - handle = create_file( - str(path), - _GENERIC_READ, - _FILE_SHARE_ALL, - None, - _OPEN_EXISTING, - _FILE_FLAG_OPEN_REPARSE_POINT, - None, - ) - invalid_handle = ctypes.c_void_p(-1).value - if handle == invalid_handle: - raise getattr(ctypes, "WinError")(getattr(ctypes, "get_last_error")()) - close_handle = kernel32.CloseHandle - close_handle.argtypes = (wintypes.HANDLE,) - close_handle.restype = wintypes.BOOL - try: - attributes = wintypes.DWORD() - get_attributes = kernel32.GetFileInformationByHandleEx - get_attributes.argtypes = ( - wintypes.HANDLE, - ctypes.c_int, - wintypes.LPVOID, - wintypes.DWORD, - ) - get_attributes.restype = wintypes.BOOL - # FileAttributeTagInfo is 9; its first DWORD contains the attributes. - attribute_tag_info = (wintypes.DWORD * 2)() - if not get_attributes( - handle, - _FILE_ATTRIBUTE_TAG_INFO, - ctypes.byref(attribute_tag_info), - ctypes.sizeof(attribute_tag_info), - ): - raise getattr(ctypes, "WinError")(getattr(ctypes, "get_last_error")()) - attributes.value = attribute_tag_info[0] - if attributes.value & _FILE_ATTRIBUTE_REPARSE_POINT: - raise OSError( - f"Decoded data cache entries cannot be reparse points: {path}" - ) - if attributes.value & _FILE_ATTRIBUTE_DIRECTORY: - raise OSError(f"Decoded data cache entries must be regular files: {path}") - descriptor = getattr(msvcrt, "open_osfhandle")( - handle, os.O_RDONLY | getattr(os, "O_BINARY") - ) - except Exception: - close_handle(handle) - raise - return os.fdopen(descriptor, "rb") def _raw_preload_open_regular(path): - """Open a regular cache file without following a final-component link.""" - if _IS_WINDOWS: - return _raw_preload_open_windows(path) - flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) - try: - flags |= os.O_NOFOLLOW - except AttributeError as error: - raise PermissionError( - "Automatic Raw preload caching requires no-follow file opens" - ) from error - descriptor = os.open(path, flags) + """Open and validate a regular cache file.""" + file = open(path, "rb") try: - _raw_preload_fstat(descriptor) + if not stat.S_ISREG(os.fstat(file.fileno()).st_mode): + raise OSError("Decoded data cache entries must be regular files") except Exception: - os.close(descriptor) + file.close() raise - return os.fdopen(descriptor, "rb") - - -def _raw_preload_path_stat(path): - """Return the identity token for a path using one validated handle.""" - with _raw_preload_open_regular(path) as file: - return _raw_preload_fstat(file.fileno()) + return file def _raw_preload_source_signature(raw): @@ -207,7 +42,18 @@ def _raw_preload_source_signature(raw): "or an explicit memory-map path" ) path = Path(filename).resolve(strict=True) - sources.append(dict(path=str(path), **_raw_preload_path_stat(path))) + result = path.stat() + if not stat.S_ISREG(result.st_mode): + raise OSError("Raw source data must be regular files") + sources.append( + dict( + path=str(path), + size=int(result.st_size), + mtime_ns=int(result.st_mtime_ns), + device=int(result.st_dev), + inode=int(result.st_ino), + ) + ) return sources @@ -225,46 +71,9 @@ def _raw_preload_cache_dir(cache_root=None): cache_dir.mkdir(mode=0o700, parents=True, exist_ok=True) if cache_dir.is_symlink() or not cache_dir.is_dir(): raise OSError(f"Decoded data cache must be a regular directory: {cache_dir}") - if not _IS_WINDOWS: - _raw_preload_validate_directory(cache_dir) - cache_dir.chmod(0o700) - if cache_dir.stat().st_mode & 0o077: - raise PermissionError(f"Decoded data cache is not private: {cache_dir}") return cache_dir -def _raw_preload_validate_directory(cache_dir): - """Reject cache paths that another local user could replace.""" - user_id = os.geteuid() - child_stat = cache_dir.lstat() - if stat.S_ISLNK(child_stat.st_mode) or not stat.S_ISDIR(child_stat.st_mode): - raise OSError(f"Decoded data cache must be a regular directory: {cache_dir}") - if child_stat.st_uid != user_id: - raise PermissionError( - f"Decoded data cache must be owned by the current user: {cache_dir}" - ) - if child_stat.st_mode & 0o077: - raise PermissionError( - f"Decoded data cache must already be private: {cache_dir}" - ) - for parent in cache_dir.parents: - parent_stat = parent.lstat() - if stat.S_ISLNK(parent_stat.st_mode) or not stat.S_ISDIR(parent_stat.st_mode): - raise PermissionError( - f"Decoded data cache cannot have a symlink ancestor: {parent}" - ) - if parent_stat.st_uid not in (0, user_id): - raise PermissionError( - "Decoded data cache cannot have an untrusted owner in its " - f"physical ancestry: {parent}" - ) - if parent_stat.st_mode & 0o022 and not parent_stat.st_mode & stat.S_ISVTX: - raise PermissionError( - "Decoded data cache cannot have an untrusted writable ancestor: " - f"{parent}" - ) - - def _raw_preload_cache_info(raw): """Return the managed cache location and expected array description.""" cache_identity = raw._decoded_cache_identity() @@ -304,11 +113,6 @@ def _raw_preload_cache_info(raw): return cache_dir, key, sources, shape, dtype -def _raw_preload_generation_name(key, token): - """Return a unique immutable generation basename.""" - return f"{key}.{token}.data" - - def _raw_preload_generation_valid(name, key): """Check that a manifest generation is a managed basename.""" prefix = f"{key}." @@ -327,8 +131,7 @@ def _raw_preload_read_manifest(cache_dir, key): """Read one manifest through its validated handle.""" path = cache_dir / f"{key}.json" with _raw_preload_open_regular(path) as file: - file_stat = _raw_preload_fstat(file.fileno()) - if file_stat["size"] > 4096: + if os.fstat(file.fileno()).st_size > 4096: raise ValueError("Oversized Raw preload manifest") return json.loads(file.read().decode("utf-8")) @@ -337,26 +140,14 @@ def _raw_preload_cache_read(raw, cache_dir, key, sources, shape, dtype): """Read and validate one managed decoded-data cache entry.""" try: manifest = _raw_preload_read_manifest(cache_dir, key) - if set(manifest) != { - "version", - "generation", - "generation_stat", - }: + if set(manifest) != {"generation"}: return None nbytes = int(np.prod(shape, dtype=np.int64)) * dtype.itemsize - if manifest[ - "version" - ] != _RAW_PRELOAD_CACHE_VERSION or not _raw_preload_generation_valid( - manifest["generation"], key - ): + if not _raw_preload_generation_valid(manifest["generation"], key): return None generation = cache_dir / manifest["generation"] with _raw_preload_open_regular(generation) as file: - generation_stat = _raw_preload_fstat(file.fileno()) - if ( - generation_stat != manifest["generation_stat"] - or generation_stat["size"] != nbytes - ): + if os.fstat(file.fileno()).st_size != nbytes: return None data = np.memmap(file, mode="c", dtype=dtype, shape=shape) data.filename = str(generation) # ty: ignore[invalid-assignment] @@ -369,138 +160,6 @@ def _raw_preload_cache_read(raw, cache_dir, key, sources, shape, dtype): return data -def _raw_preload_fsync_directory(path): - """Sync publication metadata where directory fsync is supported.""" - if _IS_WINDOWS: - return - descriptor = os.open(path, os.O_RDONLY) - try: - try: - os.fsync(descriptor) - except OSError as error: - unsupported = {errno.EINVAL, getattr(errno, "ENOTSUP", errno.EINVAL)} - if error.errno not in unsupported: - raise - finally: - os.close(descriptor) - - -def _raw_preload_protect_generation(descriptor): - """Make a generation owner-read-only where file modes support it.""" - if not _IS_WINDOWS: - os.fchmod(descriptor, 0o400) - - -def _raw_preload_replace_manifest(source, destination): - """Atomically replace a manifest, tolerating transient Windows locks.""" - for attempt in range(5): - try: - os.replace(source, destination) - except PermissionError: - if attempt == 4: - raise - time.sleep(0.01 * (attempt + 1)) - else: - return - - -def _raw_preload_process_alive(process_id): - """Return whether a local process still owns a cache publication lock.""" - if process_id <= 0: - return False - if _IS_WINDOWS: - kernel32 = getattr(ctypes, "WinDLL")("kernel32", use_last_error=True) - open_process = kernel32.OpenProcess - open_process.argtypes = (wintypes.DWORD, wintypes.BOOL, wintypes.DWORD) - open_process.restype = wintypes.HANDLE - wait = kernel32.WaitForSingleObject - wait.argtypes = (wintypes.HANDLE, wintypes.DWORD) - wait.restype = wintypes.DWORD - close = kernel32.CloseHandle - close.argtypes = (wintypes.HANDLE,) - close.restype = wintypes.BOOL - handle = open_process(_PROCESS_SYNCHRONIZE, False, process_id) - if not handle: - # Access denied is known-alive; unknown failures are also treated - # conservatively. ERROR_INVALID_PARAMETER is the invalid-PID case. - return getattr(ctypes, "get_last_error")() != _ERROR_INVALID_PARAMETER - try: - result = wait(handle, 0) - if result == _WAIT_OBJECT_0: - return False - return True # WAIT_TIMEOUT, WAIT_FAILED, or an unknown result - finally: - close(handle) - try: - os.kill(process_id, 0) - except ProcessLookupError: - return False - except (OSError, PermissionError): - return True - return True - - -def _raw_preload_try_lock(lock_path, owner): - """Try to claim a cache lock without blocking.""" - try: - descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) - except FileExistsError: - return False - try: - with os.fdopen(descriptor, "w", encoding="ascii") as file: - file.write(owner) - file.flush() - os.fsync(file.fileno()) - except Exception: - lock_path.unlink(missing_ok=True) - raise - return True - - -def _raw_preload_remove_stale_lock(lock_path): - """Remove a dead or abandoned cache lock without touching its target.""" - try: - lock_stat = lock_path.lstat() - if lock_path.is_symlink(): - stale = True - else: - age = time.time() - lock_stat.st_mtime - try: - content = lock_path.read_text(encoding="ascii") - if len(content) > 256: - raise ValueError - process_id = int(content.split()[0]) - alive = _raw_preload_process_alive(process_id) - except (IndexError, OSError, OverflowError, ValueError): - stale = age > 5.0 - else: - stale = not alive - current = lock_path.lstat() - unchanged = (current.st_dev, current.st_ino, current.st_mtime_ns) == ( - lock_stat.st_dev, - lock_stat.st_ino, - lock_stat.st_mtime_ns, - ) - if stale and unchanged: - lock_path.unlink() - return True - except FileNotFoundError: - return True - return False - - -def _raw_preload_release_lock(lock_path, owner): - """Release a cache lock only when it is still owned by this process.""" - try: - if ( - not lock_path.is_symlink() - and lock_path.read_text(encoding="ascii") == owner - ): - lock_path.unlink() - except FileNotFoundError: - pass - - def _raw_preload_scavenge_key(cache_dir, key): """Remove abandoned temporary and unreferenced same-key generations.""" referenced = None @@ -527,7 +186,7 @@ def _raw_preload_scavenge_key(cache_dir, key): def _raw_preload_cache_create(raw, cache_dir, key, sources, shape, dtype): """Decode, durably publish, and reopen an immutable cache generation.""" token = os.urandom(16).hex() - generation_name = _raw_preload_generation_name(key, token) + generation_name = f"{key}.{token}.data" generation = cache_dir / generation_name temporary = cache_dir / f".{generation_name}.tmp" manifest_temporary = None @@ -537,32 +196,19 @@ def _raw_preload_cache_create(raw, cache_dir, key, sources, shape, dtype): try: with os.fdopen(descriptor, "r+b") as file: file.truncate(nbytes) - data_buffer = np.memmap(file, mode="r+", dtype=dtype, shape=shape) - data = raw._read_segment(data_buffer=data_buffer) + data = np.memmap(file, mode="r+", dtype=dtype, shape=shape) try: + raw._read_segment(data_buffer=data) data.flush() - except BaseException: - try: - data._mmap.close() # memmap private - except Exception: - pass - raise - else: - data._mmap.close() # memmap private - del data, data_buffer - _raw_preload_protect_generation(file.fileno()) + finally: + data._mmap.close() # ty: ignore[unresolved-attribute] # memmap private os.fsync(file.fileno()) if _raw_preload_source_signature(raw) != sources: raise RuntimeError( "Source data changed while decoded cache was created; retry" ) os.replace(temporary, generation) - _raw_preload_fsync_directory(cache_dir) - manifest = dict( - version=_RAW_PRELOAD_CACHE_VERSION, - generation=generation_name, - generation_stat=_raw_preload_path_stat(generation), - ) + manifest = dict(generation=generation_name) manifest_temporary = cache_dir / f".{key}.{os.urandom(16).hex()}.json.tmp" descriptor = os.open( manifest_temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600 @@ -571,14 +217,9 @@ def _raw_preload_cache_create(raw, cache_dir, key, sources, shape, dtype): json.dump(manifest, file, sort_keys=True, separators=(",", ":")) file.flush() os.fsync(file.fileno()) - _raw_preload_replace_manifest(manifest_temporary, cache_dir / f"{key}.json") + os.replace(manifest_temporary, cache_dir / f"{key}.json") manifest_published = True - _raw_preload_fsync_directory(cache_dir) with _raw_preload_open_regular(generation) as file: - if _raw_preload_fstat(file.fileno()) != manifest["generation_stat"]: - raise RuntimeError( - "Decoded cache generation changed during publication" - ) result = np.memmap(file, mode="c", dtype=dtype, shape=shape) result.filename = str(generation) # ty: ignore[invalid-assignment] return result @@ -600,29 +241,17 @@ def _raw_preload_auto(raw): """Reuse or create an automatic decoded-data cache entry.""" cache_dir, key, sources, shape, dtype = _raw_preload_cache_info(raw) key_lock = cache_dir / f"{key}.lock" - owner = f"{os.getpid()} {os.urandom(16).hex()}" - deadline = time.monotonic() + _RAW_PRELOAD_LOCK_TIMEOUT - while True: + data = _raw_preload_cache_read(raw, cache_dir, key, sources, shape, dtype) + if data is not None: + return data + # Importing filelock is measurable, so keep it off the cache-hit path. + filelock = _soft_import("filelock", "locking the decoded-data cache") + + with filelock.FileLock(key_lock, timeout=_RAW_PRELOAD_LOCK_TIMEOUT): + _raw_preload_scavenge_key(cache_dir, key) data = _raw_preload_cache_read(raw, cache_dir, key, sources, shape, dtype) - if data is not None: - return data - if _raw_preload_try_lock(key_lock, owner): - try: - _raw_preload_scavenge_key(cache_dir, key) - data = _raw_preload_cache_read( - raw, cache_dir, key, sources, shape, dtype - ) - if data is None: - logger.info(f"Creating decoded data cache in {cache_dir}") - data = _raw_preload_cache_create( - raw, cache_dir, key, sources, shape, dtype - ) - _raw_preload_scavenge_key(cache_dir, key) - return data - finally: - _raw_preload_release_lock(key_lock, owner) - elif _raw_preload_remove_stale_lock(key_lock): - continue - if time.monotonic() >= deadline: - raise TimeoutError(f"Timed out waiting for decoded cache lock {key_lock}") - time.sleep(0.025) + if data is None: + logger.info(f"Creating decoded data cache in {cache_dir}") + data = _raw_preload_cache_create(raw, cache_dir, key, sources, shape, dtype) + _raw_preload_scavenge_key(cache_dir, key) + return data diff --git a/mne/io/tests/test_preload_cache.py b/mne/io/tests/test_preload_cache.py index 7e7bc7b012d..972d54973e0 100644 --- a/mne/io/tests/test_preload_cache.py +++ b/mne/io/tests/test_preload_cache.py @@ -4,7 +4,6 @@ # License: BSD-3-Clause # Copyright the MNE-Python contributors. -import errno import gc import hashlib import json @@ -12,9 +11,8 @@ import os import shutil import threading -import time from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor -from contextlib import chdir, nullcontext +from contextlib import chdir from pathlib import Path import numpy as np @@ -92,7 +90,7 @@ def auto_cache(tmp_path, cache_root): return source, cache_root -def _fail_manifest_replace(source, destination): +def _fail_manifest_dump(*args, **kwargs): raise OSError("injected manifest failure") @@ -100,135 +98,37 @@ def _fail_memmap_flush(self): raise OSError("injected flush failure") -def _raise_unsupported_fsync(descriptor): - raise OSError(errno.EINVAL, "unsupported") - - -def _raise_fsync_io_error(descriptor): - raise OSError(errno.EIO, "I/O failure") - - -def test_live_writer_lock_is_never_stale(tmp_path): - """Test that age alone cannot evict a positively live publisher.""" - lock = tmp_path / "entry.lock" - lock.write_text(f"{os.getpid()} token", encoding="ascii") - old = time.time() - 7200.0 - os.utime(lock, (old, old)) - assert not _preload_cache._raw_preload_remove_stale_lock(lock) - assert lock.is_file() - - -@pytest.mark.parametrize( - ("replacement", "error"), - ( - (_raise_unsupported_fsync, None), - (_raise_fsync_io_error, "I/O failure"), - ), -) -@pytest.mark.skipif(os.name == "nt", reason="directory fsync is POSIX-only") -def test_directory_fsync_unsupported_only(replacement, error, tmp_path, monkeypatch): - """Test narrow handling of filesystems without directory fsync.""" - monkeypatch.setattr(_preload_cache.os, "fsync", replacement) - context = nullcontext() if error is None else pytest.raises(OSError, match=error) - with context: - _preload_cache._raw_preload_fsync_directory(tmp_path) - - -def test_windows_identity_uses_change_time_and_file_id(tmp_path, monkeypatch): - """Test that Windows identity comes from the validated file handle.""" - source = tmp_path / "source.data" - source.write_bytes(b"source") - expected = dict( - size=len(b"source"), - mtime_ns=10, - change_ns=20, - device=30, - inode=40, - ) - descriptor = os.open(source, os.O_RDONLY) - try: - monkeypatch.setattr(_preload_cache, "_IS_WINDOWS", True) - monkeypatch.setattr( - _preload_cache, - "_raw_preload_windows_fstat", - lambda descriptor, result: expected, - raising=False, - ) - identity = _preload_cache._raw_preload_fstat(descriptor) - finally: - os.close(descriptor) - assert identity == expected - - -def test_windows_open_uses_reparse_safe_handle(tmp_path, monkeypatch): - """Test that Windows cache opens use the no-reparse helper.""" - source = tmp_path / "source.data" - source.write_bytes(b"source") - opened = [] - - def _open_windows(path): - opened.append(path) - return path.open("rb") - - monkeypatch.setattr(_preload_cache, "_IS_WINDOWS", True) - monkeypatch.setattr( - _preload_cache, - "_raw_preload_open_windows", - _open_windows, - raising=False, - ) - with _preload_cache._raw_preload_open_regular(source) as file: - assert file.read() == b"source" - assert opened == [source] - - -def test_windows_uses_configured_cache_directory(tmp_path, monkeypatch): - """Test that Windows can create the managed cache directory.""" - cache_root = tmp_path / "cache" - cache_root.mkdir() - path_class = _preload_cache.Path - monkeypatch.setattr(_preload_cache, "_IS_WINDOWS", True) - monkeypatch.setattr( - _preload_cache, - "Path", - lambda value: value if isinstance(value, path_class) else path_class(value), - ) - managed = _preload_cache._raw_preload_cache_dir(cache_root) - assert managed == cache_root / "raw-preload-v1" - assert managed.is_dir() +def test_unlocked_cache_lock_never_blocks(auto_cache, monkeypatch): + """Test that an abandoned unlocked file cannot block cache creation.""" + source, _ = auto_cache + raw = _RawArange(preload=False, filename=source) + cache_dir, key, _, _, _ = _preload_cache._raw_preload_cache_info(raw) + (cache_dir / f"{key}.lock").write_text("abandoned", encoding="ascii") + monkeypatch.setattr(_preload_cache, "_RAW_PRELOAD_LOCK_TIMEOUT", 0.1) + raw.load_data(memmap="auto") -def test_windows_does_not_use_posix_fchmod(monkeypatch): - """Test that Windows publication avoids unavailable POSIX permissions.""" - monkeypatch.setattr(_preload_cache, "_IS_WINDOWS", True) - monkeypatch.setattr( - _preload_cache.os, - "fchmod", - lambda *args: pytest.fail("os.fchmod was called"), - ) - _preload_cache._raw_preload_protect_generation(1) + assert_array_equal(raw.get_data()[:, 0], np.arange(1, 9)) @pytest.mark.skipif(os.name == "nt", reason="POSIX directory permissions") -def test_cache_rejects_untrusted_ancestor(tmp_path): - """Test that another local user cannot replace the managed directory.""" +def test_cache_accepts_configured_shared_ancestor(tmp_path): + """Test that the explicitly configured cache location is trusted.""" shared = tmp_path / "shared" shared.mkdir(mode=0o777) shared.chmod(0o777) cache_root = shared / "cache" cache_root.mkdir(mode=0o700) - with pytest.raises(PermissionError, match="untrusted writable ancestor"): - _preload_cache._raw_preload_cache_dir(cache_root) + assert _preload_cache._raw_preload_cache_dir(cache_root).is_dir() @pytest.mark.skipif(os.name == "nt", reason="POSIX directory permissions") -def test_cache_rejects_existing_public_managed_directory(tmp_path): - """Test that making an already exposed cache private is insufficient.""" +def test_cache_accepts_configured_public_managed_directory(tmp_path): + """Test that an explicitly configured existing cache is trusted.""" managed = tmp_path / "raw-preload-v1" managed.mkdir(mode=0o777) managed.chmod(0o777) - with pytest.raises(PermissionError, match="already be private"): - _preload_cache._raw_preload_cache_dir(tmp_path) + assert _preload_cache._raw_preload_cache_dir(tmp_path) == managed assert managed.stat().st_mode & 0o777 == 0o777 @@ -294,6 +194,24 @@ def test_auto_preload_first_miss_is_copy_on_write(auto_cache, tmp_path): assert not (tmp_path / "auto").exists() +def test_auto_preload_ignores_generation_metadata(auto_cache): + """Test that generation metadata does not cause an expensive re-decode.""" + source, _ = auto_cache + raw = _RawArange(preload="auto", filename=source) + generation = Path(raw._data.filename) + del raw + gc.collect() + result = generation.stat() + os.utime( + generation, + ns=(result.st_atime_ns, result.st_mtime_ns + 1_000_000), + ) + + other = _RawArange(preload="auto", filename=source) + + assert Path(other._data.filename) == generation + + def test_auto_preload_scavenges_same_key(auto_cache): """Test that a retry removes abandoned files for its cache key.""" source, _ = auto_cache @@ -318,14 +236,14 @@ def test_auto_preload_cleans_failed_publication(auto_cache, monkeypatch): source, cache_root = auto_cache with monkeypatch.context() as context: - context.setattr( - _preload_cache, "_raw_preload_replace_manifest", _fail_manifest_replace - ) + context.setattr(_preload_cache.json, "dump", _fail_manifest_dump) with pytest.raises(OSError, match="injected manifest failure"): _RawArange(preload="auto", filename=source) cache_dir = cache_root / "raw-preload-v1" - assert list(cache_dir.iterdir()) == [] + assert not list(cache_dir.glob("*.data")) + assert not list(cache_dir.glob("*.json")) + assert not list(cache_dir.glob("*.tmp")) raw = _RawArange(preload="auto", filename=source) assert_array_equal(raw.get_data()[:, 0], np.arange(1, 9)) @@ -340,7 +258,10 @@ def test_auto_preload_closes_failed_flush(auto_cache, monkeypatch): _RawArangeRecording(preload="auto", filename=source) assert mappings[0]._mmap.closed - assert list((cache_root / "raw-preload-v1").iterdir()) == [] + cache_dir = cache_root / "raw-preload-v1" + assert not list(cache_dir.glob("*.data")) + assert not list(cache_dir.glob("*.json")) + assert not list(cache_dir.glob("*.tmp")) def test_auto_preload_invalidates_mne_version(auto_cache, monkeypatch): @@ -385,8 +306,8 @@ def test_auto_preload_api_contract(tmp_path, monkeypatch): @pytest.mark.skipif(os.name == "nt", reason="POSIX file permissions") -def test_auto_preload_private_storage(auto_cache): - """Test private permissions for managed decoded data.""" +def test_auto_preload_storage_permissions(auto_cache): + """Test that newly created cache files are not shared by default.""" source, cache_root = auto_cache raw = _RawArange(preload="auto", filename=source) cache_dir = next(cache_root.iterdir()) @@ -395,7 +316,6 @@ def test_auto_preload_private_storage(auto_cache): assert cache_dir.stat().st_mode & 0o077 == 0 assert manifest.stat().st_mode & 0o077 == 0 assert generation.stat().st_mode & 0o077 == 0 - assert generation.stat().st_mode & 0o222 == 0 def test_auto_preload_rejects_cache_symlink(tmp_path, cache_root): @@ -450,11 +370,9 @@ def test_auto_preload_cache_formats(reader_name, relative_path, cache_root): "symlink_manifest", "missing_field", "unknown_field", - "version", "traversal_generation", "missing_generation", "wrong_size_generation", - "same_size_generation", "symlink_generation", ), ) @@ -482,14 +400,11 @@ def test_auto_preload_cache_corruption(corruption, auto_cache, tmp_path): except OSError: pytest.skip("symlink creation is unavailable") elif corruption == "missing_field": - manifest.pop("version") + manifest.pop("generation") manifest_path.write_text(json.dumps(manifest), encoding="utf-8") elif corruption == "unknown_field": manifest["unknown"] = True manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - elif corruption == "version": - manifest[corruption] = -1 - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") elif corruption == "traversal_generation": manifest["generation"] = f"../{manifest['generation']}" manifest_path.write_text(json.dumps(manifest), encoding="utf-8") @@ -498,12 +413,6 @@ def test_auto_preload_cache_corruption(corruption, auto_cache, tmp_path): elif corruption == "wrong_size_generation": generation.chmod(0o600) generation.write_bytes(b"short") - elif corruption == "same_size_generation": - generation.chmod(0o600) - stat = generation.stat() - with generation.open("r+b") as file: - file.write(b"\x00" * 8) - os.utime(generation, ns=(stat.st_atime_ns, stat.st_mtime_ns)) else: outside = tmp_path / "outside.dat" outside.write_bytes(b"outside") @@ -549,14 +458,12 @@ def test_auto_preload_recovers_crashed_publisher(cache_root): assert process.exitcode == 91 cache_dir = next(cache_root.iterdir()) assert len(list(cache_dir.glob("*.data"))) == 1 - assert len(list(cache_dir.glob("*.lock"))) == 1 assert not list(cache_dir.glob("*.json")) raw = mne.io.read_raw_edf(source, preload="auto", verbose="error") reference = mne.io.read_raw_edf(source, preload=True, verbose="error") assert raw._data.mode == "c" assert_array_equal(raw.get_data(), reference.get_data()) - assert not list(cache_dir.glob("*.lock")) assert not list(cache_dir.glob("*.tmp")) assert len(list(cache_dir.glob("*.data"))) == 1 @@ -619,13 +526,16 @@ def test_auto_preload_numeric_invalidation(tmp_path, cache_root): ) assert Path(excluded._data.filename) != generation assert excluded._data.shape[0] == original._data.shape[0] - 1 - stat = edf_source.stat() + result = edf_source.stat() with edf_source.open("r+b") as file: file.seek(-1, os.SEEK_END) byte = file.read(1) file.seek(-1, os.SEEK_END) file.write(bytes([byte[0] ^ 1])) - os.utime(edf_source, ns=(stat.st_atime_ns, stat.st_mtime_ns)) + os.utime( + edf_source, + ns=(result.st_atime_ns, result.st_mtime_ns + 10_000_000_000), + ) changed = mne.io.read_raw_edf(edf_source, preload="auto", verbose="error") assert Path(changed._data.filename) != generation diff --git a/mne/utils/config.py b/mne/utils/config.py index 92597d84344..ced003f5d44 100644 --- a/mne/utils/config.py +++ b/mne/utils/config.py @@ -54,10 +54,8 @@ def set_cache_dir(cache_dir): ----- Persistent decoded Raw entries are not automatically size-limited. They are stored below ``cache_dir`` in a versioned ``raw-preload`` directory. - On POSIX, physical ancestors must be owned by the current user or root, and - writable shared parents require sticky-directory semantics. On Windows, use - a private local cache directory controlled by the current account; MNE does - not verify its ACL. + The configured location is trusted and should only be writable by users who + are allowed to access its cached data. """ if cache_dir is not None and not op.exists(cache_dir): raise OSError(f"Directory {cache_dir} does not exist") diff --git a/pyproject.toml b/pyproject.toml index 2ea1ff516dc..e1ac39f69d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,7 @@ classifiers = [ ] dependencies = [ "decorator >= 5.1", + "filelock >= 3.18.0", "jinja2 >= 3.1", "lazy_loader >= 0.3", "matplotlib >= 3.9", # released 2024-05-15, will become 3.10 on 2026-12-14 @@ -170,7 +171,6 @@ full-no-qt = [ "dipy >= 1.9", # released 2024-03-08, will become 1.10 on 2026-12-12 "edfio >= 0.4.10", "eeglabio", - "filelock >= 3.18.0", "h5py >= 2.4", "imageio >= 2.6.1", "imageio-ffmpeg >= 0.4.1", From ade3eb3d660cb66e222dc2dcb88084a4d3ba38f7 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 08:23:16 +0000 Subject: [PATCH 3/9] [autofix.ci] apply automated fixes --- README.rst | 1 + tools/pylock.ci-old.toml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 3508426cba8..2068e97a2b4 100644 --- a/README.rst +++ b/README.rst @@ -74,6 +74,7 @@ The minimum required dependencies to run MNE-Python are: - `Python `__ ≥ 3.11 - `decorator `__ ≥ 5.1 +- `filelock `__ ≥ 3.18.0 - `Jinja2 `__ ≥ 3.1 - `lazy-loader `__ ≥ 0.3 - `Matplotlib `__ ≥ 3.9 diff --git a/tools/pylock.ci-old.toml b/tools/pylock.ci-old.toml index 340ee161f77..ae68bfc727e 100644 --- a/tools/pylock.ci-old.toml +++ b/tools/pylock.ci-old.toml @@ -93,6 +93,12 @@ version = "2.2.1" sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", upload-time = 2025-09-01T09:48:10Z, size = 1129488, hashes = { sha256 = "3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4" } } wheels = [{ url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", upload-time = 2025-09-01T09:48:08Z, size = 28317, hashes = { sha256 = "760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017" } }] +[[packages]] +name = "filelock" +version = "3.18.0" +sdist = { url = "https://files.pythonhosted.org/packages/0a/10/c23352565a6544bdc5353e0b15fc1c563352101f30e24bf500207a54df9a/filelock-3.18.0.tar.gz", upload-time = 2025-03-14T07:11:40Z, size = 18075, hashes = { sha256 = "adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2" } } +wheels = [{ url = "https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl", upload-time = 2025-03-14T07:11:39Z, size = 16215, hashes = { sha256 = "c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de" } }] + [[packages]] name = "fonttools" version = "4.61.1" From b35640a26bdfe9d608d27b41a1dfebe1f992bffd Mon Sep 17 00:00:00 2001 From: Bru Date: Thu, 27 Aug 2026 11:38:02 +0200 Subject: [PATCH 4/9] Simplify Raw preload cache identity and tests --- mne/io/_preload_cache.py | 26 +- mne/io/base.py | 6 +- mne/io/brainvision/brainvision.py | 6 - mne/io/edf/edf.py | 34 -- mne/io/fiff/raw.py | 14 - mne/io/fiff/tests/test_raw_fiff.py | 10 +- mne/io/tests/test_preload_cache.py | 496 +++++------------------------ mne/io/tests/test_raw.py | 28 -- mne/utils/docs.py | 4 +- 9 files changed, 88 insertions(+), 536 deletions(-) diff --git a/mne/io/_preload_cache.py b/mne/io/_preload_cache.py index 88993f7641d..f82aece4ebc 100644 --- a/mne/io/_preload_cache.py +++ b/mne/io/_preload_cache.py @@ -42,18 +42,16 @@ def _raw_preload_source_signature(raw): "or an explicit memory-map path" ) path = Path(filename).resolve(strict=True) + if path.suffix == ".gz": + raise ValueError( + 'preload="auto" supports only uncompressed source files; use ' + "preload=True for compressed files" + ) result = path.stat() if not stat.S_ISREG(result.st_mode): raise OSError("Raw source data must be regular files") - sources.append( - dict( - path=str(path), - size=int(result.st_size), - mtime_ns=int(result.st_mtime_ns), - device=int(result.st_dev), - inode=int(result.st_ino), - ) - ) + # ponytail: hash contents only if path, size, and mtime prove insufficient. + sources.append((str(path), int(result.st_size), int(result.st_mtime_ns))) return sources @@ -76,13 +74,6 @@ def _raw_preload_cache_dir(cache_root=None): def _raw_preload_cache_info(raw): """Return the managed cache location and expected array description.""" - cache_identity = raw._decoded_cache_identity() - if cache_identity is None: - raise ValueError( - f'preload="auto" is not supported for {type(raw).__name__}; use ' - "preload=True or an explicit memory-map path" - ) - decoder_abi, decoder_state = cache_identity cache_dir = _raw_preload_cache_dir() sources = _raw_preload_source_signature(raw) dtype = np.dtype(raw._dtype) @@ -91,9 +82,8 @@ def _raw_preload_cache_info(raw): version=_RAW_PRELOAD_CACHE_VERSION, mne_version=MNE_VERSION, reader=(type(raw).__module__, type(raw).__qualname__), - decoder_abi=decoder_abi, sources=sources, - decoder_state=decoder_state, + raw_extras=raw._raw_extras, read_picks=raw._read_picks, cals=raw._cals, projector=raw._projector, diff --git a/mne/io/base.py b/mne/io/base.py index f6a4bdad993..09c894b286c 100644 --- a/mne/io/base.py +++ b/mne/io/base.py @@ -207,10 +207,6 @@ class BaseRaw( _filenames: list[Path | None] _data: np.ndarray | None - def _decoded_cache_identity(self): - """Return ``(ABI, state)`` for numeric decoding, or ``None``.""" - return None - @verbose def __init__( self, @@ -617,7 +613,7 @@ def load_data( and is responsible for removing it after the Raw object is no longer in use. For supported file readers, ``"auto"`` instead reuses the persistent decoded-data cache configured by :func:`mne.set_cache_dir`. - Cache entries for superseded source or decoder identities remain in + Cache entries for superseded source identities remain in a versioned ``raw-preload`` directory below the configured path. If ``None`` (default), preload data into RAM. diff --git a/mne/io/brainvision/brainvision.py b/mne/io/brainvision/brainvision.py index bca37afa9cc..9c9d978a861 100644 --- a/mne/io/brainvision/brainvision.py +++ b/mne/io/brainvision/brainvision.py @@ -95,12 +95,6 @@ class RawBrainVision(BaseRaw): _extra_attributes = ("impedances",) - def _decoded_cache_identity(self): - """Return identity that determines numeric BrainVision decoding.""" - keys = ("offsets", "fmt", "order", "n_samples", "orig_nchan") - state = [{key: extra[key] for key in keys} for extra in self._raw_extras] - return (1, state) - @verbose def __init__( self, diff --git a/mne/io/edf/edf.py b/mne/io/edf/edf.py index e26774370d1..390289ae6bb 100644 --- a/mne/io/edf/edf.py +++ b/mne/io/edf/edf.py @@ -59,34 +59,6 @@ class FileType(Enum): } -def _edf_decoded_cache_identity(raw): - """Return identity that determines numeric EDF/BDF decoding.""" - if any(extra["blob"] is not None for extra in raw._raw_extras): - raise ValueError('preload="auto" does not support file-like EDF/BDF inputs') - # Keep this in the same order as the values consumed by - # ``_read_segment_file`` below. Header metadata that cannot affect decoded - # samples (for example channel types and filter descriptions) stays live. - state = [ - ( - extra["n_samps"], - extra["max_samp"], - extra["dtype_np"], - extra["dtype_byte"], - extra["data_offset"], - extra["stim_channel_idxs"], - extra["sel"], - extra["tal_idx"], - extra["subtype"], - extra["cal"], - extra["offsets"], - extra["units"], - extra["nsamples"], - ) - for extra in raw._raw_extras - ] - return (1, state) - - @fill_doc class RawEDF(BaseRaw): """Raw object from EDF, EDF+ file. @@ -182,9 +154,6 @@ class RawEDF(BaseRaw): encoded in such analog stim channels. """ - def _decoded_cache_identity(self): - return _edf_decoded_cache_identity(self) - @verbose def __init__( self, @@ -397,9 +366,6 @@ class RawBDF(BaseRaw): encoded in such analog stim channels. """ - def _decoded_cache_identity(self): - return _edf_decoded_cache_identity(self) - @verbose def __init__( self, diff --git a/mne/io/fiff/raw.py b/mne/io/fiff/raw.py index 779005dda72..094de8ed049 100644 --- a/mne/io/fiff/raw.py +++ b/mne/io/fiff/raw.py @@ -83,20 +83,6 @@ class Raw(BaseRaw): Indicates whether raw data are in memory. """ - def _decoded_cache_identity(self): - """Return identity that determines numeric FIF decoding.""" - if any( - filename is not None and filename.suffix == ".gz" - for filename in self.filenames - ): - raise ValueError( - 'preload="auto" supports only uncompressed FIF files; use ' - "preload=True for gzip-compressed FIF" - ) - keys = ("ent", "bounds", "orig_nchan") - state = [{key: extra[key] for key in keys} for extra in self._raw_extras] - return (1, state) - _extra_attributes = ( "fix_mag_coil_types", "acqparser", diff --git a/mne/io/fiff/tests/test_raw_fiff.py b/mne/io/fiff/tests/test_raw_fiff.py index 93298307978..d6ead398654 100644 --- a/mne/io/fiff/tests/test_raw_fiff.py +++ b/mne/io/fiff/tests/test_raw_fiff.py @@ -2122,9 +2122,7 @@ def test_file_like(kind, preload, split, tmp_path): def test_file_like_auto_preload_rejected(tmp_path, monkeypatch): """Test that automatic caching cannot misidentify a named stream.""" - cache_dir = tmp_path / "cache" - cache_dir.mkdir() - monkeypatch.setenv("MNE_CACHE_DIR", str(cache_dir)) + monkeypatch.setenv("MNE_CACHE_DIR", str(tmp_path)) stream = BytesIO(test_fif_fname.read_bytes()) stream.name = str(test_fif_fname) with pytest.raises(ValueError, match="stable source files"): @@ -2133,13 +2131,11 @@ def test_file_like_auto_preload_rejected(tmp_path, monkeypatch): def test_compressed_auto_preload_rejected(tmp_path, monkeypatch): """Test that gzip FIF does not advertise ineffective decoded caching.""" - cache_dir = tmp_path / "cache" - cache_dir.mkdir() - monkeypatch.setenv("MNE_CACHE_DIR", str(cache_dir)) + monkeypatch.setenv("MNE_CACHE_DIR", str(tmp_path)) with pytest.raises(ValueError, match="uncompressed FIF"): read_raw_fif(test_fif_gz_fname, preload="auto") raw = read_raw_fif(test_fif_gz_fname, preload=False) - with pytest.raises(ValueError, match="uncompressed FIF"): + with pytest.raises(ValueError, match="uncompressed"): raw.load_data(memmap="auto") diff --git a/mne/io/tests/test_preload_cache.py b/mne/io/tests/test_preload_cache.py index 972d54973e0..9b7fae5225e 100644 --- a/mne/io/tests/test_preload_cache.py +++ b/mne/io/tests/test_preload_cache.py @@ -1,4 +1,4 @@ -"""Tests for persistent Raw preload-cache infrastructure.""" +"""Tests for persistent Raw preload caching.""" # Authors: The MNE-Python contributors. # License: BSD-3-Clause @@ -6,12 +6,10 @@ import gc import hashlib -import json import multiprocessing import os import shutil -import threading -from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor +from concurrent.futures import ProcessPoolExecutor from contextlib import chdir from pathlib import Path @@ -22,17 +20,14 @@ import mne from mne._fiff.pick import pick_info from mne.io import RawArray, _preload_cache -from mne.io.tests.test_raw import ( - _RawArange, - _read_raw_arange, -) +from mne.io.tests.test_raw import _RawArange, _read_raw_arange _ORIGINAL_CACHE_REPLACE = None _IO_DATA_DIR = Path(mne.io.__file__).parent def _auto_preload_process(reader_name, source, cache_dir): - """Read one automatic cache entry in an isolated process.""" + """Read one cache entry in an isolated process.""" os.environ["MNE_CACHE_DIR"] = cache_dir raw = getattr(mne.io, reader_name)(source, preload="auto", verbose="error") digest = hashlib.sha256(raw.get_data().tobytes()).hexdigest() @@ -40,14 +35,14 @@ def _auto_preload_process(reader_name, source, cache_dir): def _replace_cache_generation_then_exit(source, destination): - """Publish one generation and simulate an immediate process crash.""" + """Crash after publishing data but before publishing its manifest.""" _ORIGINAL_CACHE_REPLACE(source, destination) if str(destination).endswith(".data"): os._exit(91) def _auto_preload_crash_process(source, cache_dir): - """Crash a cache writer after its generation becomes durable.""" + """Run the simulated crash in an isolated process.""" global _ORIGINAL_CACHE_REPLACE os.environ["MNE_CACHE_DIR"] = cache_dir @@ -56,26 +51,9 @@ def _auto_preload_crash_process(source, cache_dir): mne.io.read_raw_edf(source, preload="auto", verbose="error") -class _RawArangeBarrier(_RawArange): - _barrier = None - - def _read_segment(self, *args, **kwargs): - self._barrier.wait(timeout=2.0) - return super()._read_segment(*args, **kwargs) - - -class _RawArangeRecording(_RawArange): - _mappings = None - - def _read_segment(self, *args, **kwargs): - data = super()._read_segment(*args, **kwargs) - self._mappings.append(data) - return data - - @pytest.fixture def cache_root(tmp_path, monkeypatch): - """Configure and return an isolated preload cache.""" + """Configure an isolated cache directory.""" cache_root = tmp_path / "cache" cache_root.mkdir() monkeypatch.setenv("MNE_CACHE_DIR", str(cache_root)) @@ -84,255 +62,33 @@ def cache_root(tmp_path, monkeypatch): @pytest.fixture def auto_cache(tmp_path, cache_root): - """Create one stable source for the isolated preload cache.""" + """Create one stable source file.""" source = tmp_path / "source.bin" source.write_bytes(b"source identity") return source, cache_root -def _fail_manifest_dump(*args, **kwargs): - raise OSError("injected manifest failure") - - -def _fail_memmap_flush(self): - raise OSError("injected flush failure") - - -def test_unlocked_cache_lock_never_blocks(auto_cache, monkeypatch): - """Test that an abandoned unlocked file cannot block cache creation.""" - source, _ = auto_cache - raw = _RawArange(preload=False, filename=source) - cache_dir, key, _, _, _ = _preload_cache._raw_preload_cache_info(raw) - (cache_dir / f"{key}.lock").write_text("abandoned", encoding="ascii") - monkeypatch.setattr(_preload_cache, "_RAW_PRELOAD_LOCK_TIMEOUT", 0.1) - - raw.load_data(memmap="auto") - - assert_array_equal(raw.get_data()[:, 0], np.arange(1, 9)) - - -@pytest.mark.skipif(os.name == "nt", reason="POSIX directory permissions") -def test_cache_accepts_configured_shared_ancestor(tmp_path): - """Test that the explicitly configured cache location is trusted.""" - shared = tmp_path / "shared" - shared.mkdir(mode=0o777) - shared.chmod(0o777) - cache_root = shared / "cache" - cache_root.mkdir(mode=0o700) - assert _preload_cache._raw_preload_cache_dir(cache_root).is_dir() - - -@pytest.mark.skipif(os.name == "nt", reason="POSIX directory permissions") -def test_cache_accepts_configured_public_managed_directory(tmp_path): - """Test that an explicitly configured existing cache is trusted.""" - managed = tmp_path / "raw-preload-v1" - managed.mkdir(mode=0o777) - managed.chmod(0o777) - assert _preload_cache._raw_preload_cache_dir(tmp_path) == managed - assert managed.stat().st_mode & 0o777 == 0o777 - - -@pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics") -def test_cache_canonicalizes_symlink_ancestor(tmp_path): - """Test that a configured symlink is fixed to one physical location.""" - real = tmp_path / "real" - real.mkdir() - link = tmp_path / "link" - try: - link.symlink_to(real, target_is_directory=True) - except OSError: - pytest.skip("symlink creation is unavailable") - managed = _preload_cache._raw_preload_cache_dir(link) - assert managed == real / "raw-preload-v1" - - -def test_distinct_keys_publish_concurrently(tmp_path, cache_root, monkeypatch): - """Test that unrelated first-time decodes do not share a long-held lock.""" - sources = [tmp_path / f"source-{index}.bin" for index in range(2)] - for index, source in enumerate(sources): - source.write_bytes(bytes([index])) - raws = [_RawArangeBarrier(preload=False, filename=source) for source in sources] - barrier = threading.Barrier(2) - monkeypatch.setattr(_RawArangeBarrier, "_barrier", barrier) - with ThreadPoolExecutor(max_workers=2) as pool: - list(pool.map(lambda raw: raw.load_data(memmap="auto"), raws)) - assert len({Path(raw._data.filename) for raw in raws}) == 2 - - -@pytest.mark.skipif(os.name == "nt", reason="POSIX replacement semantics") -def test_generation_mapping_uses_validated_handle(tmp_path): - """Test that pathname replacement cannot change the mapped generation.""" - generation = tmp_path / "generation.data" - replacement = tmp_path / "replacement.data" - np.arange(4.0).tofile(generation) - np.full(4, 99.0).tofile(replacement) - with _preload_cache._raw_preload_open_regular(generation) as file: - os.replace(replacement, generation) - data = np.memmap(file, mode="c", dtype=np.float64, shape=(4,)) - np.testing.assert_array_equal(data, np.arange(4.0)) - - -def test_auto_preload_first_miss_is_copy_on_write(auto_cache, tmp_path): - """Test that an automatic cache miss publishes immutable data.""" - source, _ = auto_cache - with chdir(tmp_path): - raw = _RawArange(preload="auto", filename=source) - - assert isinstance(raw._data, np.memmap) - assert raw._data.mode == "c" - generation = Path(raw._data.filename) - expected = raw.get_data() - raw._data[0, 0] = 99.0 - del raw - gc.collect() - - with chdir(tmp_path): - other = _RawArange(preload="auto", filename=source) - assert other._data.mode == "c" - assert Path(other._data.filename) == generation - assert_array_equal(other.get_data(), expected) - assert not (tmp_path / "auto").exists() - - -def test_auto_preload_ignores_generation_metadata(auto_cache): - """Test that generation metadata does not cause an expensive re-decode.""" - source, _ = auto_cache - raw = _RawArange(preload="auto", filename=source) - generation = Path(raw._data.filename) - del raw - gc.collect() - result = generation.stat() - os.utime( - generation, - ns=(result.st_atime_ns, result.st_mtime_ns + 1_000_000), - ) - - other = _RawArange(preload="auto", filename=source) - - assert Path(other._data.filename) == generation - - -def test_auto_preload_scavenges_same_key(auto_cache): - """Test that a retry removes abandoned files for its cache key.""" - source, _ = auto_cache - raw = _RawArange(preload="auto", filename=source) - cache_dir = Path(raw._data.filename).parent - manifest_path = next(cache_dir.glob("*.json")) - key = manifest_path.stem - orphan = cache_dir / f"{key}.{'0' * 32}.data" - temporary = cache_dir / f".{key}.abandoned.tmp" - orphan.write_bytes(b"orphan") - temporary.write_bytes(b"temporary") - manifest_path.unlink() - - other = _RawArange(preload="auto", filename=source) - assert_array_equal(other.get_data(), raw.get_data()) - assert len(list(cache_dir.glob(f"{key}.*.data"))) == 1 - assert not temporary.exists() - - -def test_auto_preload_cleans_failed_publication(auto_cache, monkeypatch): - """Test cleanup and retry after manifest publication fails.""" - source, cache_root = auto_cache - - with monkeypatch.context() as context: - context.setattr(_preload_cache.json, "dump", _fail_manifest_dump) - with pytest.raises(OSError, match="injected manifest failure"): - _RawArange(preload="auto", filename=source) - - cache_dir = cache_root / "raw-preload-v1" - assert not list(cache_dir.glob("*.data")) - assert not list(cache_dir.glob("*.json")) - assert not list(cache_dir.glob("*.tmp")) - raw = _RawArange(preload="auto", filename=source) - assert_array_equal(raw.get_data()[:, 0], np.arange(1, 9)) - - -def test_auto_preload_closes_failed_flush(auto_cache, monkeypatch): - """Test that a flush failure closes its temporary mapping.""" - source, cache_root = auto_cache - mappings = [] - monkeypatch.setattr(_RawArangeRecording, "_mappings", mappings) - monkeypatch.setattr(np.memmap, "flush", _fail_memmap_flush) - with pytest.raises(OSError, match="injected flush failure"): - _RawArangeRecording(preload="auto", filename=source) - - assert mappings[0]._mmap.closed - cache_dir = cache_root / "raw-preload-v1" - assert not list(cache_dir.glob("*.data")) - assert not list(cache_dir.glob("*.json")) - assert not list(cache_dir.glob("*.tmp")) - - -def test_auto_preload_invalidates_mne_version(auto_cache, monkeypatch): - """Test that decoded cache data do not cross MNE version boundaries.""" - source, _ = auto_cache - raw = _RawArange(preload="auto", filename=source) - first = Path(raw._data.filename) - - monkeypatch.setattr(_preload_cache, "MNE_VERSION", "next-version") - other = _RawArange(preload="auto", filename=source) - assert Path(other._data.filename) != first - assert_array_equal(other.get_data(), raw.get_data()) - - -def test_auto_preload_api_contract(tmp_path, monkeypatch): - """Test automatic preload errors and the literal-path escape.""" +def test_auto_preload_api(tmp_path, monkeypatch): + """Test cache configuration and the literal-path escape.""" source = tmp_path / "source.bin" source.write_bytes(b"source identity") monkeypatch.setattr(_preload_cache, "get_config", lambda *args, **kwargs: None) with pytest.raises(ValueError, match="set_cache_dir"): _RawArange(preload="auto", filename=source) - raw = _RawArange(preload=False, filename=source) - with pytest.raises(ValueError, match="set_cache_dir"): - raw.load_data(memmap="auto") - identity_method = _RawArange._decoded_cache_identity - monkeypatch.setattr(_RawArange, "_decoded_cache_identity", lambda self: None) + monkeypatch.setattr( _preload_cache, "get_config", lambda *args, **kwargs: str(tmp_path) ) - with pytest.raises(ValueError, match="is not supported"): - _RawArange(preload="auto", filename=source) - with chdir(tmp_path): literal = _RawArange(preload=Path("auto"), filename=source) assert literal._data.mode == "w+" assert (tmp_path / "auto").is_file() - monkeypatch.setattr(_RawArange, "_decoded_cache_identity", identity_method) lazy = _RawArange(preload=False, filename=source) lazy.load_data(memmap="auto") assert lazy._data.mode == "c" -@pytest.mark.skipif(os.name == "nt", reason="POSIX file permissions") -def test_auto_preload_storage_permissions(auto_cache): - """Test that newly created cache files are not shared by default.""" - source, cache_root = auto_cache - raw = _RawArange(preload="auto", filename=source) - cache_dir = next(cache_root.iterdir()) - manifest = next(cache_dir.glob("*.json")) - generation = Path(raw._data.filename) - assert cache_dir.stat().st_mode & 0o077 == 0 - assert manifest.stat().st_mode & 0o077 == 0 - assert generation.stat().st_mode & 0o077 == 0 - - -def test_auto_preload_rejects_cache_symlink(tmp_path, cache_root): - """Test that the private managed directory cannot be redirected.""" - source = tmp_path / "source.bin" - source.write_bytes(b"source identity") - outside = tmp_path / "outside" - outside.mkdir() - try: - os.symlink(outside, cache_root / "raw-preload-v1") - except OSError: - pytest.skip("symlink creation is unavailable") - with pytest.raises(OSError, match="regular directory"): - _RawArange(preload="auto", filename=source) - assert not list(outside.iterdir()) - - @pytest.mark.parametrize( ("reader_name", "relative_path"), ( @@ -342,93 +98,75 @@ def test_auto_preload_rejects_cache_symlink(tmp_path, cache_root): ("read_raw_brainvision", "brainvision/tests/data/test.vhdr"), ), ) -def test_auto_preload_cache_formats(reader_name, relative_path, cache_root): - """Test exact automatic preload reuse across supported formats.""" +def test_auto_preload_formats(reader_name, relative_path, cache_root): + """Test exact copy-on-write cache reuse across file formats.""" source = _IO_DATA_DIR / relative_path reader = getattr(mne.io, reader_name) - reference = reader(source, preload=True, verbose="error").get_data() - + expected = reader(source, preload=True, verbose="error").get_data() raw = reader(source, preload="auto", verbose="error") - assert raw._data.mode == "c" generation = Path(raw._data.filename) - assert_array_equal(raw.get_data(), reference) + assert raw._data.mode == "c" + assert_array_equal(raw.get_data(), expected) raw._data[0, 0] += 1.0 del raw gc.collect() other = reader(source, preload="auto", verbose="error") - assert other._data.mode == "c" assert Path(other._data.filename) == generation - assert_array_equal(other.get_data(), reference) + assert_array_equal(other.get_data(), expected) -@pytest.mark.parametrize( - "corruption", - ( - "truncated_json", - "oversized_manifest", - "symlink_manifest", - "missing_field", - "unknown_field", - "traversal_generation", - "missing_generation", - "wrong_size_generation", - "symlink_generation", - ), -) -def test_auto_preload_cache_corruption(corruption, auto_cache, tmp_path): - """Test that corrupt cache entries always become safe misses.""" +def test_auto_preload_key(tmp_path, cache_root): + """Test numeric options and source modification invalidate the cache.""" + data_dir = _IO_DATA_DIR / "brainvision/tests/data" + for name in ("test.vhdr", "test.vmrk", "test.eeg"): + shutil.copy(data_dir / name, tmp_path / name) + source = tmp_path / "test.vhdr" + raw = mne.io.read_raw_brainvision(source, preload="auto", verbose="error") + scaled = mne.io.read_raw_brainvision( + source, scale=2.0, preload="auto", verbose="error" + ) + assert Path(scaled._data.filename) != Path(raw._data.filename) + assert_array_equal(scaled.get_data(), 2.0 * raw.get_data()) + + text = source.read_text(encoding="utf-8") + source.write_text( + text.replace("DataOrientation=MULTIPLEXED", "DataOrientation=VECTORIZED"), + encoding="utf-8", + ) + expected = mne.io.read_raw_brainvision(source, preload=True, verbose="error") + changed = mne.io.read_raw_brainvision(source, preload="auto", verbose="error") + assert Path(changed._data.filename) != Path(raw._data.filename) + assert_array_equal(changed.get_data(), expected.get_data()) + + source = tmp_path / "copy.edf" + shutil.copy(_IO_DATA_DIR / "edf/tests/data/test.edf", source) + raw = mne.io.read_raw_edf(source, preload="auto", verbose="error") + generation = Path(raw._data.filename) + result = source.stat() + os.utime(source, ns=(result.st_atime_ns, result.st_mtime_ns + 1_000_000_000)) + other = mne.io.read_raw_edf(source, preload="auto", verbose="error") + assert Path(other._data.filename) != generation + + +@pytest.mark.parametrize("corruption", ("manifest", "data")) +def test_auto_preload_recovers_corruption(corruption, auto_cache): + """Test that malformed cache entries become misses.""" source, cache_root = auto_cache raw = _RawArange(preload="auto", filename=source) expected = raw.get_data() + generation = Path(raw._data.filename) del raw + gc.collect() cache_dir = next(cache_root.iterdir()) - manifest_path = next(cache_dir.glob("*.json")) - manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - generation = cache_dir / manifest["generation"] - - if corruption == "truncated_json": - manifest_path.write_text('{"version":', encoding="utf-8") - elif corruption == "oversized_manifest": - manifest_path.write_text(" " * 4097, encoding="utf-8") - elif corruption == "symlink_manifest": - outside = tmp_path / "outside.json" - outside.write_text(json.dumps(manifest), encoding="utf-8") - manifest_path.unlink() - try: - os.symlink(outside, manifest_path) - except OSError: - pytest.skip("symlink creation is unavailable") - elif corruption == "missing_field": - manifest.pop("generation") - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - elif corruption == "unknown_field": - manifest["unknown"] = True - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - elif corruption == "traversal_generation": - manifest["generation"] = f"../{manifest['generation']}" - manifest_path.write_text(json.dumps(manifest), encoding="utf-8") - elif corruption == "missing_generation": - generation.unlink() - elif corruption == "wrong_size_generation": - generation.chmod(0o600) - generation.write_bytes(b"short") + if corruption == "manifest": + next(cache_dir.glob("*.json")).write_text("{", encoding="utf-8") else: - outside = tmp_path / "outside.dat" - outside.write_bytes(b"outside") - generation.unlink() - try: - os.symlink(outside, generation) - except OSError: - pytest.skip("symlink creation is unavailable") + generation.write_bytes(b"short") other = _RawArange(preload="auto", filename=source) - assert other._data.mode == "c" + assert Path(other._data.filename) != generation assert_array_equal(other.get_data(), expected) - if corruption == "symlink_generation": - assert outside.read_bytes() == b"outside" - elif corruption == "symlink_manifest": - assert json.loads(outside.read_text(encoding="utf-8")) == manifest def test_auto_preload_concurrent_misses(cache_root): @@ -443,7 +181,6 @@ def test_auto_preload_concurrent_misses(cache_root): assert len({result[2] for result in results}) == 1 cache_dir = next(cache_root.iterdir()) assert len(list(cache_dir.glob("*.data"))) == 1 - assert not list(cache_dir.glob("*.tmp")) def test_auto_preload_recovers_crashed_publisher(cache_root): @@ -456,110 +193,28 @@ def test_auto_preload_recovers_crashed_publisher(cache_root): process.start() process.join(timeout=15) assert process.exitcode == 91 - cache_dir = next(cache_root.iterdir()) - assert len(list(cache_dir.glob("*.data"))) == 1 - assert not list(cache_dir.glob("*.json")) raw = mne.io.read_raw_edf(source, preload="auto", verbose="error") - reference = mne.io.read_raw_edf(source, preload=True, verbose="error") - assert raw._data.mode == "c" - assert_array_equal(raw.get_data(), reference.get_data()) - assert not list(cache_dir.glob("*.tmp")) + expected = mne.io.read_raw_edf(source, preload=True, verbose="error").get_data() + assert_array_equal(raw.get_data(), expected) + cache_dir = next(cache_root.iterdir()) assert len(list(cache_dir.glob("*.data"))) == 1 + assert not list(cache_dir.glob("*.tmp")) -def test_auto_preload_identity_ignores_edf_channel_type(cache_root): - """Test that live metadata does not invalidate decoded samples.""" - source = _IO_DATA_DIR / "edf/tests/data/test.edf" - raw = mne.io.read_raw_edf(source, preload="auto", verbose="error") - generation = Path(raw._data.filename) - - other = mne.io.read_raw_edf(source, eog=[0], preload="auto", verbose="error") - assert Path(other._data.filename) == generation - assert other.get_channel_types()[0] == "eog" - assert_array_equal(other.get_data(), raw.get_data()) - - -def test_auto_preload_brainvision_live_markers(tmp_path, cache_root): - """Test that markers remain live while decoded samples are reused.""" - data_dir = _IO_DATA_DIR / "brainvision/tests/data" - for name in ("test.vhdr", "test.vmrk", "test.eeg"): - shutil.copy(data_dir / name, tmp_path / name) - source = tmp_path / "test.vhdr" - raw = mne.io.read_raw_brainvision(source, preload="auto", verbose="error") - generation = Path(raw._data.filename) - annotation_count = len(raw.annotations) - with (tmp_path / "test.vmrk").open("a", encoding="utf-8") as file: - file.write("\nMk15=Stimulus,S 99,7800,1,0\n") - - other = mne.io.read_raw_brainvision(source, preload="auto", verbose="error") - assert Path(other._data.filename) == generation - assert len(other.annotations) == annotation_count + 1 - assert other.annotations.description[-1] == "Stimulus/S 99" - plain = mne.io.read_raw_brainvision( - source, ignore_marker_types=True, preload="auto", verbose="error" - ) - assert Path(plain._data.filename) == generation - assert plain.annotations.description[-1] == "S 99" - - -def test_auto_preload_numeric_invalidation(tmp_path, cache_root): - """Test numeric options and filesystem changes invalidate cached data.""" - data_dir = _IO_DATA_DIR / "brainvision/tests/data" - source = data_dir / "test.vhdr" - raw = mne.io.read_raw_brainvision(source, preload="auto", verbose="error") - scaled = mne.io.read_raw_brainvision( - source, scale=2.0, preload="auto", verbose="error" - ) - assert Path(scaled._data.filename) != Path(raw._data.filename) - assert_array_equal(scaled.get_data(), 2.0 * raw.get_data()) - - edf_source = tmp_path / "test.edf" - shutil.copy(_IO_DATA_DIR / "edf/tests/data/test.edf", edf_source) - original = mne.io.read_raw_edf(edf_source, preload="auto", verbose="error") - generation = Path(original._data.filename) - excluded = mne.io.read_raw_edf( - edf_source, - exclude=[original.ch_names[0]], - preload="auto", - verbose="error", - ) - assert Path(excluded._data.filename) != generation - assert excluded._data.shape[0] == original._data.shape[0] - 1 - result = edf_source.stat() - with edf_source.open("r+b") as file: - file.seek(-1, os.SEEK_END) - byte = file.read(1) - file.seek(-1, os.SEEK_END) - file.write(bytes([byte[0] ^ 1])) - os.utime( - edf_source, - ns=(result.st_atime_ns, result.st_mtime_ns + 10_000_000_000), - ) - changed = mne.io.read_raw_edf(edf_source, preload="auto", verbose="error") - assert Path(changed._data.filename) != generation - - alias = tmp_path / "alias.edf" +def test_auto_preload_rejects_cache_symlink(tmp_path, cache_root): + """Test that the managed cache directory cannot be redirected.""" + source = tmp_path / "source.bin" + source.write_bytes(b"source identity") + outside = tmp_path / "outside" + outside.mkdir() try: - os.symlink(edf_source, alias) + os.symlink(outside, cache_root / "raw-preload-v1") except OSError: pytest.skip("symlink creation is unavailable") - aliased = mne.io.read_raw_edf(alias, preload="auto", verbose="error") - assert Path(aliased._data.filename) == Path(changed._data.filename) - - -@pytest.mark.parametrize("attribute", ("_projector", "_comp")) -def test_auto_preload_transform_invalidation(attribute, auto_cache): - """Test delayed projection and compensation use distinct cache entries.""" - source, _ = auto_cache - transformed = _read_raw_arange(filename=source) - setattr(transformed, attribute, 2.0 * np.eye(len(transformed.ch_names))) - transformed.load_data(memmap="auto", verbose="error") - plain = _read_raw_arange(filename=source) - plain.load_data(memmap="auto", verbose="error") - - assert Path(transformed._data.filename) != Path(plain._data.filename) - assert_array_equal(transformed.get_data(), 2.0 * plain.get_data()) + with pytest.raises(OSError, match="regular directory"): + _RawArange(preload="auto", filename=source) + assert not list(outside.iterdir()) def test_add_channels_copy_on_write_memmap(tmp_path, monkeypatch): @@ -569,7 +224,6 @@ def test_add_channels_copy_on_write_memmap(tmp_path, monkeypatch): memmap_fname = tmp_path / "raw-copy-on-write-memmap.dat" raw = _read_raw_arange(preload=memmap_fname) shape = raw._data.shape - expected = raw._data.copy() raw._data._mmap.close() raw._data = np.memmap(memmap_fname, mode="c", dtype=np.float64, shape=shape) raw._data[0, 0] = 99.0 @@ -580,10 +234,8 @@ def test_add_channels_copy_on_write_memmap(tmp_path, monkeypatch): monkeypatch.setattr(channels_module.sys, "platform", "linux") raw.add_channels([extra]) - assert not isinstance(raw._data, np.memmap) assert raw._data.shape == (shape[0] + 1, shape[1]) - expected[0, 0] = 99.0 - assert_array_equal(raw._data[:-1], expected) + assert raw._data[0, 0] == 99.0 stored = np.memmap(memmap_fname, mode="r", dtype=np.float64, shape=shape) assert stored[0, 0] != 99.0 stored._mmap.close() diff --git a/mne/io/tests/test_raw.py b/mne/io/tests/test_raw.py index 53ec6444040..ceea153469f 100644 --- a/mne/io/tests/test_raw.py +++ b/mne/io/tests/test_raw.py @@ -52,16 +52,6 @@ def _fail_if_times_materialized(*args, **kwargs): pytest.fail("The full Raw.times vector was materialized") -class _CropRecorder: - def __init__(self): - self.args = None - self.kwargs = None - - def crop(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - - def assert_named_constants(info): """Assert that info['chs'] has named constants.""" # for now we just check one @@ -129,21 +119,6 @@ def test_set_annotations_does_not_materialize_times(monkeypatch): assert len(raw.annotations) == 1 -def test_set_annotations_preserves_endpoint_arithmetic(monkeypatch): - """Test annotation bounds preserve the prior floating-point operations.""" - raw = RawArray(np.zeros((1, 6)), create_info(1, 100.0), verbose="error") - annotations = Annotations([0.0], [0.0], ["test"]) - recorder = _CropRecorder() - monkeypatch.setattr(Annotations, "crop", recorder.crop) - - raw.set_annotations(annotations) - - endpoint = (raw.n_times - 1) / raw.info["sfreq"] + 1.0 / raw.info["sfreq"] - assert endpoint != raw.duration - assert recorder.args == (0, endpoint) - assert recorder.kwargs == {"emit_warning": True} - - def _test_raw_reader( reader, test_preloading=True, @@ -885,9 +860,6 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): one[idx] = np.arange(1, 9)[idx, np.newaxis] _mult_cal_one(data, one, idx, cals, mult) - def _decoded_cache_identity(self): - return (1, ()) - def _read_raw_arange(preload=False, filename=None, verbose=None): return _RawArange(preload, filename=filename, verbose=verbose) diff --git a/mne/utils/docs.py b/mne/utils/docs.py index f9416c9bd9f..445070f2d01 100644 --- a/mne/utils/docs.py +++ b/mne/utils/docs.py @@ -3727,8 +3727,8 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75): Cached data persist in a versioned ``raw-preload`` directory below the configured cache path and are mapped copy-on-write, so modifying the returned Raw does not modify later reads. A cache miss performs the normal full decode. - Valid entries for historical source or decoder identities are retained without - an automatic size limit. The configured cache path is fixed to its physical + Valid entries for historical source identities are retained without an + automatic size limit. The configured cache path is fixed to its physical location. Use ``Path("auto")``, ``"./auto"``, or an absolute path to create a file literally named ``auto``. From a138fb7a6aad021e8cc0eea9708ec4e0806e2f51 Mon Sep 17 00:00:00 2001 From: Bru Date: Thu, 27 Aug 2026 13:09:45 +0200 Subject: [PATCH 5/9] Simplify persistent Raw preload caching --- doc/changes/dev/14216.newfeature.rst | 2 +- mne/io/_preload_cache.py | 243 ++++++--------------------- mne/io/base.py | 34 ++-- mne/io/fiff/raw.py | 9 +- mne/io/fiff/tests/test_raw_fiff.py | 10 -- mne/io/tests/test_preload_cache.py | 120 +++---------- mne/io/tests/test_raw.py | 21 +-- mne/utils/config.py | 2 - mne/utils/docs.py | 14 +- 9 files changed, 97 insertions(+), 358 deletions(-) diff --git a/doc/changes/dev/14216.newfeature.rst b/doc/changes/dev/14216.newfeature.rst index ebfb87f9fdd..9ef32b674c2 100644 --- a/doc/changes/dev/14216.newfeature.rst +++ b/doc/changes/dev/14216.newfeature.rst @@ -1,2 +1,2 @@ -Speed up repeated preloading of uncompressed FIF, EDF/BDF, and BrainVision +Speed up repeated preloading of FIF, EDF/BDF, and BrainVision recordings with a persistent copy-on-write decoded-data cache, by `Bruno Aristimunha`_. diff --git a/mne/io/_preload_cache.py b/mne/io/_preload_cache.py index f82aece4ebc..53b4772cd4c 100644 --- a/mne/io/_preload_cache.py +++ b/mne/io/_preload_cache.py @@ -5,10 +5,8 @@ # Copyright the MNE-Python contributors. import hashlib -import json import os import pickle -import stat from pathlib import Path import numpy as np @@ -20,20 +18,18 @@ _RAW_PRELOAD_LOCK_TIMEOUT = 300.0 -def _raw_preload_open_regular(path): - """Open and validate a regular cache file.""" - file = open(path, "rb") - try: - if not stat.S_ISREG(os.fstat(file.fileno()).st_mode): - raise OSError("Decoded data cache entries must be regular files") - except Exception: - file.close() - raise - return file - +def _raw_preload_cache_info(raw): + """Return the cache path and decoded array description.""" + cache_root = get_config("MNE_CACHE_DIR", None) + if cache_root is None: + raise ValueError( + 'preload="auto" requires a configured cache directory; use ' + "mne.set_cache_dir(...) first" + ) + cache_dir = Path(cache_root).expanduser().resolve() + cache_dir = cache_dir / f"raw-preload-v{_RAW_PRELOAD_CACHE_VERSION}" + cache_dir.mkdir(mode=0o700, parents=True, exist_ok=True) -def _raw_preload_source_signature(raw): - """Return filesystem identities for the source data files.""" sources = [] for filename in raw.filenames: if filename is None: @@ -42,206 +38,71 @@ def _raw_preload_source_signature(raw): "or an explicit memory-map path" ) path = Path(filename).resolve(strict=True) - if path.suffix == ".gz": - raise ValueError( - 'preload="auto" supports only uncompressed source files; use ' - "preload=True for compressed files" - ) result = path.stat() - if not stat.S_ISREG(result.st_mode): - raise OSError("Raw source data must be regular files") - # ponytail: hash contents only if path, size, and mtime prove insufficient. sources.append((str(path), int(result.st_size), int(result.st_mtime_ns))) - return sources - -def _raw_preload_cache_dir(cache_root=None): - """Resolve and validate the managed cache directory.""" - if cache_root is None: - cache_root = get_config("MNE_CACHE_DIR", None) - if cache_root is None: - raise ValueError( - 'preload="auto" requires a configured cache directory; use ' - "mne.set_cache_dir(...) first" - ) - cache_root = Path(cache_root).expanduser().resolve() - cache_dir = cache_root / f"raw-preload-v{_RAW_PRELOAD_CACHE_VERSION}" - cache_dir.mkdir(mode=0o700, parents=True, exist_ok=True) - if cache_dir.is_symlink() or not cache_dir.is_dir(): - raise OSError(f"Decoded data cache must be a regular directory: {cache_dir}") - return cache_dir - - -def _raw_preload_cache_info(raw): - """Return the managed cache location and expected array description.""" - cache_dir = _raw_preload_cache_dir() - sources = _raw_preload_source_signature(raw) dtype = np.dtype(raw._dtype) shape = (int(raw.info["nchan"]), int(raw.n_times)) - identity = dict( - version=_RAW_PRELOAD_CACHE_VERSION, - mne_version=MNE_VERSION, - reader=(type(raw).__module__, type(raw).__qualname__), - sources=sources, - raw_extras=raw._raw_extras, - read_picks=raw._read_picks, - cals=raw._cals, - projector=raw._projector, - compensator=raw._comp, - first_samps=raw._first_samps, - last_samps=raw._last_samps, - dtype=dtype.str, - shape=shape, + identity = ( + _RAW_PRELOAD_CACHE_VERSION, + MNE_VERSION, + type(raw).__module__, + type(raw).__qualname__, + sources, + raw._raw_extras, + raw._cals, + dtype.str, + shape, ) try: - serialized = pickle.dumps(identity, protocol=5) + key = hashlib.sha256(pickle.dumps(identity, protocol=5)).hexdigest() except Exception as exc: raise ValueError( f'preload="auto" cannot identify this {type(raw).__name__} source' ) from exc - key = hashlib.sha256(serialized).hexdigest() - return cache_dir, key, sources, shape, dtype - - -def _raw_preload_generation_valid(name, key): - """Check that a manifest generation is a managed basename.""" - prefix = f"{key}." - suffix = ".data" - if ( - not isinstance(name, str) - or not name.startswith(prefix) - or not name.endswith(suffix) - ): - return False - token = name[len(prefix) : -len(suffix)] - return len(token) == 32 and all(char in "0123456789abcdef" for char in token) + return cache_dir / f"{key}.data", sources, shape, dtype -def _raw_preload_read_manifest(cache_dir, key): - """Read one manifest through its validated handle.""" - path = cache_dir / f"{key}.json" - with _raw_preload_open_regular(path) as file: - if os.fstat(file.fileno()).st_size > 4096: - raise ValueError("Oversized Raw preload manifest") - return json.loads(file.read().decode("utf-8")) - - -def _raw_preload_cache_read(raw, cache_dir, key, sources, shape, dtype): - """Read and validate one managed decoded-data cache entry.""" +def _raw_preload_cache_read(path, shape, dtype): + """Map a complete decoded-data cache entry.""" try: - manifest = _raw_preload_read_manifest(cache_dir, key) - if set(manifest) != {"generation"}: - return None nbytes = int(np.prod(shape, dtype=np.int64)) * dtype.itemsize - if not _raw_preload_generation_valid(manifest["generation"], key): + if path.stat().st_size != nbytes: return None - generation = cache_dir / manifest["generation"] - with _raw_preload_open_regular(generation) as file: - if os.fstat(file.fileno()).st_size != nbytes: - return None - data = np.memmap(file, mode="c", dtype=dtype, shape=shape) - data.filename = str(generation) # ty: ignore[invalid-assignment] - if _raw_preload_source_signature(raw) != sources: - data._mmap.close() # ty: ignore[unresolved-attribute] # memmap private - return None - except (OSError, TypeError, ValueError, json.JSONDecodeError): + return np.memmap(path, mode="c", dtype=dtype, shape=shape) + except OSError: return None - logger.info(f"Reusing decoded data from {generation}") - return data - - -def _raw_preload_scavenge_key(cache_dir, key): - """Remove abandoned temporary and unreferenced same-key generations.""" - referenced = None - try: - manifest = _raw_preload_read_manifest(cache_dir, key) - candidate = manifest.get("generation") - if _raw_preload_generation_valid(candidate, key): - referenced = candidate - except (OSError, TypeError, ValueError, json.JSONDecodeError): - pass - patterns = (f".{key}.*.tmp", f"{key}.*.data") - for pattern in patterns: - for path in cache_dir.glob(pattern): - if path.name == referenced: - continue - try: - path.unlink() - except OSError: - logger.debug( - f"Could not remove abandoned Raw preload cache file {path}" - ) - - -def _raw_preload_cache_create(raw, cache_dir, key, sources, shape, dtype): - """Decode, durably publish, and reopen an immutable cache generation.""" - token = os.urandom(16).hex() - generation_name = f"{key}.{token}.data" - generation = cache_dir / generation_name - temporary = cache_dir / f".{generation_name}.tmp" - manifest_temporary = None - manifest_published = False - nbytes = int(np.prod(shape, dtype=np.int64)) * dtype.itemsize - descriptor = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) - try: - with os.fdopen(descriptor, "r+b") as file: - file.truncate(nbytes) - data = np.memmap(file, mode="r+", dtype=dtype, shape=shape) - try: - raw._read_segment(data_buffer=data) - data.flush() - finally: - data._mmap.close() # ty: ignore[unresolved-attribute] # memmap private - os.fsync(file.fileno()) - if _raw_preload_source_signature(raw) != sources: - raise RuntimeError( - "Source data changed while decoded cache was created; retry" - ) - os.replace(temporary, generation) - manifest = dict(generation=generation_name) - manifest_temporary = cache_dir / f".{key}.{os.urandom(16).hex()}.json.tmp" - descriptor = os.open( - manifest_temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600 - ) - with os.fdopen(descriptor, "w", encoding="utf-8") as file: - json.dump(manifest, file, sort_keys=True, separators=(",", ":")) - file.flush() - os.fsync(file.fileno()) - os.replace(manifest_temporary, cache_dir / f"{key}.json") - manifest_published = True - with _raw_preload_open_regular(generation) as file: - result = np.memmap(file, mode="c", dtype=dtype, shape=shape) - result.filename = str(generation) # ty: ignore[invalid-assignment] - return result - finally: - for path in (temporary, manifest_temporary): - if path is not None: - try: - path.unlink(missing_ok=True) - except OSError: - pass - if not manifest_published: - try: - generation.unlink(missing_ok=True) - except OSError: - pass def _raw_preload_auto(raw): """Reuse or create an automatic decoded-data cache entry.""" - cache_dir, key, sources, shape, dtype = _raw_preload_cache_info(raw) - key_lock = cache_dir / f"{key}.lock" - data = _raw_preload_cache_read(raw, cache_dir, key, sources, shape, dtype) + path, sources, shape, dtype = _raw_preload_cache_info(raw) + data = _raw_preload_cache_read(path, shape, dtype) if data is not None: + logger.info(f"Reusing decoded data from {path}") return data + # Importing filelock is measurable, so keep it off the cache-hit path. filelock = _soft_import("filelock", "locking the decoded-data cache") - - with filelock.FileLock(key_lock, timeout=_RAW_PRELOAD_LOCK_TIMEOUT): - _raw_preload_scavenge_key(cache_dir, key) - data = _raw_preload_cache_read(raw, cache_dir, key, sources, shape, dtype) + with filelock.FileLock(f"{path}.lock", timeout=_RAW_PRELOAD_LOCK_TIMEOUT): + data = _raw_preload_cache_read(path, shape, dtype) if data is None: - logger.info(f"Creating decoded data cache in {cache_dir}") - data = _raw_preload_cache_create(raw, cache_dir, key, sources, shape, dtype) - _raw_preload_scavenge_key(cache_dir, key) + logger.info(f"Creating decoded data cache in {path.parent}") + temporary = path.with_suffix(".tmp") + try: + temporary.unlink(missing_ok=True) + data = np.memmap(temporary, mode="w+", dtype=dtype, shape=shape) + try: + raw._read_segment(data_buffer=data) + data.flush() + finally: + data._mmap.close() # ty: ignore[unresolved-attribute] + if _raw_preload_cache_info(raw)[1] != sources: + raise RuntimeError( + "Source data changed while decoded cache was created; retry" + ) + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + data = _raw_preload_cache_read(path, shape, dtype) return data diff --git a/mne/io/base.py b/mne/io/base.py index 09c894b286c..08d150ffe76 100644 --- a/mne/io/base.py +++ b/mne/io/base.py @@ -145,12 +145,10 @@ class BaseRaw( drive (slower, requires less memory). An existing file is overwritten. The caller owns the file and is responsible for removing it after the Raw object is no longer in use. For supported file readers, the exact - string ``"auto"`` instead stores and reuses decoded data in the directory - configured by :func:`mne.set_cache_dir`. Cached data persist in a - versioned ``raw-preload`` directory and are mapped copy-on-write. Use - ``Path("auto")`` or ``"./auto"`` for a literal filename. If preload is - an ndarray, the data are taken from that array. If False, data are not - read until save. + string ``"auto"`` instead reuses decoded data below the directory + configured by :func:`mne.set_cache_dir`. Use ``Path("auto")`` for a + literal filename. If preload is an ndarray, the data are taken from that + array. If False, data are not read until save. first_samps : sequence Sequence of the first sample number from each raw file. For unsplit raw files this should be a length-one list or tuple. @@ -600,22 +598,18 @@ def _check_bad_segment( def load_data( self, *, - memmap: Path | Literal["auto"] | str | None = None, + memmap: Path | str | None = None, verbose: bool | str | int | None = None, ) -> Self: """Load raw data. Parameters ---------- - memmap : path-like | "auto" | None - If a path, preload data into a freshly created memory-mapped file at - this path. An existing file is overwritten. The caller owns the file - and is responsible for removing it after the Raw object is no longer - in use. For supported file readers, ``"auto"`` instead reuses the - persistent decoded-data cache configured by :func:`mne.set_cache_dir`. - Cache entries for superseded source identities remain in - a versioned ``raw-preload`` directory below the configured path. - If ``None`` (default), preload data into RAM. + memmap : path-like | None + If not ``None``, preload data into a freshly created memory-mapped file + at this path. An existing file is overwritten. The caller owns the file + and is responsible for removing it after the Raw object is no longer in + use. If ``None`` (default), preload data into RAM. .. versionadded:: 1.13 %(verbose)s @@ -635,6 +629,7 @@ def load_data( if not self.preload: if memmap is not None: _validate_type(memmap, "path-like", "memmap") + memmap = Path(memmap) self._preload_data(memmap if memmap is not None else True) return self @@ -650,12 +645,9 @@ def _preload_data(self, preload): data_buffer = preload if isinstance(preload, bool | np.bool_) and not preload: data_buffer = None - # Avoid materializing ``self.times``; that scales with the recording - # length and can dominate a decoded-cache hit. - n_times = self.n_times - last_time = (n_times - 1) / self.info["sfreq"] + t = self.times logger.info( - f"Reading 0 ... {n_times - 1} = {0.0:9.3f} ... {last_time:9.3f} secs..." + f"Reading 0 ... {len(t) - 1} = {0.0:9.3f} ... {t[-1]:9.3f} secs..." ) self._data = self._read_segment(data_buffer=data_buffer) assert len(self._data) == self.info["nchan"] diff --git a/mne/io/fiff/raw.py b/mne/io/fiff/raw.py index 094de8ed049..93cdaecc312 100644 --- a/mne/io/fiff/raw.py +++ b/mne/io/fiff/raw.py @@ -104,11 +104,6 @@ def __init__( 'preload="auto" requires stable source files and is not ' "supported for file-like FIF inputs" ) - if isinstance(fname, Path | str) and Path(fname).suffix == ".gz": - raise ValueError( - 'preload="auto" supports only uncompressed FIF files; use ' - "preload=True for gzip-compressed FIF" - ) raws = [] do_check_ext = not _file_like(fname) next_fname = fname @@ -209,7 +204,9 @@ def _read_raw_file( check_fname(fname, "raw", endings) # filename fname = _check_fname(fname, "read", True, "fname") - whole_file = preload if fname.suffix == ".gz" else False + whole_file = ( + preload if preload != "auto" and fname.suffix == ".gz" else False + ) else: # file-like if not preload: diff --git a/mne/io/fiff/tests/test_raw_fiff.py b/mne/io/fiff/tests/test_raw_fiff.py index d6ead398654..201ccd1afbd 100644 --- a/mne/io/fiff/tests/test_raw_fiff.py +++ b/mne/io/fiff/tests/test_raw_fiff.py @@ -2129,16 +2129,6 @@ def test_file_like_auto_preload_rejected(tmp_path, monkeypatch): read_raw_fif(stream, preload="auto") -def test_compressed_auto_preload_rejected(tmp_path, monkeypatch): - """Test that gzip FIF does not advertise ineffective decoded caching.""" - monkeypatch.setenv("MNE_CACHE_DIR", str(tmp_path)) - with pytest.raises(ValueError, match="uncompressed FIF"): - read_raw_fif(test_fif_gz_fname, preload="auto") - raw = read_raw_fif(test_fif_gz_fname, preload=False) - with pytest.raises(ValueError, match="uncompressed"): - raw.load_data(memmap="auto") - - def test_str_like(): """Test handling with str-like objects.""" fname = pathlib.Path(test_fif_fname) diff --git a/mne/io/tests/test_preload_cache.py b/mne/io/tests/test_preload_cache.py index 9b7fae5225e..a456112dfa5 100644 --- a/mne/io/tests/test_preload_cache.py +++ b/mne/io/tests/test_preload_cache.py @@ -6,7 +6,6 @@ import gc import hashlib -import multiprocessing import os import shutil from concurrent.futures import ProcessPoolExecutor @@ -20,9 +19,8 @@ import mne from mne._fiff.pick import pick_info from mne.io import RawArray, _preload_cache -from mne.io.tests.test_raw import _RawArange, _read_raw_arange +from mne.io.tests.test_raw import _read_raw_arange -_ORIGINAL_CACHE_REPLACE = None _IO_DATA_DIR = Path(mne.io.__file__).parent @@ -34,23 +32,6 @@ def _auto_preload_process(reader_name, source, cache_dir): return raw._data.mode, str(raw._data.filename), digest -def _replace_cache_generation_then_exit(source, destination): - """Crash after publishing data but before publishing its manifest.""" - _ORIGINAL_CACHE_REPLACE(source, destination) - if str(destination).endswith(".data"): - os._exit(91) - - -def _auto_preload_crash_process(source, cache_dir): - """Run the simulated crash in an isolated process.""" - global _ORIGINAL_CACHE_REPLACE - - os.environ["MNE_CACHE_DIR"] = cache_dir - _ORIGINAL_CACHE_REPLACE = _preload_cache.os.replace - _preload_cache.os.replace = _replace_cache_generation_then_exit - mne.io.read_raw_edf(source, preload="auto", verbose="error") - - @pytest.fixture def cache_root(tmp_path, monkeypatch): """Configure an isolated cache directory.""" @@ -60,39 +41,29 @@ def cache_root(tmp_path, monkeypatch): return cache_root -@pytest.fixture -def auto_cache(tmp_path, cache_root): - """Create one stable source file.""" - source = tmp_path / "source.bin" - source.write_bytes(b"source identity") - return source, cache_root - - def test_auto_preload_api(tmp_path, monkeypatch): """Test cache configuration and the literal-path escape.""" - source = tmp_path / "source.bin" - source.write_bytes(b"source identity") + source = _IO_DATA_DIR / "edf/tests/data/test.edf" monkeypatch.setattr(_preload_cache, "get_config", lambda *args, **kwargs: None) with pytest.raises(ValueError, match="set_cache_dir"): - _RawArange(preload="auto", filename=source) + mne.io.read_raw_edf(source, preload="auto", verbose="error") - monkeypatch.setattr( - _preload_cache, "get_config", lambda *args, **kwargs: str(tmp_path) - ) with chdir(tmp_path): - literal = _RawArange(preload=Path("auto"), filename=source) + literal = mne.io.read_raw_edf(source, preload=Path("auto"), verbose="error") assert literal._data.mode == "w+" assert (tmp_path / "auto").is_file() - lazy = _RawArange(preload=False, filename=source) - lazy.load_data(memmap="auto") - assert lazy._data.mode == "c" + lazy = mne.io.read_raw_edf(source, preload=False, verbose="error") + with chdir(tmp_path): + lazy.load_data(memmap="auto") + assert lazy._data.mode == "w+" @pytest.mark.parametrize( ("reader_name", "relative_path"), ( ("read_raw_fif", "tests/data/test_raw.fif"), + ("read_raw_fif", "tests/data/test_raw.fif.gz"), ("read_raw_edf", "edf/tests/data/test.edf"), ("read_raw_bdf", "edf/tests/data/test.bdf"), ("read_raw_brainvision", "brainvision/tests/data/test.vhdr"), @@ -116,8 +87,8 @@ def test_auto_preload_formats(reader_name, relative_path, cache_root): assert_array_equal(other.get_data(), expected) -def test_auto_preload_key(tmp_path, cache_root): - """Test numeric options and source modification invalidate the cache.""" +def test_auto_preload_identity(tmp_path, cache_root): + """Test reader options and source modification invalidate the cache.""" data_dir = _IO_DATA_DIR / "brainvision/tests/data" for name in ("test.vhdr", "test.vmrk", "test.eeg"): shutil.copy(data_dir / name, tmp_path / name) @@ -129,16 +100,6 @@ def test_auto_preload_key(tmp_path, cache_root): assert Path(scaled._data.filename) != Path(raw._data.filename) assert_array_equal(scaled.get_data(), 2.0 * raw.get_data()) - text = source.read_text(encoding="utf-8") - source.write_text( - text.replace("DataOrientation=MULTIPLEXED", "DataOrientation=VECTORIZED"), - encoding="utf-8", - ) - expected = mne.io.read_raw_brainvision(source, preload=True, verbose="error") - changed = mne.io.read_raw_brainvision(source, preload="auto", verbose="error") - assert Path(changed._data.filename) != Path(raw._data.filename) - assert_array_equal(changed.get_data(), expected.get_data()) - source = tmp_path / "copy.edf" shutil.copy(_IO_DATA_DIR / "edf/tests/data/test.edf", source) raw = mne.io.read_raw_edf(source, preload="auto", verbose="error") @@ -149,28 +110,23 @@ def test_auto_preload_key(tmp_path, cache_root): assert Path(other._data.filename) != generation -@pytest.mark.parametrize("corruption", ("manifest", "data")) -def test_auto_preload_recovers_corruption(corruption, auto_cache): - """Test that malformed cache entries become misses.""" - source, cache_root = auto_cache - raw = _RawArange(preload="auto", filename=source) - expected = raw.get_data() +def test_auto_preload_recovers_corruption(cache_root): + """Test that a truncated deterministic cache entry is rebuilt.""" + source = _IO_DATA_DIR / "edf/tests/data/test.edf" + raw = mne.io.read_raw_edf(source, preload="auto", verbose="error") + expected = raw.get_data().copy() generation = Path(raw._data.filename) del raw gc.collect() - cache_dir = next(cache_root.iterdir()) - if corruption == "manifest": - next(cache_dir.glob("*.json")).write_text("{", encoding="utf-8") - else: - generation.write_bytes(b"short") + generation.write_bytes(b"short") - other = _RawArange(preload="auto", filename=source) - assert Path(other._data.filename) != generation + other = mne.io.read_raw_edf(source, preload="auto", verbose="error") + assert Path(other._data.filename) == generation assert_array_equal(other.get_data(), expected) def test_auto_preload_concurrent_misses(cache_root): - """Test that concurrent misses publish one exact generation.""" + """Test that concurrent misses publish one exact cache entry.""" source = _IO_DATA_DIR / "edf/tests/data/test.edf" args = ("read_raw_edf", str(source), str(cache_root)) with ProcessPoolExecutor(max_workers=4) as pool: @@ -179,42 +135,6 @@ def test_auto_preload_concurrent_misses(cache_root): assert {result[0] for result in results} == {"c"} assert len({result[1] for result in results}) == 1 assert len({result[2] for result in results}) == 1 - cache_dir = next(cache_root.iterdir()) - assert len(list(cache_dir.glob("*.data"))) == 1 - - -def test_auto_preload_recovers_crashed_publisher(cache_root): - """Test recovery when a writer dies before manifest publication.""" - source = _IO_DATA_DIR / "edf/tests/data/test.edf" - context = multiprocessing.get_context("spawn") - process = context.Process( - target=_auto_preload_crash_process, args=(str(source), str(cache_root)) - ) - process.start() - process.join(timeout=15) - assert process.exitcode == 91 - - raw = mne.io.read_raw_edf(source, preload="auto", verbose="error") - expected = mne.io.read_raw_edf(source, preload=True, verbose="error").get_data() - assert_array_equal(raw.get_data(), expected) - cache_dir = next(cache_root.iterdir()) - assert len(list(cache_dir.glob("*.data"))) == 1 - assert not list(cache_dir.glob("*.tmp")) - - -def test_auto_preload_rejects_cache_symlink(tmp_path, cache_root): - """Test that the managed cache directory cannot be redirected.""" - source = tmp_path / "source.bin" - source.write_bytes(b"source identity") - outside = tmp_path / "outside" - outside.mkdir() - try: - os.symlink(outside, cache_root / "raw-preload-v1") - except OSError: - pytest.skip("symlink creation is unavailable") - with pytest.raises(OSError, match="regular directory"): - _RawArange(preload="auto", filename=source) - assert not list(outside.iterdir()) def test_add_channels_copy_on_write_memmap(tmp_path, monkeypatch): diff --git a/mne/io/tests/test_raw.py b/mne/io/tests/test_raw.py index ceea153469f..9254f96e902 100644 --- a/mne/io/tests/test_raw.py +++ b/mne/io/tests/test_raw.py @@ -103,13 +103,6 @@ def test_orig_units(): BaseRaw(info, last_samps=[1], orig_units=True) -def test_preload_does_not_materialize_times(monkeypatch): - """Test preloading does not construct the full time vector.""" - monkeypatch.setattr("mne.io.base._arange_div", _fail_if_times_materialized) - raw = read_raw_fif(raw_fname, preload=True, verbose="error") - assert raw.preload - - def test_set_annotations_does_not_materialize_times(monkeypatch): """Test annotation bounds use the scalar recording endpoint.""" raw = read_raw_fif(raw_fname, preload=False, verbose="error") @@ -844,15 +837,9 @@ def test_repr(sfreq): # A class that sets channel data to np.arange, for testing _test_raw_reader class _RawArange(BaseRaw): - def __init__(self, preload=False, filename=None, verbose=None): + def __init__(self, preload=False, verbose=None): info = create_info(list(str(x) for x in range(1, 9)), 1000.0, "eeg") - super().__init__( - info, - preload, - last_samps=(999,), - filenames=(filename,), - verbose=verbose, - ) + super().__init__(info, preload, last_samps=(999,), verbose=verbose) assert len(self.times) == 1000 def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): @@ -861,8 +848,8 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): _mult_cal_one(data, one, idx, cals, mult) -def _read_raw_arange(preload=False, filename=None, verbose=None): - return _RawArange(preload, filename=filename, verbose=verbose) +def _read_raw_arange(preload=False, verbose=None): + return _RawArange(preload, verbose) @pytest.mark.parametrize("method", ("constructor", "load_data")) diff --git a/mne/utils/config.py b/mne/utils/config.py index ced003f5d44..5664b00c5ce 100644 --- a/mne/utils/config.py +++ b/mne/utils/config.py @@ -54,8 +54,6 @@ def set_cache_dir(cache_dir): ----- Persistent decoded Raw entries are not automatically size-limited. They are stored below ``cache_dir`` in a versioned ``raw-preload`` directory. - The configured location is trusted and should only be writable by users who - are allowed to access its cached data. """ if cache_dir is not None and not op.exists(cache_dir): raise OSError(f"Directory {cache_dir} does not exist") diff --git a/mne/utils/docs.py b/mne/utils/docs.py index 445070f2d01..5863a8c7a60 100644 --- a/mne/utils/docs.py +++ b/mne/utils/docs.py @@ -3721,16 +3721,10 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75): freshly created memory-mapped file used to store the data on the hard drive (slower, requires less memory). An existing file is overwritten. The caller owns the file and is responsible for removing it after the - Raw object is no longer in use. For uncompressed FIF, EDF/BDF, and - BrainVision readers, the exact string ``"auto"`` instead stores and reuses - decoded data in the directory configured by :func:`mne.set_cache_dir`. - Cached data persist in a versioned ``raw-preload`` directory below the - configured cache path and are mapped copy-on-write, so modifying the returned - Raw does not modify later reads. A cache miss performs the normal full decode. - Valid entries for historical source identities are retained without an - automatic size limit. The configured cache path is fixed to its physical - location. Use ``Path("auto")``, - ``"./auto"``, or an absolute path to create a file literally named ``auto``. + Raw object is no longer in use. For supported Raw readers, the exact string + ``"auto"`` instead reuses decoded data below the directory configured by + :func:`mne.set_cache_dir`. Entries persist without a size limit and are mapped + copy-on-write. Use ``Path("auto")`` for a literal filename. .. versionchanged:: 1.13 Support for the ``"auto"`` decoded-data cache was added.""" From 8e0a2474c33871bb8d2339c2154e1ec24d8eb382 Mon Sep 17 00:00:00 2001 From: Bru Date: Thu, 27 Aug 2026 17:43:37 +0200 Subject: [PATCH 6/9] perf: keep the Raw preload cache lock-free and lift the "auto" sentinel Publish cache entries through a per-process temporary and os.replace instead of a FileLock, so filelock stays an optional dependency rather than a required one. Make load_data(memmap="auto") resolve the same sentinel as preload="auto"; previously it meant a file literally named "auto". Stop reusing a live-mapped filename in test_auto_preload_api, which Windows rejects when truncating a file that still has a mapping. --- README.rst | 1 - doc/changes/dev/14216.newfeature.rst | 3 +- mne/io/_preload_cache.py | 51 ++++++++++++++-------------- mne/io/base.py | 11 ++++-- mne/io/tests/test_preload_cache.py | 4 +-- pyproject.toml | 2 +- tools/pylock.ci-old.toml | 6 ---- 7 files changed, 38 insertions(+), 40 deletions(-) diff --git a/README.rst b/README.rst index 2068e97a2b4..3508426cba8 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,6 @@ The minimum required dependencies to run MNE-Python are: - `Python `__ ≥ 3.11 - `decorator `__ ≥ 5.1 -- `filelock `__ ≥ 3.18.0 - `Jinja2 `__ ≥ 3.1 - `lazy-loader `__ ≥ 0.3 - `Matplotlib `__ ≥ 3.9 diff --git a/doc/changes/dev/14216.newfeature.rst b/doc/changes/dev/14216.newfeature.rst index 9ef32b674c2..1a69f50d896 100644 --- a/doc/changes/dev/14216.newfeature.rst +++ b/doc/changes/dev/14216.newfeature.rst @@ -1,2 +1 @@ -Speed up repeated preloading of FIF, EDF/BDF, and BrainVision -recordings with a persistent copy-on-write decoded-data cache, by `Bruno Aristimunha`_. +Speed up repeated preloading of any file-backed :class:`~mne.io.Raw` with ``preload="auto"``, which persists decoded data in a copy-on-write cache below :func:`mne.set_cache_dir`, by `Bruno Aristimunha`_. diff --git a/mne/io/_preload_cache.py b/mne/io/_preload_cache.py index 53b4772cd4c..453c51e0420 100644 --- a/mne/io/_preload_cache.py +++ b/mne/io/_preload_cache.py @@ -12,10 +12,9 @@ import numpy as np from .. import __version__ as MNE_VERSION # ty: ignore[unresolved-import] -from ..utils import _soft_import, get_config, logger +from ..utils import get_config, logger _RAW_PRELOAD_CACHE_VERSION = 1 -_RAW_PRELOAD_LOCK_TIMEOUT = 300.0 def _raw_preload_cache_info(raw): @@ -82,27 +81,29 @@ def _raw_preload_auto(raw): logger.info(f"Reusing decoded data from {path}") return data - # Importing filelock is measurable, so keep it off the cache-hit path. - filelock = _soft_import("filelock", "locking the decoded-data cache") - with filelock.FileLock(f"{path}.lock", timeout=_RAW_PRELOAD_LOCK_TIMEOUT): - data = _raw_preload_cache_read(path, shape, dtype) - if data is None: - logger.info(f"Creating decoded data cache in {path.parent}") - temporary = path.with_suffix(".tmp") - try: - temporary.unlink(missing_ok=True) - data = np.memmap(temporary, mode="w+", dtype=dtype, shape=shape) - try: - raw._read_segment(data_buffer=data) - data.flush() - finally: - data._mmap.close() # ty: ignore[unresolved-attribute] - if _raw_preload_cache_info(raw)[1] != sources: - raise RuntimeError( - "Source data changed while decoded cache was created; retry" - ) - os.replace(temporary, path) - finally: - temporary.unlink(missing_ok=True) - data = _raw_preload_cache_read(path, shape, dtype) + # The temporary is per-process and os.replace is atomic, so concurrent + # misses need no lock; they at worst decode the same entry twice. + logger.info(f"Creating decoded data cache in {path.parent}") + temporary = path.with_suffix(f".{os.getpid()}.tmp") + try: + data = np.memmap(temporary, mode="w+", dtype=dtype, shape=shape) + try: + raw._read_segment(data_buffer=data) + data.flush() + finally: + data._mmap.close() # ty: ignore[unresolved-attribute] + if _raw_preload_cache_info(raw)[1] != sources: + raise RuntimeError( + "Source data changed while decoded cache was created; retry" + ) + try: + os.replace(temporary, path) + except OSError: + # Windows refuses to replace an entry another process already mapped. + pass + finally: + temporary.unlink(missing_ok=True) + data = _raw_preload_cache_read(path, shape, dtype) + if data is None: + raise RuntimeError(f"Could not read back the decoded data cache at {path}") return data diff --git a/mne/io/base.py b/mne/io/base.py index 08d150ffe76..587c9c084df 100644 --- a/mne/io/base.py +++ b/mne/io/base.py @@ -605,11 +605,14 @@ def load_data( Parameters ---------- - memmap : path-like | None + memmap : path-like | str | None If not ``None``, preload data into a freshly created memory-mapped file at this path. An existing file is overwritten. The caller owns the file and is responsible for removing it after the Raw object is no longer in - use. If ``None`` (default), preload data into RAM. + use. The exact string ``"auto"`` instead means the same as + ``preload="auto"``: reuse decoded data below the directory configured by + :func:`mne.set_cache_dir`. Use ``Path("auto")`` for a literal filename. + If ``None`` (default), preload data into RAM. .. versionadded:: 1.13 %(verbose)s @@ -627,7 +630,9 @@ def load_data( .. versionadded:: 0.10.0 """ if not self.preload: - if memmap is not None: + if isinstance(memmap, str) and memmap == "auto": + pass # sentinel, resolved in _preload_data + elif memmap is not None: _validate_type(memmap, "path-like", "memmap") memmap = Path(memmap) self._preload_data(memmap if memmap is not None else True) diff --git a/mne/io/tests/test_preload_cache.py b/mne/io/tests/test_preload_cache.py index a456112dfa5..9e077f12073 100644 --- a/mne/io/tests/test_preload_cache.py +++ b/mne/io/tests/test_preload_cache.py @@ -53,10 +53,10 @@ def test_auto_preload_api(tmp_path, monkeypatch): assert literal._data.mode == "w+" assert (tmp_path / "auto").is_file() + # load_data(memmap="auto") resolves the same sentinel as preload="auto" lazy = mne.io.read_raw_edf(source, preload=False, verbose="error") - with chdir(tmp_path): + with pytest.raises(ValueError, match="set_cache_dir"): lazy.load_data(memmap="auto") - assert lazy._data.mode == "w+" @pytest.mark.parametrize( diff --git a/pyproject.toml b/pyproject.toml index e1ac39f69d5..2ea1ff516dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,7 +115,6 @@ classifiers = [ ] dependencies = [ "decorator >= 5.1", - "filelock >= 3.18.0", "jinja2 >= 3.1", "lazy_loader >= 0.3", "matplotlib >= 3.9", # released 2024-05-15, will become 3.10 on 2026-12-14 @@ -171,6 +170,7 @@ full-no-qt = [ "dipy >= 1.9", # released 2024-03-08, will become 1.10 on 2026-12-12 "edfio >= 0.4.10", "eeglabio", + "filelock >= 3.18.0", "h5py >= 2.4", "imageio >= 2.6.1", "imageio-ffmpeg >= 0.4.1", diff --git a/tools/pylock.ci-old.toml b/tools/pylock.ci-old.toml index ae68bfc727e..340ee161f77 100644 --- a/tools/pylock.ci-old.toml +++ b/tools/pylock.ci-old.toml @@ -93,12 +93,6 @@ version = "2.2.1" sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", upload-time = 2025-09-01T09:48:10Z, size = 1129488, hashes = { sha256 = "3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4" } } wheels = [{ url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", upload-time = 2025-09-01T09:48:08Z, size = 28317, hashes = { sha256 = "760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017" } }] -[[packages]] -name = "filelock" -version = "3.18.0" -sdist = { url = "https://files.pythonhosted.org/packages/0a/10/c23352565a6544bdc5353e0b15fc1c563352101f30e24bf500207a54df9a/filelock-3.18.0.tar.gz", upload-time = 2025-03-14T07:11:40Z, size = 18075, hashes = { sha256 = "adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2" } } -wheels = [{ url = "https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl", upload-time = 2025-03-14T07:11:39Z, size = 16215, hashes = { sha256 = "c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de" } }] - [[packages]] name = "fonttools" version = "4.61.1" From bf5a3d0f606e2efb32b035b2e1dcd69f3f24844d Mon Sep 17 00:00:00 2001 From: Bru Date: Fri, 28 Aug 2026 11:55:42 +0200 Subject: [PATCH 7/9] Test preload="auto" in _test_raw_reader across formats Move the per-format preload="auto" coverage into _test_raw_reader so every reader with test_preloading=True exercises the decoded-data cache, stat directory sources (e.g. CTF .ds) member-wise, and make RawCurry honor a non-bool preload instead of silently ignoring it. --- mne/io/_preload_cache.py | 9 +++++++-- mne/io/curry/curry.py | 5 +++-- mne/io/tests/test_preload_cache.py | 25 +++++++------------------ mne/io/tests/test_raw.py | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 22 deletions(-) diff --git a/mne/io/_preload_cache.py b/mne/io/_preload_cache.py index 453c51e0420..4500d75c10e 100644 --- a/mne/io/_preload_cache.py +++ b/mne/io/_preload_cache.py @@ -37,8 +37,13 @@ def _raw_preload_cache_info(raw): "or an explicit memory-map path" ) path = Path(filename).resolve(strict=True) - result = path.stat() - sources.append((str(path), int(result.st_size), int(result.st_mtime_ns))) + # some formats (e.g., CTF) name a directory rather than a single file + members = sorted(path.rglob("*")) if path.is_dir() else [path] + for member in members: + if not member.is_file(): + continue + result = member.stat() + sources.append((str(member), int(result.st_size), int(result.st_mtime_ns))) dtype = np.dtype(raw._dtype) shape = (int(raw.info["nchan"]), int(raw.n_times)) diff --git a/mne/io/curry/curry.py b/mne/io/curry/curry.py index 50b87381f8e..c39471694db 100644 --- a/mne/io/curry/curry.py +++ b/mne/io/curry/curry.py @@ -807,8 +807,9 @@ def __init__(self, fname, preload=False, on_bad_hpi_match="warn", verbose=None): # scale data to SI units self._cals = np.array(cals) - if isinstance(preload, bool | np.bool_) and preload: - self.load_data() + if not isinstance(preload, bool | np.bool_) or preload: + # preload can also be a memory-map path or the "auto" sentinel + self._preload_data(preload) # set events / annotations # format from curryreader: sample, etype, startsample, endsample diff --git a/mne/io/tests/test_preload_cache.py b/mne/io/tests/test_preload_cache.py index 9e077f12073..e87311cb7fe 100644 --- a/mne/io/tests/test_preload_cache.py +++ b/mne/io/tests/test_preload_cache.py @@ -59,30 +59,19 @@ def test_auto_preload_api(tmp_path, monkeypatch): lazy.load_data(memmap="auto") -@pytest.mark.parametrize( - ("reader_name", "relative_path"), - ( - ("read_raw_fif", "tests/data/test_raw.fif"), - ("read_raw_fif", "tests/data/test_raw.fif.gz"), - ("read_raw_edf", "edf/tests/data/test.edf"), - ("read_raw_bdf", "edf/tests/data/test.bdf"), - ("read_raw_brainvision", "brainvision/tests/data/test.vhdr"), - ), -) -def test_auto_preload_formats(reader_name, relative_path, cache_root): - """Test exact copy-on-write cache reuse across file formats.""" - source = _IO_DATA_DIR / relative_path - reader = getattr(mne.io, reader_name) - expected = reader(source, preload=True, verbose="error").get_data() - raw = reader(source, preload="auto", verbose="error") +@pytest.mark.parametrize("fname", ("test_raw.fif", "test_raw.fif.gz")) +def test_auto_preload_fif(fname, cache_root): + """Test cache reuse for FIF, whose reader tests skip test_preloading.""" + source = _IO_DATA_DIR / "tests/data" / fname + expected = mne.io.read_raw_fif(source, preload=True, verbose="error").get_data() + raw = mne.io.read_raw_fif(source, preload="auto", verbose="error") generation = Path(raw._data.filename) assert raw._data.mode == "c" assert_array_equal(raw.get_data(), expected) - raw._data[0, 0] += 1.0 del raw gc.collect() - other = reader(source, preload="auto", verbose="error") + other = mne.io.read_raw_fif(source, preload="auto", verbose="error") assert Path(other._data.filename) == generation assert_array_equal(other.get_data(), expected) diff --git a/mne/io/tests/test_raw.py b/mne/io/tests/test_raw.py index 9254f96e902..0c16ca9e43d 100644 --- a/mne/io/tests/test_raw.py +++ b/mne/io/tests/test_raw.py @@ -6,11 +6,13 @@ import gc import math +import os import re from contextlib import chdir, redirect_stdout from io import StringIO from os import path as op from pathlib import Path +from unittest import mock import numpy as np import pytest @@ -182,6 +184,22 @@ def _test_raw_reader( assert_allclose(data1, data2, err_msg="Data mismatch with preload") assert_allclose(times1, times2) + # preload="auto" decodes once into a reusable cache entry (gh-14216) + if None not in raw.filenames: # e.g. RawArray has no source file + with mock.patch.dict(os.environ, {"MNE_CACHE_DIR": tempdir}): + entries = set() + for _ in range(2): # miss, then hit + auto = reader(preload="auto", **kwargs) + assert_allclose(auto[picks, :][0], raw[picks, :][0]) + # readers that hand BaseRaw an in-memory array (e.g. EEGLAB + # with embedded data) never reach the cache + if isinstance(auto._data, np.memmap): + assert auto._data.mode == "c" + entries.add(str(auto._data.filename)) + del auto + gc.collect() + assert len(entries) in (0, 1) + # test projection vs cals and data units other_raw = reader(preload=False, **kwargs) other_raw.del_proj() From 3a283277651d75a81aee56f3990a07a1b7fc4aee Mon Sep 17 00:00:00 2001 From: Bru Date: Fri, 28 Aug 2026 12:16:03 +0200 Subject: [PATCH 8/9] Cover FIL in _test_raw_reader Teach the generic reader test about the binfile keyword and run it for read_raw_fil, which also exercises preload="auto" for that format. --- mne/io/fil/tests/test_fil.py | 2 ++ mne/io/tests/test_raw.py | 1 + 2 files changed, 3 insertions(+) diff --git a/mne/io/fil/tests/test_fil.py b/mne/io/fil/tests/test_fil.py index af1a63303dd..1ff3692620c 100644 --- a/mne/io/fil/tests/test_fil.py +++ b/mne/io/fil/tests/test_fil.py @@ -13,6 +13,7 @@ from mne.datasets import testing from mne.io import read_raw_fil from mne.io.fil.sensors import _get_pos_units +from mne.io.tests.test_raw import _test_raw_reader from mne.utils import copytree_rw fil_path = testing.data_path(download=False) / "FIL" @@ -155,6 +156,7 @@ def test_fil_complete(): _fil_megmag(raw, mat) _fil_stim(raw, mat) _fil_sensorpos(raw, mat) + _test_raw_reader(read_raw_fil, binfile=binname) @testing.requires_testing_data diff --git a/mne/io/tests/test_raw.py b/mne/io/tests/test_raw.py index 0c16ca9e43d..b6593046df8 100644 --- a/mne/io/tests/test_raw.py +++ b/mne/io/tests/test_raw.py @@ -497,6 +497,7 @@ def _test_raw_reader( "pdf_fname", # BTi "directory", # CTF "filename", # nedf + "binfile", # FIL ): try: fname = kwargs[key] From b282e6d0d547f3b098eb6021e0acd238b246909d Mon Sep 17 00:00:00 2001 From: Bru Date: Fri, 28 Aug 2026 12:20:57 +0200 Subject: [PATCH 9/9] Apply projectors in the ANT reader RawANT._read_segment_file assigned every channel into the output buffer, which fails whenever a projector shrinks it, so route the chunk through _mult_cal_one and cover the reader with _test_raw_reader. --- doc/changes/dev/14216.bugfix.rst | 1 + mne/io/ant/ant.py | 7 ++----- mne/io/ant/tests/test_ant.py | 7 +++++++ 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 doc/changes/dev/14216.bugfix.rst diff --git a/doc/changes/dev/14216.bugfix.rst b/doc/changes/dev/14216.bugfix.rst new file mode 100644 index 00000000000..da8a39acfaa --- /dev/null +++ b/doc/changes/dev/14216.bugfix.rst @@ -0,0 +1 @@ +Fix bug where :func:`mne.io.read_raw_ant` and :func:`mne.io.read_raw_curry` ignored projections and non-boolean ``preload`` values respectively when reading data lazily, by `Bruno Aristimunha`_. diff --git a/mne/io/ant/ant.py b/mne/io/ant/ant.py index 8d42ec84e4d..eeab827f31c 100644 --- a/mne/io/ant/ant.py +++ b/mne/io/ant/ant.py @@ -11,6 +11,7 @@ from ..._fiff.constants import FIFF from ..._fiff.meas_info import create_info +from ..._fiff.utils import _mult_cal_one from ...annotations import Annotations from ...utils import ( _check_fname, @@ -185,11 +186,7 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): one = read_data(cnt, i_start, i_stop) _scale_data(one, ch_units) data_view = data[:, i_start - start : i_stop - start] - if isinstance(idx, slice): - data_view[:] = one[idx] - else: - # faster than doing one = one[idx] - np.take(one, idx, axis=0, out=data_view) + _mult_cal_one(data_view, one, idx, cals, mult) def _handle_bipolar_channels( diff --git a/mne/io/ant/tests/test_ant.py b/mne/io/ant/tests/test_ant.py index 6a39469d929..a6fdc2d8006 100644 --- a/mne/io/ant/tests/test_ant.py +++ b/mne/io/ant/tests/test_ant.py @@ -14,6 +14,7 @@ from mne.datasets import testing from mne.io import BaseRaw, read_raw, read_raw_ant, read_raw_brainvision from mne.io.ant.ant import RawANT +from mne.io.tests.test_raw import _test_raw_reader pytest.importorskip("antio", minversion="0.5.0") data_path = testing.data_path(download=False) / "antio" @@ -428,6 +429,12 @@ def test_annotations_and_preload(ca_208: TypeDataset): assert raw_cnt.annotations.description[0] == "impedance" +@testing.requires_testing_data +def test_ant_raw_reader(ca_208: TypeDataset): + """Test the generic reader checks, including projected lazy reads.""" + _test_raw_reader(read_raw_ant, fname=ca_208["cnt"]["short"]) + + @testing.requires_testing_data def test_read_raw(ca_208: TypeDataset): """Test loading through read_raw."""