From 12b455c449c2a30b56d839d1f7c23d5d0a0c2293 Mon Sep 17 00:00:00 2001 From: Bru Date: Fri, 28 Aug 2026 15:28:27 +0200 Subject: [PATCH 1/2] Read BrainVision data in cache-sized blocks _read_segments_file read up to 100 MB at a time. Letting a caller choose a smaller budget and having BrainVision ask for 8 MiB keeps the working set in cache: preloading a 101 MB .eeg (32 channels, 404 MB decoded) goes from 61.5-63.4 ms to 54.0-55.3 ms, min of 5 per process over 3 interleaved A/B rounds against a pristine main worktree. The default is unchanged, so the other eight callers of _read_segments_file read exactly as before. Output is bit-identical on all 13 BrainVision fixtures plus the 101 MB file. block_size is also floored at one channel frame, since a budget smaller than a frame would otherwise leave it at zero and the read would not advance. --- doc/changes/dev/14241.newfeature.rst | 1 + mne/_fiff/utils.py | 9 +++++++-- mne/io/brainvision/brainvision.py | 4 ++++ mne/io/brainvision/tests/test_brainvision.py | 13 +++++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 doc/changes/dev/14241.newfeature.rst diff --git a/doc/changes/dev/14241.newfeature.rst b/doc/changes/dev/14241.newfeature.rst new file mode 100644 index 00000000000..80a0e6095be --- /dev/null +++ b/doc/changes/dev/14241.newfeature.rst @@ -0,0 +1 @@ +Speed up :func:`mne.io.read_raw_brainvision` by reading binary data in cache-sized blocks, by `Bruno Aristimunha`_. diff --git a/mne/_fiff/utils.py b/mne/_fiff/utils.py index 2d9c0b0d53c..3d1e2b1e0f1 100644 --- a/mne/_fiff/utils.py +++ b/mne/_fiff/utils.py @@ -214,6 +214,7 @@ def _read_segments_file( n_channels=None, offset=0, trigger_ch=None, + max_block_bytes=int(100e6), ): """Read a chunk of raw data.""" if n_channels is None: @@ -225,8 +226,12 @@ def _read_segments_file( data_offset = n_channels * start * n_bytes + offset data_left = (stop - start) * n_channels - # Read up to 100 MB of data at a time, block_size is in data samples - block_size = ((int(100e6) // n_bytes) // n_channels) * n_channels + # Read up to max_block_bytes of data at a time; block_size is in data + # samples and spans whole channel frames + # never below one complete channel frame, or the loop would not advance + block_size = max( + n_channels, ((max_block_bytes // n_bytes) // n_channels) * n_channels + ) block_size = min(data_left, block_size) with open(raw.filenames[fi], "rb", buffering=0) as fid: fid.seek(data_offset) diff --git a/mne/io/brainvision/brainvision.py b/mne/io/brainvision/brainvision.py index 9c9d978a861..7cd869b3669 100644 --- a/mne/io/brainvision/brainvision.py +++ b/mne/io/brainvision/brainvision.py @@ -35,6 +35,9 @@ ) from ..base import BaseRaw +# read in cache-sized blocks rather than one huge one +_BLOCK_BYTES = 8 * 1024**2 + @fill_doc class RawBrainVision(BaseRaw): @@ -191,6 +194,7 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): mult, dtype=dtype, n_channels=n_data_ch, + max_block_bytes=_BLOCK_BYTES, ) else: offsets = self._raw_extras[fi]["offsets"] diff --git a/mne/io/brainvision/tests/test_brainvision.py b/mne/io/brainvision/tests/test_brainvision.py index 3a500dc428c..377394f5eb7 100644 --- a/mne/io/brainvision/tests/test_brainvision.py +++ b/mne/io/brainvision/tests/test_brainvision.py @@ -20,6 +20,7 @@ from mne.annotations import events_from_annotations from mne.datasets import testing from mne.io import read_raw_brainvision, read_raw_fif +from mne.io.brainvision import brainvision from mne.io.tests.test_raw import _test_raw_reader from mne.utils import _record_warnings, _stamp_to_dt, object_diff @@ -1163,3 +1164,15 @@ def test_ahdr_format(): assert raw.info["nchan"] == expected_num_channels assert raw.info["highpass"] == expected_hp assert raw.info["lowpass"] == expected_lp + + +@pytest.mark.parametrize("block_bytes", (100, 1)) # partial frame, sub-frame +def test_read_block_size_does_not_change_data(block_bytes, monkeypatch): + """Test the size of the read blocks does not change the decoded data.""" + want = read_raw_brainvision(vhdr_path, preload=True).get_data() + # neither budget is a whole number of channel frames, so the read splits + # into many blocks whose edges must still land on frame boundaries; a + # budget below one frame must still make progress rather than hang + monkeypatch.setattr(brainvision, "_BLOCK_BYTES", block_bytes) + got = read_raw_brainvision(vhdr_path, preload=True).get_data() + assert_array_equal(want, got) From 475ada57575e7da6071c6a7f363eecc1564a0c5e Mon Sep 17 00:00:00 2001 From: Bru Date: Fri, 28 Aug 2026 20:45:52 +0200 Subject: [PATCH 2/2] Read Artemis123, Curry, EEGLAB, Eximia, FIL and NSx in cache-sized blocks Follow-up to #14241, which added max_block_bytes= to _read_segments_file and had BrainVision request 8 MiB. A maintainer asked for the same treatment on other formats. Six more readers opt in. The block size is measured per reader rather than copied, because the optimum is not a fixed number of bytes -- it tracks the time points per block, so it moves with the channel count: eximia 2 MiB 2.4x on a 102 MB file nsx 4 MiB 2.3x on a 102 MB file eeglab 4 MiB 2.1x on a 102 MB file fil 16 MiB 1.9x on a 197 MB file artemis123 16 MiB 1.5x on a 176 MB file curry 16 MiB 1.5x on a 107 MB file Sizes were also chosen so no shipped fixture regresses. Copying BrainVision's 8 MiB everywhere would have: it measured 0.83x on Curry's real 11.8 MB fixture and 0.81-0.85x on Artemis123's 17.6 MB one, because it splits a read that otherwise fits in a single block. FIL is 16 MiB rather than the faster 4 MiB for the same reason -- 4 MiB is 2.5x on a large file but 0.96x on the 9.8 MB fixture. In-process A/B varying only the constant now shows every real fixture within noise: eximia 1.003, fil 1.001, curry 1.003, artemis123 0.988. Output is bit-identical to main: 25/25 arrays compared with np.array_equal across every readable fixture of the six formats. pytest over the six readers plus brainvision and _fiff: 305 passed. --- doc/changes/dev/14246.newfeature.rst | 1 + mne/io/artemis123/artemis123.py | 17 ++++++++++++++++- mne/io/curry/curry.py | 16 +++++++++++++++- mne/io/eeglab/eeglab.py | 18 +++++++++++++++++- mne/io/eximia/eximia.py | 17 ++++++++++++++++- mne/io/fil/fil.py | 18 +++++++++++++++++- mne/io/nsx/nsx.py | 6 ++++++ 7 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 doc/changes/dev/14246.newfeature.rst diff --git a/doc/changes/dev/14246.newfeature.rst b/doc/changes/dev/14246.newfeature.rst new file mode 100644 index 00000000000..f08e4464881 --- /dev/null +++ b/doc/changes/dev/14246.newfeature.rst @@ -0,0 +1 @@ +Speed up reading large Artemis123, Curry, EEGLAB, Eximia, FIL and NSx files by reading them in cache-sized blocks, by `Bruno Aristimunha`_. diff --git a/mne/io/artemis123/artemis123.py b/mne/io/artemis123/artemis123.py index 54d83b4296c..a7ca0378675 100644 --- a/mne/io/artemis123/artemis123.py +++ b/mne/io/artemis123/artemis123.py @@ -19,6 +19,10 @@ from ..base import BaseRaw from .utils import _load_mne_locs, _read_pos +# read in cache-sized blocks rather than one huge one (1.5x on a 176 MB file); see +# _read_segments_file() for why a smaller block is faster +_BLOCK_BYTES = 16 * 1024**2 + @verbose def read_raw_artemis123( @@ -536,4 +540,15 @@ def __init__( def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): """Read a chunk of raw data.""" - _read_segments_file(self, data, idx, fi, start, stop, cals, mult, dtype=">f4") + _read_segments_file( + self, + data, + idx, + fi, + start, + stop, + cals, + mult, + dtype=">f4", + max_block_bytes=_BLOCK_BYTES, + ) diff --git a/mne/io/curry/curry.py b/mne/io/curry/curry.py index c39471694db..8f62050f2e3 100644 --- a/mne/io/curry/curry.py +++ b/mne/io/curry/curry.py @@ -209,6 +209,11 @@ def _get_curry_recording_type(fname): return "evoked" +# read in cache-sized blocks rather than one huge one (1.5x on a 107 MB file); see +# _read_segments_file() for why a smaller block is faster +_BLOCK_BYTES = 16 * 1024**2 + + def _get_curry_epoch_info(fname): _soft_import("curryreader", "read epoch info") _soft_import("pandas", "dataframe integration") @@ -865,7 +870,16 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult): else: _read_segments_file( - self, data, idx, fi, start, stop, cals, mult, dtype="