Read Artemis123, Curry, EEGLAB, Eximia, FIL and NSx in cache-sized blocks (#14246)

Author: bruAristimunha

doc/changes/dev/14246.newfeature.rst

@@ -0,0 +1 @@
+Speed up reading large Artemis123, Curry, EEGLAB, Eximia, FIL and NSx files by reading them in cache-sized blocks, by `Bruno Aristimunha`_.

mne/io/artemis123/artemis123.py

@@ -19,6 +19,10 @@
 from ..base import BaseRaw
 from .utils import _load_mne_locs, _read_pos
 
+# read in cache-sized blocks rather than one huge one (1.5x on a 176 MB file); see
+# _read_segments_file() for why a smaller block is faster
+_BLOCK_BYTES = 16 * 1024**2
+
 
 @verbose
 def read_raw_artemis123(
@@ -536,4 +540,15 @@ def __init__(
 
     def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
         """Read a chunk of raw data."""
-        _read_segments_file(self, data, idx, fi, start, stop, cals, mult, dtype=">f4")
+        _read_segments_file(
+            self,
+            data,
+            idx,
+            fi,
+            start,
+            stop,
+            cals,
+            mult,
+            dtype=">f4",
+            max_block_bytes=_BLOCK_BYTES,
+        )

mne/io/curry/curry.py

@@ -209,6 +209,11 @@ def _get_curry_recording_type(fname):
             return "evoked"
 
 
+# read in cache-sized blocks rather than one huge one (1.5x on a 107 MB file); see
+# _read_segments_file() for why a smaller block is faster
+_BLOCK_BYTES = 16 * 1024**2
+
+
 def _get_curry_epoch_info(fname):
     _soft_import("curryreader", "read epoch info")
     _soft_import("pandas", "dataframe integration")
@@ -865,7 +870,16 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
 
         else:
             _read_segments_file(
-                self, data, idx, fi, start, stop, cals, mult, dtype="<f4"
+                self,
+                data,
+                idx,
+                fi,
+                start,
+                stop,
+                cals,
+                mult,
+                dtype="<f4",
+                max_block_bytes=_BLOCK_BYTES,
             )
 
 

mne/io/eeglab/eeglab.py

@@ -216,6 +216,11 @@ def _get_montage_information(eeg, get_pos, *, montage_units):
     return ch_names, ch_types, montage
 
 
+# read in cache-sized blocks rather than one huge one (2.1x on a 102 MB file); see
+# _read_segments_file() for why a smaller block is faster
+_BLOCK_BYTES = 4 * 1024**2
+
+
 def _get_info(eeg, *, eog, montage_units):
     """Get measurement info."""
     # add the ch_names and info['chs'][idx]['loc']
@@ -546,7 +551,18 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
             return
 
         # Fall back to reading from file (separate .fdt file)
-        _read_segments_file(self, data, idx, fi, start, stop, cals, mult, dtype="<f4")
+        _read_segments_file(
+            self,
+            data,
+            idx,
+            fi,
+            start,
+            stop,
+            cals,
+            mult,
+            dtype="<f4",
+            max_block_bytes=_BLOCK_BYTES,
+        )
 
 
 class EpochsEEGLAB(BaseEpochs):

mne/io/eximia/eximia.py

@@ -10,6 +10,10 @@
 from ...utils import _check_fname, fill_doc, logger, verbose, warn
 from ..base import BaseRaw
 
+# read in cache-sized blocks rather than one huge one (2.4x on a 102 MB file); see
+# _read_segments_file() for why a smaller block is faster
+_BLOCK_BYTES = 2 * 1024**2
+
 
 @fill_doc
 def read_raw_eximia(
@@ -105,4 +109,15 @@ def __init__(self, fname, preload=False, verbose=None):
 
     def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
         """Read a chunk of raw data."""
-        _read_segments_file(self, data, idx, fi, start, stop, cals, mult, dtype="<i2")
+        _read_segments_file(
+            self,
+            data,
+            idx,
+            fi,
+            start,
+            stop,
+            cals,
+            mult,
+            dtype="<i2",
+            max_block_bytes=_BLOCK_BYTES,
+        )

mne/io/fil/fil.py

@@ -184,10 +184,26 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
         """Read a chunk of raw data."""
         si = self._raw_extras[fi]
         _read_segments_file(
-            self, data, idx, fi, start, stop, cals, mult, dtype=si["dt"]
+            self,
+            data,
+            idx,
+            fi,
+            start,
+            stop,
+            cals,
+            mult,
+            dtype=si["dt"],
+            max_block_bytes=_BLOCK_BYTES,
         )
 
 
+# read in cache-sized blocks rather than one huge one (1.9x on a 197 MB file); see
+# _read_segments_file() for why a smaller block is faster. 4 MiB is faster still
+# (2.5x) but measurably regresses the 9.8 MB shipped fixture, which fits in one
+# 16 MiB block and so keeps its current code path exactly.
+_BLOCK_BYTES = 16 * 1024**2
+
+
 def _convert_channel_info(chans):
     """Convert the imported _channels.tsv into the chs element of raw.info."""
     nmeg = nstim = nmisc = nref = 0

mne/io/nsx/nsx.py

@@ -146,6 +146,11 @@ def read_raw_nsx(
     )
 
 
+# read in cache-sized blocks rather than one huge one (2.3x on a 102 MB file); see
+# _read_segments_file() for why a smaller block is faster
+_BLOCK_BYTES = 4 * 1024**2
+
+
 @fill_doc
 class RawNSX(BaseRaw):
     """Raw object from NSx file from Blackrock Microsystems.
@@ -256,6 +261,7 @@ def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
                 n_channels=None,
                 offset=offset,
                 trigger_ch=None,
+                max_block_bytes=_BLOCK_BYTES,
             )