diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae12c331..e0c76bbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: branches: [main] pull_request: branches: [main] + schedule: + # Keep the trusted documentation dataset cache warm. + - cron: "17 5 * * 1,4" workflow_dispatch: concurrency: @@ -14,6 +17,7 @@ concurrency: jobs: lint: name: Lint + if: github.event_name != 'schedule' runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -64,6 +68,8 @@ jobs: docs: name: Documentation runs-on: ubuntu-latest + env: + MNE_DATA: ${{ github.workspace }}/.mne-data permissions: contents: write steps: @@ -76,10 +82,38 @@ jobs: run: | python -m pip install --upgrade pip pip install -e .[docs] + - name: Restore documentation datasets + id: docs-data-cache + uses: actions/cache/restore@v5 + with: + path: ${{ env.MNE_DATA }} + key: docs-data-${{ runner.os }}-${{ hashFiles('scripts/prefetch_docs_data.py', 'pyproject.toml') }} + restore-keys: | + docs-data-${{ runner.os }}- + - name: Prefetch and verify documentation datasets + run: python scripts/prefetch_docs_data.py --data-dir "$MNE_DATA" + - name: Save trusted documentation dataset cache + if: >- + steps.docs-data-cache.outputs.cache-hit != 'true' && + (github.event_name == 'push' || + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch') + uses: actions/cache/save@v5 + with: + path: ${{ env.MNE_DATA }} + key: docs-data-${{ runner.os }}-${{ hashFiles('scripts/prefetch_docs_data.py', 'pyproject.toml') }} - name: Build documentation run: make -C docs html SPHINXOPTS="-W --keep-going" - name: Add .nojekyll to the build run: touch docs/_build/html/.nojekyll + - name: Verify the build produced a complete site + run: | + set -e + test -s docs/_build/html/index.html + test -s docs/_build/html/auto_examples/index.html + test -s docs/_build/html/auto_examples/zapline/plot_02_parameter_tuning.html + test -s docs/_build/html/auto_examples/zapline/plot_03_epoched_data.html + echo "$(find docs/_build/html -name '*.html' | wc -l) HTML pages built" - name: Upload documentation artifact uses: actions/upload-artifact@v6 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ecc2344..a2fa69a0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -276,6 +276,9 @@ Documentation is built with Sphinx and hosted on GitHub Pages. # Build HTML documentation make -C docs html +# Populate the real-data cache before a full gallery build +python scripts/prefetch_docs_data.py + # View in browser open docs/_build/html/index.html # macOS xdg-open docs/_build/html/index.html # Linux @@ -289,6 +292,12 @@ start docs/_build/html/index.html # Windows - `docs/dss.md` - DSS module guide - `examples/` - Gallery examples (rendered by sphinx-gallery) +Documentation CI executes the complete gallery. Its MNE Sample, Somato, and +EEGBCI downloads are prefetched and cached using an inventory-derived key; a +twice-weekly trusted build keeps that cache available to pull requests. When +adding a new real dataset to an example, also add it to +`scripts/prefetch_docs_data.py` so failures happen before Sphinx starts. + ### Adding Examples Examples are Python scripts in the `examples/` directory: diff --git a/docs/changes/devel/72.bugfix.rst b/docs/changes/devel/72.bugfix.rst new file mode 100644 index 00000000..c77a9e1a --- /dev/null +++ b/docs/changes/devel/72.bugfix.rst @@ -0,0 +1,4 @@ +Made documentation builds resilient without dropping real-data gallery output: +CI now prefetches and caches the complete MNE dataset inventory, ZapLine uses a +maintained MNE recording instead of the unavailable NoiseTools host, and an +incomplete build cannot replace the published site. diff --git a/docs/conf.py b/docs/conf.py index 74205ae3..2439c86f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -65,6 +65,9 @@ "../examples/zapline", "../examples/asr", "../examples/spectrum_interpolation", + # Keep new example sections buildable until they receive an + # intentional position above this fallback. + "*", ] ), "within_subsection_order": FileNameSortKey, diff --git a/examples/zapline/plot_02_parameter_tuning.py b/examples/zapline/plot_02_parameter_tuning.py index d68b681a..58065cd8 100644 --- a/examples/zapline/plot_02_parameter_tuning.py +++ b/examples/zapline/plot_02_parameter_tuning.py @@ -3,8 +3,8 @@ ========================================= This example shows how the main ZapLine tuning parameters change the cleaning -behavior on synthetic data, then applies the same workflow to a real NoiseTools -MEG recording. +behavior on synthetic data, then applies the same workflow to the real MNE +Sample MEG recording. Authors: Sina Esmaeili (sina.esmaeili@umontreal.ca) Hamza Abdelhedi (hamza.abdelhedi@umontreal.ca) @@ -13,14 +13,10 @@ # %% # Imports # ------- -from pathlib import Path -from urllib.request import urlretrieve - import matplotlib.pyplot as plt import mne import numpy as np from scipy import signal -from scipy.io import loadmat from mne_denoise.viz import ( plot_component_cleaning_summary, @@ -30,40 +26,6 @@ ) from mne_denoise.zapline import ZapLine -NOISETOOLS_BASE_URL = "http://audition.ens.fr/adc/NoiseTools/DATA" - - -def _find_repo_root(): - """Return the repository root for this example.""" - starts = [] - if "__file__" in globals(): - starts.append(Path(__file__).resolve()) - starts.append(Path.cwd().resolve()) - - for start in starts: - current = start if start.is_dir() else start.parent - for candidate in (current, *current.parents): - mne_ok = (candidate / "mne_denoise").exists() - ex_ok = (candidate / "examples").exists() - if mne_ok and ex_ok: - return candidate - - raise FileNotFoundError("Could not locate the repository root.") - - -def _load_or_fetch_data_file(name): - """Return one ZapLine example data file, downloading it if needed.""" - path = _find_repo_root() / "examples" / "zapline" / "data" / name - if path.exists(): - return path - - path.parent.mkdir(parents=True, exist_ok=True) - url = f"{NOISETOOLS_BASE_URL}/{name}" - print(f"Downloading {name} to {path}...") - urlretrieve(url, str(path)) - return path - - # %% # Part 1: n_remove Parameter # -------------------------- @@ -243,24 +205,28 @@ def _load_or_fetch_data_file(name): plt.show() # %% -# Part 4: Real MEG Data (NoiseTools) +# Part 4: Real MEG Data (MNE Sample) # ---------------------------------- -# Apply ZapLine to real MEG data from NoiseTools dataset. -# The file is cached under ``examples/zapline/data`` after the first run. +# Apply ZapLine to a real MEG recording managed by MNE's dataset fetcher. MNE +# verifies the dataset archive and reuses its local copy across examples. print("\nPart 4: Real MEG Data") -# Load data1.mat (MEG with large near-DC fluctuations) -data1_path = _load_or_fetch_data_file("data1.mat") -mat = loadmat(str(data1_path)) -meg_data = mat["data"].T # Transpose to (channels, times) -sfreq_meg = float(mat["sr"].flatten()[0]) +# Load 30 seconds of gradiometer data and resample to keep the gallery build +# compact. The MNE Sample recording was acquired in a 60 Hz mains environment. +sample_path = mne.datasets.sample.data_path() +raw_path = sample_path / "MEG" / "sample" / "sample_audvis_raw.fif" +raw_meg = mne.io.read_raw_fif(raw_path, preload=False, verbose="ERROR") +grad_picks = mne.pick_types(raw_meg.info, meg="grad", exclude="bads") +raw_meg.pick(grad_picks).crop(0, 30).load_data().resample(300, verbose="ERROR") +meg_data = raw_meg.get_data() * 1e12 # Constant scale for readable values +sfreq_meg = raw_meg.info["sfreq"] # Demean meg_data = meg_data - np.mean(meg_data, axis=1, keepdims=True) -print(f"Loaded data1.mat: {meg_data.shape}, sfreq={sfreq_meg} Hz") -print("MEG data with large near-DC fluctuations") +print(f"Loaded MNE Sample data: {meg_data.shape}, sfreq={sfreq_meg} Hz") +print("Real gradiometer data with 60 Hz line interference") # %% # Apply ZapLine to MEG Data diff --git a/examples/zapline/plot_03_epoched_data.py b/examples/zapline/plot_03_epoched_data.py index b2cfc8cc..cb3ddfc3 100644 --- a/examples/zapline/plot_03_epoched_data.py +++ b/examples/zapline/plot_03_epoched_data.py @@ -3,8 +3,7 @@ ============================================== This example shows how ZapLine can be applied when the data are naturally -epoched, then extends the same workflow to larger real MEG arrays from the -NoiseTools examples. +epoched, then extends the same workflow to real MNE Sample MEG data. Authors: Sina Esmaeili (sina.esmaeili@umontreal.ca) Hamza Abdelhedi (hamza.abdelhedi@umontreal.ca) @@ -13,12 +12,9 @@ # %% # Imports # ------- -from pathlib import Path -from urllib.request import urlretrieve - +import mne import numpy as np from scipy import signal -from scipy.io import loadmat from mne_denoise.viz import ( plot_component_cleaning_summary, @@ -26,40 +22,6 @@ ) from mne_denoise.zapline import ZapLine -NOISETOOLS_BASE_URL = "http://audition.ens.fr/adc/NoiseTools/DATA" - - -def _find_repo_root(): - """Return the repository root for this example.""" - starts = [] - if "__file__" in globals(): - starts.append(Path(__file__).resolve()) - starts.append(Path.cwd().resolve()) - - for start in starts: - current = start if start.is_dir() else start.parent - for candidate in (current, *current.parents): - mne_ok = (candidate / "mne_denoise").exists() - ex_ok = (candidate / "examples").exists() - if mne_ok and ex_ok: - return candidate - - raise FileNotFoundError("Could not locate the repository root.") - - -def _load_or_fetch_data_file(name): - """Return one ZapLine example data file, downloading it if needed.""" - path = _find_repo_root() / "examples" / "zapline" / "data" / name - if path.exists(): - return path - - path.parent.mkdir(parents=True, exist_ok=True) - url = f"{NOISETOOLS_BASE_URL}/{name}" - print(f"Downloading {name} to {path}...") - urlretrieve(url, str(path)) - return path - - # %% # Part 1: Synthetic Epoched Data # ------------------------------ @@ -113,7 +75,7 @@ def _load_or_fetch_data_file(name): print("\nApplying ZapLine to epoched data...") # Concatenate epochs for ZapLine -data_concat = epochs_data.reshape(n_channels, -1) # (channels, epochs*times) +data_concat = epochs_data.transpose(1, 0, 2).reshape(n_channels, -1) print(f"Concatenated shape: {data_concat.shape}") # Apply ZapLine @@ -122,7 +84,7 @@ def _load_or_fetch_data_file(name): cleaned = est.transform(data_concat) # Reshape back to epochs -cleaned_epochs = cleaned.reshape(n_epochs, n_channels, n_times) +cleaned_epochs = cleaned.reshape(n_channels, n_epochs, n_times).transpose(1, 0, 2) print(f"Cleaned epochs shape: {cleaned_epochs.shape}") # %% @@ -152,42 +114,40 @@ def _load_or_fetch_data_file(name): ) # %% -# Part 2: Real MEG Epoched Data (NoiseTools data3.mat) -# ---------------------------------------------------- -# MEG epoched data from NoiseTools. -# Shape: (900 times, 151 channels, 30 epochs), sr=300 Hz +# Part 2: Real MEG Epoched Data (MNE Sample) +# ------------------------------------------ +# Split a continuous real recording into fixed-length epochs, then concatenate +# those epochs explicitly for fitting. print("\nPart 2: Real MEG Epoched Data") -# Load data3.mat (MEG epoched) -data3_path = _load_or_fetch_data_file("data3.mat") -mat = loadmat(str(data3_path)) -meg_epochs = mat["data"] # (times, channels, epochs) = (900, 151, 30) -sfreq_meg = float(mat["sr"].flatten()[0]) - -# Use first 10 epochs as in MATLAB example -meg_epochs = meg_epochs[:, :, :10] # (900, 151, 10) - -# Transpose to (channels, times*epochs) for ZapLine -n_times_meg, n_ch_meg, n_ep_meg = meg_epochs.shape -meg_concat = meg_epochs.transpose(1, 0, 2).reshape(n_ch_meg, -1) # (151, 9000) - -# Demean -meg_concat = meg_concat - np.mean(meg_concat, axis=1, keepdims=True) - -# Scale to reasonable units (MEG data is in Tesla, very small values) -scale_factor = 1e12 # Convert to pT -meg_concat = meg_concat * scale_factor - -print(f"Loaded data3.mat: {n_ep_meg} epochs, {n_ch_meg} channels, {n_times_meg} times") +sample_path = mne.datasets.sample.data_path() +raw_path = sample_path / "MEG" / "sample" / "sample_audvis_raw.fif" +raw_sample = mne.io.read_raw_fif(raw_path, preload=False, verbose="ERROR") +raw_sample.crop(0, 35).load_data().resample(300, verbose="ERROR") +grad_picks = mne.pick_types(raw_sample.info, meg="grad", exclude="bads") +raw_grad = raw_sample.copy().pick(grad_picks) +epochs_meg = mne.make_fixed_length_epochs( + raw_grad, duration=3.0, preload=True, verbose="ERROR" +) +meg_epochs = epochs_meg.get_data(copy=False)[:10] * 1e12 +sfreq_meg = epochs_meg.info["sfreq"] +n_ep_meg, n_ch_meg, n_times_meg = meg_epochs.shape +meg_concat = meg_epochs.transpose(1, 0, 2).reshape(n_ch_meg, -1) +meg_concat -= np.mean(meg_concat, axis=1, keepdims=True) + +print( + f"Loaded MNE Sample epochs: {n_ep_meg} epochs, {n_ch_meg} channels, " + f"{n_times_meg} times" +) print(f"Concatenated shape: {meg_concat.shape}") print(f"Sampling rate: {sfreq_meg} Hz") -# Apply ZapLine (50 Hz) +# Apply ZapLine (60 Hz) est_meg = ZapLine( - line_freq=50, + line_freq=60, sfreq=sfreq_meg, - n_select=2, # As in MATLAB example + n_select=2, ) est_meg.fit(meg_concat) cleaned_meg = est_meg.transform(meg_concat) @@ -201,7 +161,7 @@ def _load_or_fetch_data_file(name): meg_concat, cleaned_meg, sfreq=sfreq_meg, - line_freq=50, + line_freq=60, fmax=150, show=True, ) @@ -210,42 +170,32 @@ def _load_or_fetch_data_file(name): nperseg = min(meg_concat.shape[1], int(sfreq_meg * 2)) freqs, psd_orig = signal.welch(meg_concat, sfreq_meg, nperseg=nperseg) _, psd_clean = signal.welch(cleaned_meg, sfreq_meg, nperseg=nperseg) -idx_50 = np.argmin(np.abs(freqs - 50)) -ratio = np.mean(psd_orig[:, idx_50]) / np.mean(psd_clean[:, idx_50]) +idx_60 = np.argmin(np.abs(freqs - 60)) +ratio = np.mean(psd_orig[:, idx_60]) / np.mean(psd_clean[:, idx_60]) reduction_db = 10 * np.log10(ratio) -print(f"50 Hz power reduction: {reduction_db:.1f} dB") +print(f"60 Hz power reduction: {reduction_db:.1f} dB") # %% -# Part 3: High-Channel MEG Data (NoiseTools example_data.mat) -# ----------------------------------------------------------- -# MEG data with many channels (275), demonstrating nkeep parameter. -# Shape: (3000 times, 275 channels, 30 epochs), sr=600 Hz +# Part 3: High-Channel MEG Data +# ----------------------------- +# Use all good MNE Sample gradiometers to demonstrate ``nkeep`` on a real +# high-dimensional recording without mixing channel units. print("\nPart 3: High-Channel MEG Data") -example_data_path = _load_or_fetch_data_file("example_data.mat") -mat = loadmat(str(example_data_path)) -meg_high = mat["meg"] # (times, channels, epochs) = (3000, 275, 30) -sfreq_high = float(mat["sr"].flatten()[0]) - -# Use first epoch as in MATLAB example -meg_high = meg_high[:, :, 0].T # (275, 3000) - -# Demean -meg_high = meg_high - np.mean(meg_high, axis=1, keepdims=True) +raw_high = raw_sample.copy().pick(grad_picks).crop(0, 10) +meg_high = raw_high.get_data() * 1e12 +meg_high -= np.mean(meg_high, axis=1, keepdims=True) +sfreq_high = raw_high.info["sfreq"] -# Scale to reasonable units (MEG data is in Tesla, very small values) -scale_factor = 1e12 # Convert to pT -meg_high = meg_high * scale_factor - -print(f"Loaded example_data.mat: {meg_high.shape}") +print(f"Loaded high-channel MNE Sample data: {meg_high.shape}") print(f"Sampling rate: {sfreq_high} Hz") # Apply ZapLine with nkeep est_high = ZapLine( - line_freq=50, + line_freq=60, sfreq=sfreq_high, - n_select=6, # As in MATLAB example + n_select=6, nkeep=50, # Reduce dimensionality ) est_high.fit(meg_high) @@ -258,7 +208,7 @@ def _load_or_fetch_data_file(name): meg_high, cleaned_high, sfreq=sfreq_high, - line_freq=50, + line_freq=60, fmax=150, show=True, ) @@ -286,10 +236,10 @@ def _load_or_fetch_data_file(name): nperseg = min(meg_high.shape[1], int(sfreq_high * 2)) freqs, psd_orig = signal.welch(meg_high, sfreq_high, nperseg=nperseg) _, psd_clean = signal.welch(cleaned_high, sfreq_high, nperseg=nperseg) -idx_50 = np.argmin(np.abs(freqs - 50)) -ratio = np.mean(psd_orig[:, idx_50]) / np.mean(psd_clean[:, idx_50]) +idx_60 = np.argmin(np.abs(freqs - 60)) +ratio = np.mean(psd_orig[:, idx_60]) / np.mean(psd_clean[:, idx_60]) reduction_db = 10 * np.log10(ratio) -print(f"50 Hz power reduction: {reduction_db:.1f} dB") +print(f"60 Hz power reduction: {reduction_db:.1f} dB") # %% # Conclusion diff --git a/scripts/prefetch_docs_data.py b/scripts/prefetch_docs_data.py new file mode 100644 index 00000000..e99d5a66 --- /dev/null +++ b/scripts/prefetch_docs_data.py @@ -0,0 +1,72 @@ +"""Download and verify every dataset required by the documentation gallery.""" + +from __future__ import annotations + +import argparse +import os +import time +from collections.abc import Callable +from pathlib import Path + +from mne.datasets import eegbci, sample, somato + + +def _fetch_with_retries( + name: str, fetch: Callable[[], object], *, attempts: int = 3 +) -> object: + """Run one MNE dataset fetch with bounded retries.""" + delays = (5, 15) + for attempt in range(1, attempts + 1): + try: + result = fetch() + except Exception as error: + if attempt == attempts: + raise RuntimeError( + f"Could not prepare the {name} documentation dataset after " + f"{attempts} attempts." + ) from error + delay = delays[attempt - 1] + print(f"{name} fetch attempt {attempt} failed; retrying in {delay}s") + time.sleep(delay) + else: + print(f"Prepared {name}: {result}") + return result + raise AssertionError("unreachable") + + +def prefetch_docs_data(data_dir: Path) -> None: + """Populate ``data_dir`` with all real datasets used by gallery examples.""" + data_dir.mkdir(parents=True, exist_ok=True) + _fetch_with_retries( + "MNE Sample", + lambda: sample.data_path(path=data_dir, update_path=False, verbose=True), + ) + _fetch_with_retries( + "MNE Somato", + lambda: somato.data_path(path=data_dir, update_path=False, verbose=True), + ) + _fetch_with_retries( + "EEGBCI subject 1 run 1", + lambda: eegbci.load_data( + subjects=[1], + runs=[1], + path=data_dir, + update_path=False, + verbose=True, + ), + ) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--data-dir", + type=Path, + default=Path(os.environ.get("MNE_DATA", "~/mne_data")).expanduser(), + help="Shared MNE dataset directory (default: MNE_DATA or ~/mne_data).", + ) + return parser.parse_args() + + +if __name__ == "__main__": + prefetch_docs_data(_parse_args().data_dir.resolve())