diff --git a/.circleci/config.yml b/.circleci/config.yml index e4eb51ddfa0..24cfbae59df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -100,6 +100,14 @@ jobs: name: Set BASH_ENV command: ./tools/circleci_bash_env.sh + # Writes testing_version.txt and misc_version.txt, used in the cache keys of + # the two datasets whose versions actually change on a regular basis. Caches + # are immutable on CircleCI, so a dataset with a fixed key would be pinned to + # whatever was first cached and re-downloaded in full on every later run. + - run: + name: Get dataset versions + command: ./tools/get_testing_version.sh + - run: name: Install fonts needed for diagrams command: | @@ -110,13 +118,15 @@ jobs: curl https://codeload.github.com/adobe-fonts/source-sans/tar.gz/3.028R | tar xz -C $HOME/.fonts fc-cache -f - # Load uv cache - - restore_cache: - keys: - - uv-cache-0 + # Load uv cache. Caches are immutable on CircleCI, so a fixed key would freeze + # the cache at whatever the dependencies looked like the first time it was + # written; keying on pyproject.toml refreshes it when the dependencies change, + # and the prefix-only fallback still gets a warm (if slightly stale) cache in + # between. - restore_cache: keys: - - user-install-bin-cache-311 + - uv-cache-1-{{ checksum "pyproject.toml" }} + - uv-cache-1- # Hack in uninstalls of libraries as necessary if pip doesn't do the right thing in upgrading for us... - run: @@ -125,14 +135,9 @@ jobs: ./tools/circleci_dependencies.sh - save_cache: - key: uv-cache-0 + key: uv-cache-1-{{ checksum "pyproject.toml" }} paths: - ~/.cache/uv - - save_cache: - key: user-install-bin-cache-311 - paths: - - ~/.local/lib/python3.11/site-packages - - ~/.local/bin - run: name: Check Qt @@ -142,7 +147,7 @@ jobs: # Load tiny cache so that ~/.mne does not need to be created below - restore_cache: keys: - - data-cache-tiny-0 + - data-cache-tiny-0-{{ checksum "misc_version.txt" }} # Look at what we have and fail early if there is some library conflict - run: @@ -211,7 +216,7 @@ jobs: - data-cache-spm-face - restore_cache: keys: - - data-cache-testing + - data-cache-testing-{{ checksum "testing_version.txt" }} - restore_cache: keys: - data-cache-visual-1 @@ -239,87 +244,12 @@ jobs: echo "BUILD=$(cat build.txt)" ls -al ~/mne_data; - # Run doctest (if it's full or front) before building the docs - - run: - name: make test-doc - command: | - if [[ $(cat gitlog.txt) == *"[circle front]"* ]] || [[ $(cat build.txt) == "html-memory" ]] ; then - make test-doc; - mkdir -p doc/_build/test-results/test-doc; - cp junit-results.xml doc/_build/test-results/test-doc/junit.xml; - cp coverage.xml doc/_build/test-results/test-doc/coverage.xml; - fi; - # Build docs - - run: - name: make html - command: | # we have -o pipefail in #BASH_ENV so we should be okay - set -x - PATTERN=$(cat pattern.txt) make -C doc $(cat build.txt) 2>&1 | tee sphinx_log.txt - - run: - name: Check sphinx log for warnings (which are treated as errors) - when: always - command: | - ! grep "^.*\(WARNING\|ERROR\): " sphinx_log.txt - - run: - name: Show profiling output - when: always - command: | - if compgen -G "doc/*.dat" > /dev/null; then - mkdir -p doc/generated - mprof plot doc/*.dat --output doc/generated/memory.png - else - echo "No profile data found in doc/" - fi - - run: - name: Sanity check system state - command: | - python -c "import mne; level = mne.get_config('MNE_LOGGING_LEVEL'); assert level.lower() == 'info', repr(level)" - - # Reduce upload time of artifacts we will (almost) never look at - - run: - name: Reduce artifact upload time - command: | - if grep -q html-pattern-memory build.txt; then - zip -rm doc/_build/html/_downloads.zip doc/_build/html/_downloads - fi - for NAME in generated auto_tutorials auto_examples; do - zip -rm doc/${NAME}.zip doc/${NAME} - done - - # Save the JUnit file - - store_test_results: - path: doc/_build/test-results - - store_artifacts: - path: doc/_build/test-results - destination: test-results - # Upload test results to Codecov - - run: - name: Upload test results to Codecov - environment: - CODECOV_TOKEN: fb4c4a94-72d7-4743-bb08-af25b623a29a - command: | - if [[ -f doc/_build/test-results/test-doc/coverage.xml ]]; then - bash <(curl -s https://codecov.io/bash) -f doc/_build/test-results/test-doc/coverage.xml || true - fi - # Save the SG RST - - store_artifacts: - path: doc/auto_examples.zip - - store_artifacts: - path: doc/auto_tutorials.zip - - store_artifacts: - path: doc/generated.zip - # Save the HTML - - store_artifacts: - path: doc/_build/html/ - destination: html - - persist_to_workspace: - root: doc/_build - paths: - - html - - # Keep these separate, maybe better in terms of size limitations (?) + # Saved here rather than at the end of the job: everything above has + # finished downloading, and a failure in the (long, flaky) doc build below + # should not throw the downloads away. Kept as separate caches, maybe + # better in terms of size limitations (?) - save_cache: - key: data-cache-tiny-0 # < 100 M, might as well combine + key: data-cache-tiny-0-{{ checksum "misc_version.txt" }} # < 100 M, might as well combine paths: - ~/.mne - ~/mne_data/MNE-kiloword-data # (28 M) @@ -394,7 +324,7 @@ jobs: paths: - ~/mne_data/MNE-spm-face # (1.5 G) - save_cache: - key: data-cache-testing + key: data-cache-testing-{{ checksum "testing_version.txt" }} paths: - ~/mne_data/MNE-testing-data # (2.5 G) - save_cache: @@ -414,6 +344,84 @@ jobs: paths: - ~/mne_data/ds004388 # (1.8 G) + # Run doctest (if it's full or front) before building the docs + - run: + name: make test-doc + command: | + if [[ $(cat gitlog.txt) == *"[circle front]"* ]] || [[ $(cat build.txt) == "html-memory" ]] ; then + make test-doc; + mkdir -p doc/_build/test-results/test-doc; + cp junit-results.xml doc/_build/test-results/test-doc/junit.xml; + cp coverage.xml doc/_build/test-results/test-doc/coverage.xml; + fi; + # Build docs + - run: + name: make html + command: | # we have -o pipefail in #BASH_ENV so we should be okay + set -x + PATTERN=$(cat pattern.txt) make -C doc $(cat build.txt) 2>&1 | tee sphinx_log.txt + - run: + name: Check sphinx log for warnings (which are treated as errors) + when: always + command: | + ! grep "^.*\(WARNING\|ERROR\): " sphinx_log.txt + - run: + name: Show profiling output + when: always + command: | + if compgen -G "doc/*.dat" > /dev/null; then + mkdir -p doc/generated + mprof plot doc/*.dat --output doc/generated/memory.png + else + echo "No profile data found in doc/" + fi + - run: + name: Sanity check system state + command: | + python -c "import mne; level = mne.get_config('MNE_LOGGING_LEVEL'); assert level.lower() == 'info', repr(level)" + + # Reduce upload time of artifacts we will (almost) never look at + - run: + name: Reduce artifact upload time + command: | + if grep -q html-pattern-memory build.txt; then + zip -rm doc/_build/html/_downloads.zip doc/_build/html/_downloads + fi + for NAME in generated auto_tutorials auto_examples; do + zip -rm doc/${NAME}.zip doc/${NAME} + done + + # Save the JUnit file + - store_test_results: + path: doc/_build/test-results + - store_artifacts: + path: doc/_build/test-results + destination: test-results + # Upload test results to Codecov + - run: + name: Upload test results to Codecov + environment: + CODECOV_TOKEN: fb4c4a94-72d7-4743-bb08-af25b623a29a + command: | + if [[ -f doc/_build/test-results/test-doc/coverage.xml ]]; then + bash <(curl -s https://codecov.io/bash) -f doc/_build/test-results/test-doc/coverage.xml || true + fi + # Save the SG RST + - store_artifacts: + path: doc/auto_examples.zip + - store_artifacts: + path: doc/auto_tutorials.zip + - store_artifacts: + path: doc/generated.zip + # Save the HTML + - store_artifacts: + path: doc/_build/html/ + destination: html + - persist_to_workspace: + root: doc/_build + paths: + - html + linkcheck: # there are a few files excluded from this for expediency, see Makefile @@ -442,7 +450,8 @@ jobs: command: ./tools/circleci_bash_env.sh - restore_cache: keys: - - uv-cache-0 + - uv-cache-1-{{ checksum "pyproject.toml" }} + - uv-cache-1- - run: name: Get Python running command: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9cc24da0d2a..b598df23035 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -136,14 +136,12 @@ jobs: project-root: ${{ github.workspace }} groups: lockfile_extras if: matrix.kind == 'old' - # Minimal commands on Linux (macOS stalls) - - uses: actions/cache@v6.1.0 - with: - key: minimal-commands-1 - path: ~/minimal_cmds - if: startswith(matrix.os, 'ubuntu') && matrix.kind != 'minimal' && matrix.kind != 'old' && matrix.kind != 'pip-ft' - - run: bash ./tools/get_minimal_commands.sh - if: startswith(matrix.os, 'ubuntu') && matrix.kind != 'minimal' && matrix.kind != 'old' && matrix.kind != 'pip-ft' + # MNE-C minimal commands. The action caches the download and no-ops on + # platforms that have no binaries (macOS arm64, Windows, Linux arm64), so it + # only needs to be gated on the kinds that deliberately do without them. + - name: Setup minimal commands + uses: mne-tools/mne-tools/actions/setup-minimal-commands@main + if: matrix.kind != 'minimal' && matrix.kind != 'old' && matrix.kind != 'pip-ft' - run: bash ./tools/github_actions_infos.sh - name: Check Qt import uses: mne-tools/mne-tools/actions/check-qt-import@main @@ -156,7 +154,7 @@ jobs: - run: ./tools/get_testing_version.sh - uses: actions/cache@v6.1.0 with: - key: ${{ env.TESTING_VERSION }} + key: mne_data-testing-${{ env.TESTING_VERSION }}-misc-${{ env.MISC_VERSION }} path: ~/mne_data - run: bash ./tools/github_actions_download.sh - run: bash ./tools/github_actions_test.sh # for some reason on macOS we need to run "bash X" in order for a failed test run to show up diff --git a/.gitignore b/.gitignore index e967f8017ab..3219187e890 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,11 @@ venv/ .ipynb_checkpoints/ /.claude/ uv.lock + +# written into the repo root by the CI helper scripts in tools/ +/build.txt +/gitlog.txt +/merge.txt +/misc_version.txt +/pattern.txt +/testing_version.txt diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 84b9b0c731e..2e365dd744f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -59,7 +59,7 @@ stages: key: minimal-cmds-1 path: /home/vsts/minimal_cmds displayName: Cache minimal commands - - bash: ./tools/get_minimal_commands.sh + - bash: $(MNE_TOOLS_DIR)/tools/get_minimal_commands.sh displayName: Get minimal commands - task: UsePythonVersion@0 inputs: @@ -82,7 +82,7 @@ stages: displayName: Get testing version - task: Cache@2 inputs: - key: $(testing_version) + key: mne_data-testing-$(testing_version)-misc-$(misc_version) path: /home/vsts/mne_data displayName: Cache testing data - bash: ./tools/github_actions_download.sh @@ -133,7 +133,7 @@ stages: displayName: Get testing version - task: Cache@2 inputs: - key: $(testing_version) + key: mne_data-testing-$(testing_version)-misc-$(misc_version) path: /home/vsts/mne_data displayName: Cache testing data - bash: ./tools/github_actions_download.sh @@ -211,7 +211,7 @@ stages: displayName: Get testing version - task: Cache@2 inputs: - key: $(testing_version) + key: mne_data-testing-$(testing_version)-misc-$(misc_version) path: C:\Users\VssAdministrator\mne_data displayName: Cache testing data - bash: ./tools/github_actions_download.sh diff --git a/mne/conftest.py b/mne/conftest.py index da87c72468f..f03109351ee 100644 --- a/mne/conftest.py +++ b/mne/conftest.py @@ -95,6 +95,8 @@ def pytest_configure(config: pytest.Config): "slowtest: mark a test as slow", "ultraslowtest: mark a test as ultraslow or to be run rarely", "pgtest: mark a test as relevant for mne-qt-browser", + "mne_c: mark a test as requiring the MNE-C command line tools", + "freesurfer: mark a test as requiring the FreeSurfer command line tools", # used by PyVista's MNE integration tests (but also useful in some testing): "pvtest: mark a test as relevant for pyvista", ): diff --git a/mne/utils/_testing.py b/mne/utils/_testing.py index b6f8f701d4a..d2c9500bf52 100644 --- a/mne/utils/_testing.py +++ b/mne/utils/_testing.py @@ -10,7 +10,7 @@ import sys import tempfile import traceback -from functools import wraps +from functools import partial, wraps from shutil import rmtree from unittest import SkipTest @@ -57,16 +57,28 @@ def __del__(self): # noqa: D105 rmtree(self._path, ignore_errors=True) +def _apply_marks(marks, func): + """Apply several pytest marks to a test function.""" + for mark in reversed(marks): + func = mark(func) + return func + + def requires_mne(func): """Decorate a function as requiring MNE.""" - return requires_mne_mark()(func) + return _apply_marks(requires_mne_mark(), func) def requires_mne_mark(): """Mark pytest tests that require MNE-C.""" import pytest - return pytest.mark.skipif(not has_mne_c(), reason="Requires MNE-C") + # The mne_c mark carries no condition; it exists so that CI can select or + # deselect the MNE-C tests as a group with `-m`. + return [ + pytest.mark.mne_c, + pytest.mark.skipif(not has_mne_c(), reason="Requires MNE-C"), + ] def requires_openmeeg_mark(): @@ -80,6 +92,9 @@ def requires_freesurfer(arg): """Require FreeSurfer.""" import pytest + # As for mne_c above, the freesurfer mark is unconditional and only exists so + # that CI can select or deselect these tests as a group with `-m`. + marks = [pytest.mark.freesurfer] reason = "Requires FreeSurfer" if isinstance(arg, str): # Calling as @requires_freesurfer('progname'): return decorator @@ -91,13 +106,13 @@ def requires_freesurfer(arg): skip = True else: skip = False - return pytest.mark.skipif(skip, reason=reason) + marks.append(pytest.mark.skipif(skip, reason=reason)) + return partial(_apply_marks, marks) else: # Calling directly as @requires_freesurfer: return decorated function # and just check env var existence - return pytest.mark.skipif(not has_freesurfer(), reason="Requires FreeSurfer")( - arg - ) + marks.append(pytest.mark.skipif(not has_freesurfer(), reason=reason)) + return _apply_marks(marks, arg) def requires_good_network(func): diff --git a/tools/get_minimal_commands.sh b/tools/get_minimal_commands.sh deleted file mode 100755 index 26dd6699e2f..00000000000 --- a/tools/get_minimal_commands.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -if [ "${DEPS}" == "minimal" ]; then - return 0 2>/dev/null || exit "0" -fi; - -pushd ~ > /dev/null -export MNE_ROOT="${PWD}/minimal_cmds" -export PATH=${MNE_ROOT}/bin:$PATH -if [ "${GITHUB_ACTIONS}" == "true" ]; then - echo "Setting MNE_ROOT for GHA" - echo "MNE_ROOT=${MNE_ROOT}" | tee -a $GITHUB_ENV; - echo "${MNE_ROOT}/bin" >> $GITHUB_PATH; -elif [ "${AZURE_CI}" == "true" ]; then - echo "Setting MNE_ROOT for Azure" - echo "##vso[task.setvariable variable=MNE_ROOT]${MNE_ROOT}" - echo "##vso[task.setvariable variable=PATH]${PATH}"; -elif [ "${CIRCLECI}" == "true" ]; then - echo "Setting MNE_ROOT for CircleCI" - echo "export MNE_ROOT=${MNE_ROOT}" >> "$BASH_ENV"; - echo "export PATH=${MNE_ROOT}/bin:\$PATH" >> "$BASH_ENV"; -fi; -if [[ "${CI_OS_NAME}" != "macos"* ]]; then - echo "Getting files for Linux..." - if [ ! -d "${PWD}/minimal_cmds" ]; then - curl -L https://osf.io/download/g7dzs?version=7 | tar xz - else - echo "Minimal commands already downloaded" - fi; - if [[ ! $(which tcsh) ]]; then - echo "tcsh not found, installing..." - sudo apt install -yq tcsh - fi - export LD_LIBRARY_PATH=${MNE_ROOT}/lib:$LD_LIBRARY_PATH - export NEUROMAG2FT_ROOT="${PWD}/minimal_cmds/bin" - export FREESURFER_HOME="${MNE_ROOT}" - if [ "${GITHUB_ACTIONS}" == "true" ]; then - echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" | tee -a "$GITHUB_ENV"; - echo "NEUROMAG2FT_ROOT=${NEUROMAG2FT_ROOT}" | tee -a "$GITHUB_ENV"; - echo "FREESURFER_HOME=${FREESURFER_HOME}" | tee -a "$GITHUB_ENV"; - fi; - if [ "${AZURE_CI}" == "true" ]; then - echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]${LD_LIBRARY_PATH}" - echo "##vso[task.setvariable variable=NEUROMAG2FT_ROOT]${NEUROMAG2FT_ROOT}" - echo "##vso[task.setvariable variable=FREESURFER_HOME]${FREESURFER_HOME}" - fi; - if [ "${CIRCLECI}" == "true" ]; then - echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> "$BASH_ENV"; - echo "export NEUROMAG2FT_ROOT=${NEUROMAG2FT_ROOT}" >> "$BASH_ENV"; - echo "export FREESURFER_HOME=${FREESURFER_HOME}" >> "$BASH_ENV"; - fi; -else - echo "Getting files for macOS Intel..." - if [ ! -d "${PWD}/minimal_cmds" ]; then - curl -L https://osf.io/download/rjcz4?version=2 | tar xz - else - echo "Minimal commands already downloaded" - fi; - export DYLD_LIBRARY_PATH=${MNE_ROOT}/lib:$DYLD_LIBRARY_PATH - if [ "${GITHUB_ACTIONS}" == "true" ]; then - echo "Setting variables for GHA" - echo "DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}" | tee -a "$GITHUB_ENV"; - set -x - wget https://github.com/XQuartz/XQuartz/releases/download/XQuartz-2.7.11/XQuartz-2.7.11.dmg - sudo hdiutil attach XQuartz-2.7.11.dmg - sudo installer -package /Volumes/XQuartz-2.7.11/XQuartz.pkg -target / - sudo ln -s /opt/X11 /usr/X11 - elif [ "${AZURE_CI}" == "true" ]; then - echo "Setting variables for Azure" - echo "##vso[task.setvariable variable=DYLD_LIBRARY_PATH]${DYLD_LIBRARY_PATH}" - elif [ "${CIRCLECI}" == "true" ]; then - echo "Setting variables for CircleCI" - echo "export DYLD_LIBRARY_PATH=${DYLD_LIBRARY_PATH}" >> "$BASH_ENV"; - fi; -fi -popd > /dev/null -echo "If one of the following fails, you should update your cache + download of minimal commands:" -set -x -which mne_process_raw -mne_process_raw --version -which mne_surf2bem -mne_surf2bem --version -which mri_average -mri_average --version -if [[ "${CI_OS_NAME}" != "macos"* ]]; then - which mkheadsurf || exit 1 - mkheadsurf --version || exit 1 -fi diff --git a/tools/get_testing_version.sh b/tools/get_testing_version.sh index aaf703dbddd..c7fbca4f5a8 100755 --- a/tools/get_testing_version.sh +++ b/tools/get_testing_version.sh @@ -2,15 +2,21 @@ set -o pipefail +# Versions of the datasets that CI downloads for the test suite, in a form usable +# in cache keys. These can be suffixed (e.g. TESTING_VERSION=${TESTING_VERSION}-1) +# to start fresh when a cache misbehaves. TESTING_VERSION=`grep -o "testing=\"[0-9.]\+\"" mne/datasets/config.py | cut -d \" -f 2 | sed "s/\./-/g"` -# This can be incremented to start fresh when the cache misbehaves, e.g.: -# TESTING_VERSION=${TESTING_VERSION}-1 +MISC_VERSION=`grep -o "misc=\"[0-9.]\+\"" mne/datasets/config.py | cut -d \" -f 2 | sed "s/\./-/g"` if [ ! -z $GITHUB_ENV ]; then echo "TESTING_VERSION="$TESTING_VERSION | tee -a $GITHUB_ENV + echo "MISC_VERSION="$MISC_VERSION | tee -a $GITHUB_ENV elif [ ! -z $AZURE_CI ]; then echo "##vso[task.setvariable variable=testing_version]$TESTING_VERSION" + echo "##vso[task.setvariable variable=misc_version]$MISC_VERSION" elif [ ! -z $CIRCLECI ]; then echo "$TESTING_VERSION" > testing_version.txt + echo "$MISC_VERSION" > misc_version.txt else echo $TESTING_VERSION + echo $MISC_VERSION fi diff --git a/tools/github_actions_download.sh b/tools/github_actions_download.sh index b67d430578e..7da8646db82 100755 --- a/tools/github_actions_download.sh +++ b/tools/github_actions_download.sh @@ -1,9 +1,17 @@ #!/bin/bash -ef +# Datasets the test suite needs. Downstream projects that reuse this script (e.g. +# MNE-BIDS) can override it to fetch only what they actually use; whatever is +# listed here must also be reflected in the actions/cache key of the caller. +MNE_CI_DATASETS="${MNE_CI_DATASETS:-testing misc}" + if [ "${MNE_CI_KIND}" != "minimal" ]; then - python -uc "import mne; mne.datasets.testing.data_path(verbose=True)" - # Make read-only to make sure we don't modify its contents - TESTING_PATH=$(python -c "import mne; print(mne.datasets.testing.data_path(verbose=False))") - echo "Testing data path: $TESTING_PATH" - chmod -R a-w "$TESTING_PATH" + # These are cached, so this only really downloads when a dataset version changes + for DATASET in ${MNE_CI_DATASETS}; do + python -uc "import mne; mne.datasets.${DATASET}.data_path(verbose=True)" + # Make read-only to make sure we don't modify its contents + DATASET_PATH=$(python -c "import mne; print(mne.datasets.${DATASET}.data_path(verbose=False))") + echo "${DATASET} data path: $DATASET_PATH" + chmod -R a-w "$DATASET_PATH" + done fi diff --git a/tools/github_actions_test.sh b/tools/github_actions_test.sh index a906960b538..94fee65b89e 100755 --- a/tools/github_actions_test.sh +++ b/tools/github_actions_test.sh @@ -16,7 +16,8 @@ elif [[ "${CI_OS_NAME}" == "macos"* ]]; then if [[ "$(uname -m)" == "arm64" ]]; then CONDITION="not (ultraslowtest or pgtest)" else - CONDITION="not (slowtest or pgtest)" + # TODO: the MNE-C tools are on PATH during setup but not once the tests run (gh-14230) + CONDITION="not (slowtest or pgtest or mne_c)" fi elif [[ "${CI_OS_NAME}" == "windows"* ]]; then CONDITION="not (slowtest or pgtest)"