From c9b2d02a57d09a6b131101d4f032f01cc81a9ec2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 17:12:48 +0000 Subject: [PATCH 1/2] Stream HDF5 files with datalad-fuse's remfile backend, without a custom cache Try moving the streaming of NWB files to the remfile backend of datalad-fuse (datalad/datalad-fuse#131), using the `remfile-dev` branch of CodyCBakerPhD/datalad-fuse (#131 plus `init_annex`): - The adapter is created with `backends="remfile,fsspec"`: remfile for HDF5-based files, fsspec for the others (and for HDF5-based files that remfile fails to open), and `init_annex=False`, so `annex_initialized()` is gone. - `_coalescing_block_cache()` and `_read_in_small_blocks()` are gone: remfile reads an NWB file's structure downloading a small part of it (0.6 of 16 MB in `test_annexed_readable_reads_little`, where fsspec's default cache downloads 64 MB). - Garbage collection is only paused while files read with fsspec are open. remfile fetches the data in the reading thread, so no finalizer can run in another thread while h5py holds its lock waiting for it. With fsspec that still deadlocks with a patched h5py, through `HDMFIO.__del__` calling `is_open()` in fsspec's I/O thread. - remfile is added to the `datalad` extra, and the `datalad-dev` CI job installs the `remfile-dev` branch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qn5WBSiQgoZoL4fytF6nEr --- .github/workflows/run-tests.yml | 9 +- dandi/cli/tests/test_cmd_validate.py | 2 +- dandi/support/datalad_fuse.py | 193 ++++++----------------- dandi/support/tests/test_datalad_fuse.py | 110 ++++--------- dandi/validate/tests/test_core.py | 4 +- docs/source/cmdline/validate.rst | 16 +- pyproject.toml | 4 + 7 files changed, 96 insertions(+), 242 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f4d85c381..ed999385a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -133,12 +133,13 @@ jobs: if: matrix.mode == 'datalad' run: pip install git-annex ".[datalad]" - # TODO: drop once datalad/datalad-fuse#131 is released - - name: Install git-annex, remfile and development datalad-fuse + # TODO: use a release once datalad/datalad-fuse#131 and init_annex are + # released + - name: Install git-annex and development datalad-fuse if: matrix.mode == 'datalad-dev' run: | - pip install git-annex remfile ".[datalad]" \ - "datalad-fuse @ git+https://github.com/datalad/datalad-fuse@refs/pull/131/head" + pip install git-annex ".[datalad]" \ + "datalad-fuse @ git+https://github.com/CodyCBakerPhD/datalad-fuse@remfile-dev" - name: Create NFS filesystem if: matrix.mode == 'nfs' diff --git a/dandi/cli/tests/test_cmd_validate.py b/dandi/cli/tests/test_cmd_validate.py index aef651740..6548bb1a0 100644 --- a/dandi/cli/tests/test_cmd_validate.py +++ b/dandi/cli/tests/test_cmd_validate.py @@ -1008,7 +1008,7 @@ def test_validate_missing_file_content_stream( from ...tests.skip import skipif skipif.no_git_annex() - pytest.importorskip("datalad_fuse.fsspec") + pytest.importorskip("datalad_fuse.adapter") _, base_url = range_http_server shutil.copy(simple3_nwb, tmp_path / "served" / "content.nwb") ds = tmp_path / "ds" diff --git a/dandi/support/datalad_fuse.py b/dandi/support/datalad_fuse.py index d423842b0..99a221f3c 100644 --- a/dandi/support/datalad_fuse.py +++ b/dandi/support/datalad_fuse.py @@ -7,22 +7,26 @@ datalad-fuse can mount such a repository with FUSE so that the content is streamed on demand, but its adapter, which the mount is built on, also opens the files directly with no mount. It asks git-annex where the content is -(``git annex whereis``) and streams it over HTTP(S) with fsspec_. That adapter -is used here, so that the files of a DataLad Dandiset can be read (e.g., for -their metadata, or to be validated) without downloading the (possibly -terabytes of) data. +(``git annex whereis``) and streams it over HTTP(S): HDF5-based files (such as +NWB) with remfile_, which is optimized for HDF5's access pattern and reads in +the calling thread, and other files with fsspec_. That adapter is used here, +so that the files of a DataLad Dandiset can be read (e.g., for their metadata, +or to be validated) without downloading the (possibly terabytes of) data. The URLs are tried in the order git-annex lists them; for a DataLad Dandiset, that is the DANDI Archive's API download URL (which redirects to S3) before the direct S3 URL. datalad-fuse does not use HTTP proxy environment variables. This requires datalad-fuse (``pip install "dandi[datalad]"``), which requires -DataLad and git-annex. +DataLad and git-annex, in a version with pluggable backends +(datalad/datalad-fuse#131) and ``init_annex`` (for now, the ``remfile-dev`` +branch of https://github.com/CodyCBakerPhD/datalad-fuse), and remfile. .. _git-annex: https://git-annex.branchable.com .. _DataLad: https://www.datalad.org .. _datalad-fuse: https://github.com/datalad/datalad-fuse .. _fsspec: https://github.com/fsspec/filesystem_spec +.. _remfile: https://github.com/flatironinstitute/remfile """ from __future__ import annotations @@ -37,7 +41,6 @@ import logging import os from pathlib import Path -import subprocess import threading from typing import IO, Any, cast @@ -53,39 +56,24 @@ def get_adapter() -> Any: root so that it serves files of any dataset, and closed at exit """ # Optional dependency: - try: - # datalad-fuse with pluggable backends (datalad/datalad-fuse#131): - # remfile for HDF5-based files (such as NWB) if installed, fsspec - # otherwise, as configured with datalad.fusefs.backends - from datalad_fuse.adapter import RemoteFilesystemAdapter as Adapter - except ImportError: - from datalad_fuse.fsspec import FsspecAdapter as Adapter - - # caching=False: do not keep the streamed blocks on disk, in the dataset. - # The root and all paths passed to the adapter must be absolute. - adapter = Adapter(Path(os.path.abspath(os.sep)), caching=False) + from datalad_fuse.adapter import RemoteFilesystemAdapter + + adapter = RemoteFilesystemAdapter( + # The root and all paths passed to the adapter must be absolute + Path(os.path.abspath(os.sep)), + # Do not keep the streamed blocks on disk, in the dataset + caching=False, + # remfile for HDF5-based files (such as NWB), fsspec for the others, + # and for HDF5-based files that remfile fails to open + backends="remfile,fsspec", + # Do not initialize git-annex (modifying the repository) where it is + # not + init_annex=False, + ) atexit.register(adapter.__exit__, None, None, None) return adapter -@cache -def annex_initialized(directory: Path) -> bool: - """ - Whether ``directory`` is in a Git repository in which git-annex is - initialized. (DataLad would otherwise initialize it, modifying the - repository.) - """ - try: - r = subprocess.run( - ["git", "-C", str(directory), "config", "--get", "annex.uuid"], - capture_output=True, - text=True, - ) - except FileNotFoundError: - return False - return r.returncode == 0 and bool(r.stdout.strip()) - - _gc_lock = threading.Lock() _gc_pauses = 0 _gc_was_enabled = False @@ -93,14 +81,16 @@ def annex_initialized(directory: Path) -> bool: class _GCPausedFile: """ - A file object streamed by datalad-fuse, during whose lifetime automatic - garbage collection is disabled - - h5py holds a global lock while it reads from a Python file object, and the - read waits for another thread (fsspec's I/O thread, or an HTTP server in - the same process) to fetch the data. If a garbage collection ran in that - thread meanwhile, finalizers of h5py objects (e.g., of an unreferenced - ``NWBHDF5IO``) would wait for the same lock, deadlocking the process. + A file object streamed by datalad-fuse with fsspec, during whose lifetime + automatic garbage collection is disabled + + h5py holds a global lock while it reads from a Python file object, and a + read of an fsspec file waits for fsspec's I/O thread to fetch the data. If + a garbage collection ran in that thread meanwhile, finalizers calling h5py + (e.g., those of h5py objects, or ``HDMFIO.__del__``, which checks whether + the file is open) would wait for the same lock, deadlocking the process. + Files read with remfile need no pause: remfile fetches the data in the + reading thread. """ def __init__(self, fp: Any) -> None: @@ -161,7 +151,10 @@ class AnnexedReadableFile(Readable): adapter: Any = field(repr=False, compare=False) def open(self) -> IO[bytes]: - return cast("IO[bytes]", _GCPausedFile(self.adapter.open(self.filepath, "rb"))) + fp = self.adapter.open(self.filepath, "rb") + if type(fp).__name__ != "RemfileWrapper": + fp = _GCPausedFile(fp) + return cast("IO[bytes]", fp) def get_size(self) -> int: return self.size @@ -196,101 +189,6 @@ def annex_fingerprint(source: Any) -> tuple[str, str] | None: return cast("tuple[str, str] | None", annex_key_fingerprint(source)) -#: Size of the blocks in which streamed content is read and cached -STREAM_BLOCK_SIZE = 256 * 1024 - -#: Maximum number of blocks cached for each open streamed file -STREAM_MAX_BLOCKS = 512 - -#: Name of the fsspec cache defined by `_coalescing_block_cache()` -COALESCING_BLOCK_CACHE = "dandi-coalescing-blockcache" - - -@cache -def _coalescing_block_cache() -> type: - """ - Define (once, so as not to import fsspec when this module is imported) and - register with fsspec a cache that, like fsspec's ``"blockcache"``, keeps - the least recently used blocks of a file, but fetches the consecutive - blocks missing for a read with a single request rather than with one - request per block, which would make reading a large dataset take one - round trip per `STREAM_BLOCK_SIZE` bytes - """ - from collections import OrderedDict - - from fsspec.caching import BaseCache, register_cache - - class CoalescingBlockCache(BaseCache): # type: ignore[misc] - name = COALESCING_BLOCK_CACHE - - def __init__( - self, - blocksize: int, - fetcher: Any, - size: int, - maxblocks: int = STREAM_MAX_BLOCKS, - ) -> None: - super().__init__(blocksize, fetcher, size) - self.maxblocks = maxblocks - self.blocks: OrderedDict[int, bytes] = OrderedDict() - - def _fetch(self, start: int | None, end: int | None) -> bytes: - if start is None: - start = 0 - if end is None or end > self.size: - end = self.size - if start >= end: - return b"" - first = start // self.blocksize - last = (end - 1) // self.blocksize - numbers = range(first, last + 1) - missing = [n for n in numbers if n not in self.blocks] - # Fetch each run of consecutive missing blocks in one request - i = 0 - while i < len(missing): - j = i - while j + 1 < len(missing) and missing[j + 1] == missing[j] + 1: - j += 1 - run_start = missing[i] * self.blocksize - run_end = min((missing[j] + 1) * self.blocksize, self.size) - data = self.fetcher(run_start, run_end) - for n in range(missing[i], missing[j] + 1): - offset = (n - missing[i]) * self.blocksize - self.blocks[n] = data[offset : offset + self.blocksize] - i = j + 1 - out = [] - for n in numbers: - self.blocks.move_to_end(n) - out.append(self.blocks[n]) - # Evict the least recently used blocks, but not those just read - while len(self.blocks) > max(self.maxblocks, len(numbers)): - self.blocks.popitem(last=False) - data = b"".join(out) - offset = first * self.blocksize - return data[start - offset : end - offset] - - register_cache(CoalescingBlockCache, clobber=True) - return CoalescingBlockCache - - -def _read_in_small_blocks(dsap: Any) -> None: - """ - Make the files of a dataset adapter be read in cached blocks of - `STREAM_BLOCK_SIZE` bytes rather than with fsspec's default: h5py reads - many small pieces scattered over the file, and with the default (5 MiB of - read-ahead, discarded whenever a read jumps elsewhere) reading an NWB file - downloads about as much as the whole file each time it is opened - """ - # Avoid heavy import when this module is imported: - from fsspec.implementations.http import HTTPFileSystem - - fs = getattr(dsap, "fs", None) - if isinstance(fs, HTTPFileSystem): - fs.block_size = STREAM_BLOCK_SIZE - _coalescing_block_cache() - fs.cache_type = COALESCING_BLOCK_CACHE - - def get_annexed_readable(path: str | Path) -> AnnexedReadableFile | None: """ Return an `AnnexedReadableFile` for streaming the content of the annexed @@ -306,15 +204,12 @@ def get_annexed_readable(path: str | Path) -> AnnexedReadableFile | None: try: from datalad_fuse.adapter import FileState except ImportError: - try: - from datalad_fuse.fsspec import FileState - except ImportError: - return None - if not annex_initialized(filepath.parent): + # Not installed, or a version without pluggable backends return None try: dsap, relpath = get_adapter().resolve_dataset(filepath) - _read_in_small_blocks(dsap) + if not dsap.annex_initialized: + return None state, key = dsap.get_file_state(relpath) except Exception as e: # E.g., a repository without any commit @@ -322,10 +217,10 @@ def get_annexed_readable(path: str | Path) -> AnnexedReadableFile | None: return None if state is not FileState.NO_CONTENT or key is None or key.size is None: return None - urls: Iterator[str] = dsap.get_urls(str(key)) - if hasattr(dsap, "get_exporttree_urls"): - # Fallback to S3 exports of datalad/datalad-fuse#131 - urls = chain(urls, dsap.get_exporttree_urls(relpath, key)) + # Including the S3 exports git-annex knows no URLs on + urls: Iterator[str] = chain( + dsap.get_urls(str(key)), dsap.get_exporttree_urls(relpath, key) + ) url = next(urls, None) if url is None: lgr.debug("%s: git-annex knows no URL for key %s", filepath, key) diff --git a/dandi/support/tests/test_datalad_fuse.py b/dandi/support/tests/test_datalad_fuse.py index f86d89b2e..d65e3c26c 100644 --- a/dandi/support/tests/test_datalad_fuse.py +++ b/dandi/support/tests/test_datalad_fuse.py @@ -12,12 +12,7 @@ from pynwb import NWBHDF5IO, NWBFile, TimeSeries import pytest -from ..datalad_fuse import ( - AnnexedReadableFile, - _coalescing_block_cache, - annex_fingerprint, - get_annexed_readable, -) +from ..datalad_fuse import AnnexedReadableFile, annex_fingerprint, get_annexed_readable from ...pynwb_utils import get_neurodata_types from ...tests.fixtures import RangeHTTPServer, make_git_annex_dandiset from ...tests.skip import mark @@ -33,7 +28,7 @@ def streamed_nwb( An annexed copy of ``simple2_nwb`` in a git-annex repository, whose content is not present but registered at a URL of the HTTP server """ - pytest.importorskip("datalad_fuse.fsspec") + pytest.importorskip("datalad_fuse.adapter") server, base_url = range_http_server shutil.copy(simple2_nwb, tmp_path / "served" / "content.nwb") make_git_annex_dandiset( @@ -66,32 +61,16 @@ def test_annexed_readable( assert server.ranged_requests > 0 -@pytest.mark.ai_generated -def test_annexed_readable_pauses_gc( - streamed_nwb: tuple[Path, RangeHTTPServer], -) -> None: - # Automatic garbage collection is disabled while any streamed file is open - # (to avoid deadlocks with h5py's lock; see `_GCPausedFile`) - nwb, _ = streamed_nwb - r = get_annexed_readable(nwb) - assert r is not None - assert gc.isenabled() - with r.open() as fp: - assert not gc.isenabled() - with r.open() as fp2: - assert fp2.read(8) == b"\x89HDF\r\n\x1a\n" - assert not gc.isenabled() - fp.seek(1) - assert fp.read(3) == b"HDF" - assert gc.isenabled() - - @pytest.mark.ai_generated def test_annexed_readable_reads_little( range_http_server: tuple[RangeHTTPServer, str], tmp_path: Path ) -> None: - """Reading an NWB file's structure downloads only a small part of it""" - pytest.importorskip("datalad_fuse.fsspec") + """ + Reading an NWB file's structure with remfile downloads only a small part of + it (with fsspec's default cache, about 4 times the file is) + """ + pytest.importorskip("datalad_fuse.adapter") + pytest.importorskip("remfile") server, base_url = range_http_server nwbfile = NWBFile("streamed", "id", datetime(2026, 1, 1, tzinfo=timezone.utc)) for i in range(4): @@ -115,57 +94,21 @@ def test_annexed_readable_reads_little( r = get_annexed_readable(tmp_path / "ds" / "sub-01" / "sub-01.nwb") assert r is not None assert get_neurodata_types.__wrapped__(r) == get_neurodata_types.__wrapped__(local) - assert server.bytes_served > 0 with r.open() as fp: - remfile_used = type(fp._fp).__name__ == "RemfileWrapper" # type: ignore[attr-defined] - if not remfile_used: - # With fsspec's default 5 MiB read-ahead, most of the file was - # downloaded. (remfile, used for HDF5 files by - # datalad/datalad-fuse#131 when installed, reads ahead by itself, in - # chunks growing up to 100 MiB that datalad-fuse does not configure.) - assert server.bytes_served < size / 10 - - -@pytest.mark.ai_generated -def test_coalescing_block_cache() -> None: - pytest.importorskip("fsspec") - content = bytes(range(256)) * 40 - requests: list[tuple[int, int]] = [] - - def fetcher(start: int, end: int) -> bytes: - requests.append((start, end)) - return content[start:end] - - c = _coalescing_block_cache()(100, fetcher, len(content), maxblocks=5) - # The blocks missing for a read are fetched with one request - assert c._fetch(150, 420) == content[150:420] - assert requests == [(100, 500)] - # Cached blocks are not fetched again, and runs of missing blocks around - # them are fetched with one request each - assert c._fetch(30, 720) == content[30:720] - assert requests[1:] == [(0, 100), (500, 800)] - # Reads at the end of the content, and beyond - assert c._fetch(10_190, 20_000) == content[10_190:] - assert requests[3:] == [(10_100, 10_240)] - assert c._fetch(20_000, 20_100) == b"" - # Only the 5 most recently used blocks are kept (but at least the blocks - # of the last read) - assert sorted(c.blocks) == [5, 6, 7, 101, 102] - requests.clear() - assert c._fetch(0, 10_240) == content - assert requests == [(0, 500), (800, 10_100)] - assert len(c.blocks) == 103 + assert type(fp).__name__ == "RemfileWrapper" + assert 0 < server.bytes_served < size / 10 @pytest.mark.ai_generated -def test_annexed_readable_large_read( +def test_annexed_readable_pauses_gc( range_http_server: tuple[RangeHTTPServer, str], tmp_path: Path ) -> None: - """A large read of streamed content is fetched with one request""" - pytest.importorskip("datalad_fuse.fsspec") + # Automatic garbage collection is disabled while any file streamed with + # fsspec is open (to avoid deadlocks with h5py's lock; see `_GCPausedFile`) + pytest.importorskip("datalad_fuse.adapter") server, base_url = range_http_server local = tmp_path / "local.bin" - content = np.random.default_rng(0).bytes(10_000_000) + content = bytes(range(256)) * 1000 local.write_bytes(content) shutil.copy(local, tmp_path / "served" / "content.bin") make_git_annex_dandiset( @@ -173,13 +116,15 @@ def test_annexed_readable_large_read( ) r = get_annexed_readable(tmp_path / "ds" / "sub-01" / "data.bin") assert r is not None + assert gc.isenabled() with r.open() as fp: - fp.seek(123) - assert fp.read(9_000_000) == content[123:9_000_123] - assert server.ranged_requests == 1 - fp.seek(5_000_000) - assert fp.read(100) == content[5_000_000:5_000_100] - assert server.ranged_requests == 1 + assert not gc.isenabled() + with r.open() as fp2: + assert fp2.read(8) == content[:8] + assert not gc.isenabled() + fp.seek(1000) + assert fp.read(3) == content[1000:1003] + assert gc.isenabled() @pytest.mark.ai_generated @@ -227,7 +172,10 @@ def test_annexed_readable_cached_by_key( def test_annexed_readable_remfile( streamed_nwb: tuple[Path, RangeHTTPServer], simple2_nwb: Path ) -> None: - """With datalad/datalad-fuse#131 and remfile, NWB files are read with remfile""" + """ + With remfile installed, NWB files are read with remfile, with garbage + collection not paused + """ pytest.importorskip("datalad_fuse.adapter") pytest.importorskip("remfile") from datalad_fuse.remfile import RemfileWrapper @@ -237,8 +185,8 @@ def test_annexed_readable_remfile( assert r is not None fp = r.open() try: - # (wrapped to pause garbage collection while it is open) - assert isinstance(fp._fp, RemfileWrapper) # type: ignore[attr-defined] + assert isinstance(fp, RemfileWrapper) + assert gc.isenabled() with h5py.File(fp, "r") as h5: assert h5.attrs["nwb_version"] finally: diff --git a/dandi/validate/tests/test_core.py b/dandi/validate/tests/test_core.py index 23c296af5..08dde76c8 100644 --- a/dandi/validate/tests/test_core.py +++ b/dandi/validate/tests/test_core.py @@ -338,7 +338,7 @@ def streamable_dandiset( its key is needed). """ skipif.no_git_annex() - pytest.importorskip("datalad_fuse.fsspec") + pytest.importorskip("datalad_fuse.adapter") server, base_url = range_http_server shutil.copy(simple3_nwb, tmp_path / "served" / "content.nwb") video = tmp_path / "video.mp4" @@ -520,7 +520,7 @@ def summary(results: list[ValidationResult]) -> list[tuple]: @pytest.mark.ai_generated def test_validate_stream_not_streamable(tmp_path: Path) -> None: """stream policy emits an error for a broken symlink that cannot be streamed.""" - pytest.importorskip("datalad_fuse.fsspec") + pytest.importorskip("datalad_fuse.adapter") skipif.no_git_annex() ds = _make_dandiset_with_broken_symlink(tmp_path) results = list(validate(ds, missing_file_content=MissingFileContent.stream)) diff --git a/docs/source/cmdline/validate.rst b/docs/source/cmdline/validate.rst index ed0d8ad26..dca6206de 100644 --- a/docs/source/cmdline/validate.rst +++ b/docs/source/cmdline/validate.rst @@ -122,11 +122,17 @@ Notes: - git-annex must be initialized in the clone, which ``datalad clone`` does (after ``git clone``, run ``git annex init``). -- With the development version of datalad-fuse from - https://github.com/datalad/datalad-fuse/pull/131 and remfile_ installed - (``pip install remfile``), NWB and other HDF5-based files are read with - remfile, which is optimized for HDF5's access pattern, and content exported - to S3 without a URL registered in git-annex can be found too. +- For now, this requires a development version of datalad-fuse, with + pluggable backends (https://github.com/datalad/datalad-fuse/pull/131) and + without initializing git-annex where it is not:: + + pip install "datalad-fuse @ git+https://github.com/CodyCBakerPhD/datalad-fuse@remfile-dev" + + NWB and other HDF5-based files are read with remfile_ (installed with + ``dandi[datalad]``), which is optimized for HDF5's access pattern and + downloads only a small part of a file to read its structure; other files are + read with fsspec. Content exported to S3 without a URL registered in + git-annex can be found too. - datalad-fuse does not use the HTTP proxy environment variables (``HTTPS_PROXY`` etc.), so the URLs must be reachable directly. - Some nwbinspector checks read data arrays (e.g., timestamps), so the amount diff --git a/pyproject.toml b/pyproject.toml index 3fcdedd51..c4355b650 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,7 +123,11 @@ test = [ # For reading the content of annexed files of DataLad datasets without # fetching it (requires git-annex) datalad = [ + # Streaming needs datalad-fuse with pluggable backends + # (datalad/datalad-fuse#131) and init_annex, not released yet "datalad-fuse >= 0.5.1", + # The datalad-fuse backend for HDF5-based files (such as NWB) + "remfile", ] tools = [ "boto3", From 5f9d5f9d3daa51f192382f2757f377cb9993fc1c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:02:32 +0000 Subject: [PATCH 2/2] CI: install the development datalad-fuse without its dependencies The fork having no tags, versioneer gives the remfile-dev branch the version 0+untagged..., which does not satisfy the datalad extra's `datalad-fuse >= 0.5.1`, so pip could not resolve the requirements. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qn5WBSiQgoZoL4fytF6nEr --- .github/workflows/run-tests.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ed999385a..ec21b6e1a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -134,12 +134,17 @@ jobs: run: pip install git-annex ".[datalad]" # TODO: use a release once datalad/datalad-fuse#131 and init_annex are - # released + # released. The branch is installed without its dependencies (which are + # then installed explicitly) as, the fork having no tags, its version is + # 0+untagged..., which does not satisfy the `datalad` extra. - name: Install git-annex and development datalad-fuse if: matrix.mode == 'datalad-dev' run: | - pip install git-annex ".[datalad]" \ + pip install git-annex ".[datalad]" + pip install --no-deps --force-reinstall \ "datalad-fuse @ git+https://github.com/CodyCBakerPhD/datalad-fuse@remfile-dev" + pip install boto3 "mfusepy ~= 3.1" "remfile ~= 0.1.15" + python -c "from datalad_fuse.adapter import DatasetAdapter as D; import inspect; assert 'init_annex' in inspect.signature(D).parameters" - name: Create NFS filesystem if: matrix.mode == 'nfs'