diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 948b34353..97c9ab948 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -73,6 +73,9 @@ jobs: - os: ubuntu-latest python: '3.11' mode: lowest-deps + - os: ubuntu-latest + python: '3.12' + mode: datalad steps: - name: Set up environment @@ -123,6 +126,10 @@ jobs: git+https://github.com/hdmf-dev/hdmf \ git+https://github.com/hdmf-dev/hdmf-zarr + - name: Install git-annex and datalad-fuse (for streaming annexed content) + if: matrix.mode == 'datalad' + run: pip install git-annex ".[datalad]" + - name: Create NFS filesystem if: matrix.mode == 'nfs' run: | diff --git a/dandi/metadata/nwb.py b/dandi/metadata/nwb.py index bf43dbcb2..d0468d607 100644 --- a/dandi/metadata/nwb.py +++ b/dandi/metadata/nwb.py @@ -22,13 +22,14 @@ metadata_cache, nwb_has_external_links, ) +from ..support.datalad_fuse import annex_fingerprint from ..utils import find_parent_directory_containing lgr = get_logger() # Disable this for clean hacking -@metadata_cache.memoize_path +@metadata_cache.memoize_path(custom_fingerprint=annex_fingerprint) def get_metadata( path: str | Path | Readable, digest: Digest | None = None ) -> dict[str, Any]: diff --git a/dandi/pynwb_utils.py b/dandi/pynwb_utils.py index 297df05f3..c49293f57 100644 --- a/dandi/pynwb_utils.py +++ b/dandi/pynwb_utils.py @@ -42,6 +42,7 @@ metadata_nwb_subject_fields, ) from .misctypes import Readable +from .support.datalad_fuse import annex_fingerprint from .utils import get_module_version, is_url from .validate._types import ( Origin, @@ -194,7 +195,7 @@ def get_neurodata_types_to_modalities_map() -> dict[str, str]: return ndtypes -@metadata_cache.memoize_path +@metadata_cache.memoize_path(custom_fingerprint=annex_fingerprint) def get_neurodata_types(filepath: str | Path | Readable) -> list[str]: with open_readable(filepath) as fp, h5py.File(fp, "r") as h5file: all_pairs = _scan_neurodata_types(h5file) @@ -808,7 +809,7 @@ def copy_nwb_file(src: str | Path, dest: str | Path) -> str: return str(dest) -@metadata_cache.memoize_path +@metadata_cache.memoize_path(custom_fingerprint=annex_fingerprint) def nwb_has_external_links(filepath: str | Path | Readable) -> bool: with open_readable(filepath) as f, h5py.File(f, "r") as fp: visited = set() diff --git a/dandi/support/datalad_fuse.py b/dandi/support/datalad_fuse.py new file mode 100644 index 000000000..3765860b8 --- /dev/null +++ b/dandi/support/datalad_fuse.py @@ -0,0 +1,320 @@ +""" +Reading the content of annexed files of git-annex_ repositories (such as +DataLad_ datasets) that is not present locally, with datalad-fuse_. + +In a git-annex repository, a (locked) annexed file is a symbolic link into +:file:`.git/annex/objects/`, which is broken until the content is fetched. +datalad-fuse can mount such a repository with FUSE so that the content is +streamed on demand, but its adapter, which the mount is built on, also opens +the files directly with no mount. It asks git-annex where the content is +(``git annex whereis``) and streams it over HTTP(S) with fsspec_. That adapter +is used here, so that the files of a DataLad Dandiset can be read (e.g., for +their metadata, or to be validated) without downloading the (possibly +terabytes of) data. + +The URLs are tried in the order git-annex lists them; for a DataLad Dandiset, +that is the DANDI Archive's API download URL (which redirects to S3) before the +direct S3 URL. datalad-fuse does not use HTTP proxy environment variables. + +This requires datalad-fuse (``pip install "dandi[datalad]"``), which requires +DataLad and git-annex. + +.. _git-annex: https://git-annex.branchable.com +.. _DataLad: https://www.datalad.org +.. _datalad-fuse: https://github.com/datalad/datalad-fuse +.. _fsspec: https://github.com/fsspec/filesystem_spec +""" + +from __future__ import annotations + +import atexit +from dataclasses import dataclass, field +from datetime import datetime +from functools import cache +import gc +import logging +import os +from pathlib import Path +import subprocess +import threading +from typing import IO, Any, cast + +from ..misctypes import Readable + +lgr = logging.getLogger("dandi.support.datalad_fuse") + + +@cache +def get_adapter() -> Any: + """ + The datalad-fuse adapter shared by all files, rooted at the filesystem + root so that it serves files of any dataset, and closed at exit + """ + # Optional dependency: + from datalad_fuse.fsspec import FsspecAdapter + + # caching=False: do not keep the streamed blocks on disk, in the dataset. + # The root and all paths passed to the adapter must be absolute. + adapter = FsspecAdapter(Path(os.path.abspath(os.sep)), caching=False) + atexit.register(adapter.__exit__, None, None, None) + return adapter + + +@cache +def annex_initialized(directory: Path) -> bool: + """ + Whether ``directory`` is in a Git repository in which git-annex is + initialized. (DataLad would otherwise initialize it, modifying the + repository.) + """ + try: + r = subprocess.run( + ["git", "-C", str(directory), "config", "--get", "annex.uuid"], + capture_output=True, + text=True, + ) + except FileNotFoundError: + return False + return r.returncode == 0 and bool(r.stdout.strip()) + + +_gc_lock = threading.Lock() +_gc_pauses = 0 +_gc_was_enabled = False + + +class _GCPausedFile: + """ + A file object streamed by datalad-fuse, during whose lifetime automatic + garbage collection is disabled + + h5py holds a global lock while it reads from a Python file object, and the + read waits for another thread (fsspec's I/O thread, or an HTTP server in + the same process) to fetch the data. If a garbage collection ran in that + thread meanwhile, finalizers of h5py objects (e.g., of an unreferenced + ``NWBHDF5IO``) would wait for the same lock, deadlocking the process. + """ + + def __init__(self, fp: Any) -> None: + global _gc_pauses, _gc_was_enabled + with _gc_lock: + if _gc_pauses == 0: + # Collect pending garbage now, in this thread, which does not + # hold h5py's lock yet + gc.collect() + _gc_was_enabled = gc.isenabled() + gc.disable() + _gc_pauses += 1 + self._fp = fp + self._paused = True + + def __getattr__(self, name: str) -> Any: + return getattr(self._fp, name) + + def __enter__(self) -> _GCPausedFile: + return self + + def __exit__(self, *_exc: Any) -> None: + self.close() + + def __del__(self) -> None: + if self._paused: + self.close() + + def close(self) -> None: + global _gc_pauses + try: + self._fp.close() + finally: + with _gc_lock: + if self._paused: + self._paused = False + _gc_pauses -= 1 + if _gc_pauses == 0 and _gc_was_enabled: + gc.enable() + + +@dataclass +class AnnexedReadableFile(Readable): + """ + A `Readable` for a (locked) annexed file at ``filepath`` (a broken symbolic + link) whose content is not present locally, which is streamed with + datalad-fuse (see the module docstring). ``url`` is only the first URL + known for the content, for reporting: if it cannot be read, the adapter + tries the others. + + Instances are obtained by calling `get_annexed_readable()`. + """ + + filepath: Path + key: str + size: int + url: str + adapter: Any = field(repr=False, compare=False) + + def open(self) -> IO[bytes]: + return cast("IO[bytes]", _GCPausedFile(self.adapter.open(self.filepath, "rb"))) + + def get_size(self) -> int: + return self.size + + def get_mtime(self) -> datetime | None: + return None + + def get_filename(self) -> str: + return self.filepath.name + + def __str__(self) -> str: + return str(self.filepath) + + +def annex_fingerprint(source: Any) -> tuple[str, str] | None: + """ + Fingerprint of ``source`` for ``PersistentCache.memoize_path`` + + Pass it as ``custom_fingerprint`` to cache the results of a function of a + local path or a `Readable` under the git-annex key of a locked annexed + file, paired with its path (see `fscacher.annex_key_fingerprint`), rather + than under ``stat()``: the content of an `AnnexedReadableFile`, which is + not present locally, cannot be ``stat()``-ed. Anything else is handled as + without this. + """ + # Avoid heavy import (fscacher imports joblib, which imports numpy) when + # this module is imported, e.g., by the CLI: + from fscacher import annex_key_fingerprint + + if isinstance(source, AnnexedReadableFile): + source = source.filepath + return cast("tuple[str, str] | None", annex_key_fingerprint(source)) + + +#: Size of the blocks in which streamed content is read and cached +STREAM_BLOCK_SIZE = 256 * 1024 + +#: Maximum number of blocks cached for each open streamed file +STREAM_MAX_BLOCKS = 512 + +#: Name of the fsspec cache defined by `_coalescing_block_cache()` +COALESCING_BLOCK_CACHE = "dandi-coalescing-blockcache" + + +@cache +def _coalescing_block_cache() -> type: + """ + Define (once, so as not to import fsspec when this module is imported) and + register with fsspec a cache that, like fsspec's ``"blockcache"``, keeps + the least recently used blocks of a file, but fetches the consecutive + blocks missing for a read with a single request rather than with one + request per block, which would make reading a large dataset take one + round trip per `STREAM_BLOCK_SIZE` bytes + """ + from collections import OrderedDict + + from fsspec.caching import BaseCache, register_cache + + class CoalescingBlockCache(BaseCache): # type: ignore[misc] + name = COALESCING_BLOCK_CACHE + + def __init__( + self, + blocksize: int, + fetcher: Any, + size: int, + maxblocks: int = STREAM_MAX_BLOCKS, + ) -> None: + super().__init__(blocksize, fetcher, size) + self.maxblocks = maxblocks + self.blocks: OrderedDict[int, bytes] = OrderedDict() + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if start is None: + start = 0 + if end is None or end > self.size: + end = self.size + if start >= end: + return b"" + first = start // self.blocksize + last = (end - 1) // self.blocksize + numbers = range(first, last + 1) + missing = [n for n in numbers if n not in self.blocks] + # Fetch each run of consecutive missing blocks in one request + i = 0 + while i < len(missing): + j = i + while j + 1 < len(missing) and missing[j + 1] == missing[j] + 1: + j += 1 + run_start = missing[i] * self.blocksize + run_end = min((missing[j] + 1) * self.blocksize, self.size) + data = self.fetcher(run_start, run_end) + for n in range(missing[i], missing[j] + 1): + offset = (n - missing[i]) * self.blocksize + self.blocks[n] = data[offset : offset + self.blocksize] + i = j + 1 + out = [] + for n in numbers: + self.blocks.move_to_end(n) + out.append(self.blocks[n]) + # Evict the least recently used blocks, but not those just read + while len(self.blocks) > max(self.maxblocks, len(numbers)): + self.blocks.popitem(last=False) + data = b"".join(out) + offset = first * self.blocksize + return data[start - offset : end - offset] + + register_cache(CoalescingBlockCache, clobber=True) + return CoalescingBlockCache + + +def _read_in_small_blocks(dsap: Any) -> None: + """ + Make the files of a dataset adapter be read in cached blocks of + `STREAM_BLOCK_SIZE` bytes rather than with fsspec's default: h5py reads + many small pieces scattered over the file, and with the default (5 MiB of + read-ahead, discarded whenever a read jumps elsewhere) reading an NWB file + downloads about as much as the whole file each time it is opened + """ + # Avoid heavy import when this module is imported: + from fsspec.implementations.http import HTTPFileSystem + + fs = getattr(dsap, "fs", None) + if isinstance(fs, HTTPFileSystem): + fs.block_size = STREAM_BLOCK_SIZE + _coalescing_block_cache() + fs.cache_type = COALESCING_BLOCK_CACHE + + +def get_annexed_readable(path: str | Path) -> AnnexedReadableFile | None: + """ + Return an `AnnexedReadableFile` for streaming the content of the annexed + file at ``path``, or `None` if datalad-fuse is not installed, ``path`` is + not a (locked) annexed file whose content is missing in a repository in + which git-annex is initialized, its key does not record the size of the + content, or git-annex knows no URL to stream it from + """ + filepath = Path(os.path.abspath(path)) + if not filepath.is_symlink() or filepath.exists(): + # Not a broken symbolic link + return None + try: + from datalad_fuse.fsspec import FileState + except ImportError: + return None + if not annex_initialized(filepath.parent): + return None + try: + dsap, relpath = get_adapter().resolve_dataset(filepath) + _read_in_small_blocks(dsap) + state, key = dsap.get_file_state(relpath) + except Exception as e: + # E.g., a repository without any commit + lgr.debug("%s: Cannot be read with datalad-fuse: %s", filepath, e) + return None + if state is not FileState.NO_CONTENT or key is None or key.size is None: + return None + url = next(dsap.get_urls(str(key)), None) + if url is None: + lgr.debug("%s: git-annex knows no URL for key %s", filepath, key) + return None + return AnnexedReadableFile( + filepath=filepath, key=str(key), size=key.size, url=url, adapter=get_adapter() + ) diff --git a/dandi/support/tests/test_datalad_fuse.py b/dandi/support/tests/test_datalad_fuse.py new file mode 100644 index 000000000..00a0674a0 --- /dev/null +++ b/dandi/support/tests/test_datalad_fuse.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +from datetime import datetime, timezone +import gc +from pathlib import Path +import shutil + +from fscacher import PersistentCache +import h5py +from hdmf.backends.hdf5 import H5DataIO +import numpy as np +from pynwb import NWBHDF5IO, NWBFile, TimeSeries +import pytest + +from ..datalad_fuse import ( + AnnexedReadableFile, + _coalescing_block_cache, + annex_fingerprint, + get_annexed_readable, +) +from ...pynwb_utils import get_neurodata_types +from ...tests.fixtures import RangeHTTPServer, make_git_annex_dandiset +from ...tests.skip import mark + +pytestmark = mark.skipif_no_git_annex + + +@pytest.fixture +def streamed_nwb( + range_http_server: tuple[RangeHTTPServer, str], simple2_nwb: Path, tmp_path: Path +) -> tuple[Path, RangeHTTPServer]: + """ + An annexed copy of ``simple2_nwb`` in a git-annex repository, whose content + is not present but registered at a URL of the HTTP server + """ + pytest.importorskip("datalad_fuse.fsspec") + server, base_url = range_http_server + shutil.copy(simple2_nwb, tmp_path / "served" / "content.nwb") + make_git_annex_dandiset( + tmp_path / "ds", + {"sub-01/sub-01.nwb": (simple2_nwb, f"{base_url}/content.nwb")}, + ) + nwb = tmp_path / "ds" / "sub-01" / "sub-01.nwb" + assert not nwb.exists() + return nwb, server + + +@pytest.mark.ai_generated +def test_annexed_readable( + streamed_nwb: tuple[Path, RangeHTTPServer], simple2_nwb: Path +) -> None: + nwb, server = streamed_nwb + r = get_annexed_readable(nwb) + assert isinstance(r, AnnexedReadableFile) + assert r.key.startswith(f"SHA256E-s{simple2_nwb.stat().st_size}--") + assert r.url.endswith("/content.nwb") + assert r.get_size() == simple2_nwb.stat().st_size + assert r.get_filename() == "sub-01.nwb" + fp = r.open() + try: + # Random access over HTTP is enough for h5py to read the file + with h5py.File(fp, "r") as h5: + assert h5.attrs["nwb_version"] + finally: + fp.close() + assert server.ranged_requests > 0 + + +@pytest.mark.ai_generated +def test_annexed_readable_pauses_gc( + streamed_nwb: tuple[Path, RangeHTTPServer], +) -> None: + # Automatic garbage collection is disabled while any streamed file is open + # (to avoid deadlocks with h5py's lock; see `_GCPausedFile`) + nwb, _ = streamed_nwb + r = get_annexed_readable(nwb) + assert r is not None + assert gc.isenabled() + with r.open() as fp: + assert not gc.isenabled() + with r.open() as fp2: + assert fp2.read(8) == b"\x89HDF\r\n\x1a\n" + assert not gc.isenabled() + fp.seek(1) + assert fp.read(3) == b"HDF" + assert gc.isenabled() + + +@pytest.mark.ai_generated +def test_annexed_readable_reads_little( + range_http_server: tuple[RangeHTTPServer, str], tmp_path: Path +) -> None: + """Reading an NWB file's structure downloads only a small part of it""" + pytest.importorskip("datalad_fuse.fsspec") + server, base_url = range_http_server + nwbfile = NWBFile("streamed", "id", datetime(2026, 1, 1, tzinfo=timezone.utc)) + for i in range(4): + nwbfile.add_acquisition( + TimeSeries( + name=f"ts{i}", + data=H5DataIO(np.arange(1_000_000, dtype="f4"), chunks=(10_000,)), + unit="V", + rate=1.0, + ) + ) + local = tmp_path / "local.nwb" + with NWBHDF5IO(local, "w") as io: + io.write(nwbfile) + size = local.stat().st_size + assert size > 15_000_000 + shutil.copy(local, tmp_path / "served" / "content.nwb") + make_git_annex_dandiset( + tmp_path / "ds", {"sub-01/sub-01.nwb": (local, f"{base_url}/content.nwb")} + ) + r = get_annexed_readable(tmp_path / "ds" / "sub-01" / "sub-01.nwb") + assert r is not None + assert get_neurodata_types.__wrapped__(r) == get_neurodata_types.__wrapped__(local) + # With fsspec's default 5 MiB read-ahead, most of the file was downloaded + assert 0 < server.bytes_served < size / 10 + + +@pytest.mark.ai_generated +def test_coalescing_block_cache() -> None: + pytest.importorskip("fsspec") + content = bytes(range(256)) * 40 + requests: list[tuple[int, int]] = [] + + def fetcher(start: int, end: int) -> bytes: + requests.append((start, end)) + return content[start:end] + + c = _coalescing_block_cache()(100, fetcher, len(content), maxblocks=5) + # The blocks missing for a read are fetched with one request + assert c._fetch(150, 420) == content[150:420] + assert requests == [(100, 500)] + # Cached blocks are not fetched again, and runs of missing blocks around + # them are fetched with one request each + assert c._fetch(30, 720) == content[30:720] + assert requests[1:] == [(0, 100), (500, 800)] + # Reads at the end of the content, and beyond + assert c._fetch(10_190, 20_000) == content[10_190:] + assert requests[3:] == [(10_100, 10_240)] + assert c._fetch(20_000, 20_100) == b"" + # Only the 5 most recently used blocks are kept (but at least the blocks + # of the last read) + assert sorted(c.blocks) == [5, 6, 7, 101, 102] + requests.clear() + assert c._fetch(0, 10_240) == content + assert requests == [(0, 500), (800, 10_100)] + assert len(c.blocks) == 103 + + +@pytest.mark.ai_generated +def test_annexed_readable_large_read( + range_http_server: tuple[RangeHTTPServer, str], tmp_path: Path +) -> None: + """A large read of streamed content is fetched with one request""" + pytest.importorskip("datalad_fuse.fsspec") + server, base_url = range_http_server + local = tmp_path / "local.bin" + content = np.random.default_rng(0).bytes(10_000_000) + local.write_bytes(content) + shutil.copy(local, tmp_path / "served" / "content.bin") + make_git_annex_dandiset( + tmp_path / "ds", {"sub-01/data.bin": (local, f"{base_url}/content.bin")} + ) + r = get_annexed_readable(tmp_path / "ds" / "sub-01" / "data.bin") + assert r is not None + with r.open() as fp: + fp.seek(123) + assert fp.read(9_000_000) == content[123:9_000_123] + assert server.ranged_requests == 1 + fp.seek(5_000_000) + assert fp.read(100) == content[5_000_000:5_000_100] + assert server.ranged_requests == 1 + + +@pytest.mark.ai_generated +def test_annexed_readable_none( + streamed_nwb: tuple[Path, RangeHTTPServer], simple2_nwb: Path, tmp_path: Path +) -> None: + nwb, _ = streamed_nwb + # The content is present + (nwb.parent / "regular.nwb").write_bytes(b"content") + assert get_annexed_readable(nwb.parent / "regular.nwb") is None + # Not in a Git repository + assert get_annexed_readable(simple2_nwb) is None + # A broken symbolic link in a repository without git-annex initialized + plain = tmp_path / "plain" + plain.mkdir() + shutil.copytree(nwb.parent, plain / "sub-01", symlinks=True) + shutil.copytree(nwb.parent.parent / ".git", plain / ".git") + (plain / ".git" / "config").write_text("[core]\n\tbare = false\n") + assert get_annexed_readable(plain / "sub-01" / "sub-01.nwb") is None + + +@pytest.mark.ai_generated +def test_annexed_readable_cached_by_key( + streamed_nwb: tuple[Path, RangeHTTPServer], simple2_nwb: Path, tmp_path: Path +) -> None: + nwb, server = streamed_nwb + cache = PersistentCache(path=tmp_path / "cache") + neurodata_types = cache.memoize_path(custom_fingerprint=annex_fingerprint)( + get_neurodata_types.__wrapped__ + ) + r = get_annexed_readable(nwb) + assert r is not None + expected = get_neurodata_types.__wrapped__(simple2_nwb) + assert neurodata_types(r) == expected + requests = server.ranged_requests + assert requests > 0 + # Served from the cache, without the content being read again: by the + # readable, and by the path of the (broken) link itself + assert neurodata_types(get_annexed_readable(nwb)) == expected + assert neurodata_types(nwb) == expected + assert server.ranged_requests == requests diff --git a/dandi/tests/fixtures.py b/dandi/tests/fixtures.py index 9b128a23d..ffdfd611d 100644 --- a/dandi/tests/fixtures.py +++ b/dandi/tests/fixtures.py @@ -3,12 +3,16 @@ from collections.abc import Callable, Iterator from dataclasses import dataclass, field, replace from datetime import datetime, timezone +from functools import partial +from http import HTTPStatus +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer import logging import os from pathlib import Path import re import shutil from subprocess import DEVNULL, check_output, run +import threading from time import sleep from typing import Any, Literal from uuid import uuid4 @@ -352,6 +356,128 @@ def fixture( return fixture +def make_git_annex_dandiset(dandiset: Path, files: dict[str, tuple[Path, str]]) -> None: + """ + Make the directory ``dandiset`` (created if needed, with a minimal + :file:`dandiset.yaml`) a git-annex repository looking like a clone of a + DataLad Dandiset whose annexed content has not been fetched: for each + ``relpath: (source, url)`` item of ``files``, a copy of ``source`` is + annexed at ``relpath`` with ``url`` registered for it, and then dropped. + This needs git-annex. + """ + env = { + **os.environ, + "GIT_AUTHOR_NAME": "DANDI tests", + "GIT_AUTHOR_EMAIL": "tests@dandiarchive.org", + "GIT_COMMITTER_NAME": "DANDI tests", + "GIT_COMMITTER_EMAIL": "tests@dandiarchive.org", + } + + def git(*args: str) -> str: + return check_output( + ["git", "-C", str(dandiset), *args], env=env, text=True + ).strip() + + dandiset.mkdir(parents=True, exist_ok=True) + (dandiset / dandiset_metadata_file).write_text( + "identifier: '000027'\nname: Test\ndescription: Test dandiset\n" + ) + git("init", "-q") + git("annex", "init", "-q") + git("config", "annex.backend", "SHA256E") + git("add", dandiset_metadata_file) + for relpath, (source, _) in files.items(): + (dandiset / relpath).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(source, dandiset / relpath) + git("annex", "add", "-q", *files) + for relpath, (_, url) in files.items(): + key = git("annex", "lookupkey", relpath) + git("annex", "registerurl", key, url) + git("commit", "-q", "-m", "Add files") + git("annex", "drop", "-q", "--force", *files) + + +class RangeHTTPServer(ThreadingHTTPServer): + """An HTTP server counting the range requests and the bytes it served""" + + ranged_requests: int = 0 + bytes_served: int = 0 + + +class RangeHTTPRequestHandler(SimpleHTTPRequestHandler): + """ + A handler serving files with support for ``HEAD`` and single-range ``GET`` + requests, like S3 does + """ + + def log_message(self, format: str, *args: Any) -> None: + pass + + def _send_headers( + self, status: HTTPStatus, length: int, content_range: str | None = None + ) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(length)) + self.send_header("Accept-Ranges", "bytes") + if content_range is not None: + self.send_header("Content-Range", content_range) + self.end_headers() + + def do_HEAD(self) -> None: + path = self.translate_path(self.path) + if not os.path.isfile(path): + self.send_error(HTTPStatus.NOT_FOUND) + return + self._send_headers(HTTPStatus.OK, os.path.getsize(path)) + + def do_GET(self) -> None: + path = self.translate_path(self.path) + if not os.path.isfile(path): + self.send_error(HTTPStatus.NOT_FOUND) + return + size = os.path.getsize(path) + start, end = 0, size - 1 + if m := re.fullmatch(r"bytes=(\d+)-(\d*)", self.headers.get("Range", "")): + start = int(m[1]) + end = min(int(m[2]) if m[2] else size - 1, size - 1) + assert isinstance(self.server, RangeHTTPServer) + self.server.ranged_requests += 1 + self._send_headers( + HTTPStatus.PARTIAL_CONTENT, + end - start + 1, + f"bytes {start}-{end}/{size}", + ) + else: + self._send_headers(HTTPStatus.OK, size) + assert isinstance(self.server, RangeHTTPServer) + self.server.bytes_served += end - start + 1 + with open(path, "rb") as fp: + fp.seek(start) + self.wfile.write(fp.read(end - start + 1)) + + +@pytest.fixture() +def range_http_server(tmp_path: Path) -> Iterator[tuple[RangeHTTPServer, str]]: + """ + Serve the files in ``tmp_path / "served"`` over HTTP with support for range + requests; yields the server and its base URL + """ + served = tmp_path / "served" + served.mkdir() + server = RangeHTTPServer( + ("127.0.0.1", 0), partial(RangeHTTPRequestHandler, directory=str(served)) + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server, f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + thread.join() + + def _make_subdirs_dandisets(path: Path) -> None: for bids_dataset_path in path.iterdir(): if bids_dataset_path.is_dir(): diff --git a/dandi/tests/skip.py b/dandi/tests/skip.py index 54d864b76..9e30b195d 100644 --- a/dandi/tests/skip.py +++ b/dandi/tests/skip.py @@ -125,6 +125,10 @@ def no_git(): return "Git not installed", shutil.which("git") is None +def no_git_annex(): + return "git-annex not installed", shutil.which("git-annex") is None + + # ### END MODIFIED CODE @@ -162,6 +166,7 @@ def on_windows(): no_docker_commands, no_docker_engine, no_git, + no_git_annex, no_network, # no_singularity, no_ssh, diff --git a/pyproject.toml b/pyproject.toml index 7a6ba7b9b..3fcdedd51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "dandischema >= 0.12.0, != 0.13.0, < 0.15.0", "etelemetry >= 0.2.2", "fasteners >= 0.19", - "fscacher >= 0.3.0", + "fscacher >= 0.5.0", # Floor raised to what `py3-lowest` actually resolves: modern pynwb pins # hdmf >= 4, so declared floor is now the tested floor. "hdmf >= 4.1.0", @@ -120,6 +120,11 @@ test = [ # 5.0 switched to urllib3.connection.VerifiedHTTPSConnection (urllib3 2.x compat) "vcrpy >= 5.0.0", ] +# For reading the content of annexed files of DataLad datasets without +# fetching it (requires git-annex) +datalad = [ + "datalad-fuse >= 0.5.1", +] tools = [ "boto3", ] @@ -215,6 +220,7 @@ follow_imports = "skip" module = [ "bidsschematools.*", "click_didyoumean.*", + "datalad_fuse.*", "etelemetry.*", "fasteners.*", "fscacher.*",