Resume interrupted data downloads instead of passing them off as complete

INSTALL.md

@@ -20,6 +20,25 @@ venv with no conflicts.
 You need three things that pip cannot install for you: **Python**, **git**, and
 **ffmpeg**. (pip and `venv` come bundled with Python.)
 
+> **macOS — install Homebrew first.** Every `brew` command below needs it, and a
+> fresh Mac doesn't have it. Run this in **Terminal directly** (it prompts you
+> interactively, so it can't be pasted into another tool):
+>
+> ```bash
+> /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+> ```
+>
+> It asks for your login password because it needs **administrator rights**. If
+> it stops with `Need sudo access on macOS` your account isn't an admin — that's
+> common on lab or managed machines. Either ask whoever administers it, or skip
+> Homebrew entirely and use the [python.org installer](https://www.python.org/downloads/)
+> plus a prebuilt [ffmpeg binary](https://ffmpeg.org/download.html#build-mac)
+> (git comes from `xcode-select --install`, which needs no admin rights).
+>
+> When it finishes it may print a `Next steps` block asking you to add `brew` to
+> your PATH — run those two commands, then **open a new terminal** so `brew` is
+> found.
+
 ### Python 3.11–3.13
 
 CaMAP requires Python in this range; 3.12 is recommended. **3.14 (the current

scripts/get_data.py

@@ -32,6 +32,11 @@
 upstream step) is kept, and only missing stages are downloaded. A stage that
 isn't in the deposit yet is simply skipped.
 
+It also **resumes**: within ``raw``, each file is checked against the archive's
+size and MD5, so an interrupted multi-GB pull re-fetches only what's missing or
+half-written. Re-running after a dropped connection is safe and cheap — it will
+not mistake a partial download for a complete one.
+
 The **live** dataset is published *during* the workshop, so its DOI isn't baked
 in — pass it at fetch time with ``--doi`` (no code edit, no ``git pull``):
 
@@ -157,6 +162,43 @@ def _human(n: float) -> str:
         n /= 1024
 
 
+def _md5(path: Path) -> str:
+    """MD5 of *path*, read in chunks (files here run to hundreds of MB)."""
+    h = hashlib.md5()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(_CHUNK), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _size_ok(path: Path, rec: dict) -> bool:
+    """True if *path* exists and matches the size the archive published.
+
+    Cheap (a ``stat``, no reading) and catches the truncated-by-Ctrl-C case,
+    which is how an interrupted download actually fails.
+    """
+    if not path.is_file():
+        return False
+    want_size = int(rec.get("size") or 0)
+    return path.stat().st_size == want_size if want_size else True
+
+
+def _is_complete(path: Path, rec: dict) -> bool:
+    """True if *path* already holds the archive's copy of *rec*.
+
+    Size first, then the MD5 the archive published — which also catches the
+    right-length-but-wrong-bytes case. Without this, a file left half-written by
+    an interrupted run is indistinguishable from a finished one.
+    """
+    if not _size_ok(path, rec):
+        return False
+    want_md5 = rec.get("md5")
+    if want_md5:
+        return _md5(path) == want_md5
+    # No hash published — size is all we can go on.
+    return bool(int(rec.get("size") or 0))
+
+
 def _dataverse_download(server: str, rec: dict, dest_path: Path, label: str = "") -> None:
     """Stream a Dataverse datafile to *dest_path*, verifying its MD5.
 
@@ -185,10 +227,27 @@ def _dataverse_download(server: str, rec: dict, dest_path: Path, label: str = ""
     print(f"       {label} done ({_human(done)})")
 
 
-def _fetch(kind: str, ctx: str, registry: dict, names: list[str], dest: Path) -> None:
-    """Download *names* (a subset of *registry*) into *dest*, verified by hash."""
+def _fetch(kind: str, ctx: str, registry: dict, names: list[str], dest: Path,
+           force: bool = False) -> None:
+    """Download *names* (a subset of *registry*) into *dest*, verified by hash.
+
+    Files already present and matching the archive (size + MD5) are skipped, so
+    an interrupted pull resumes where it left off instead of starting over —
+    only what's missing or truncated is re-fetched. ``force`` re-downloads
+    everything regardless.
+    """
     dest.mkdir(parents=True, exist_ok=True)
     if kind == "dataverse":
+        if not force:
+            local = [n for n in names if (dest / n).is_file()]
+            if local:
+                print(f"       verifying {len(local)} local file(s) ...")
+                have = {n for n in local if _is_complete(dest / n, registry[n])}
+                if have:
+                    print(f"       {len(have)} already complete - skipping.")
+                names = [n for n in names if n not in have]
+                if not names:
+                    return
         total = sum(int(registry[n].get("size") or 0) for n in names)
         if total:
             print(f"       ({len(names)} files, {_human(total)} total)")
@@ -215,6 +274,44 @@ def _safe_extract(z: zipfile.ZipFile, dest: Path) -> None:
     z.extractall(dest)
 
 
+def raw_audit(session: str, doi: str | None = None, deep: bool = False,
+              ) -> tuple[list[str], list[str]]:
+    """Compare a session's local ``raw/`` against the archive's file list.
+
+    Returns ``(missing, damaged)``: names the deposit has that aren't on disk,
+    and names whose local copy doesn't match what was published. The default
+    checks presence and size — what an interrupted download breaks — and reads
+    no file contents; *deep* also verifies every MD5 (reads every byte, so
+    roughly 10 s per 10 GB).
+
+    Only the file *list* is fetched, never the data, so this is cheap enough for
+    a pre-flight check. Raises if the deposit can't be read (offline, bad DOI)
+    so callers can degrade to local-only checks instead of reporting a failure.
+    """
+    doi = doi or SESSIONS.get(session)
+    if not doi:
+        raise ValueError(f"no DOI known for session {session!r}")
+    kind, registry, _ctx = discover(doi)
+    zips = {f"{st}.zip" for st in PROCESSED}
+    raw_dir = DATA_ROOT / session / "raw"
+    check = _is_complete if deep else _size_ok
+
+    missing: list[str] = []
+    damaged: list[str] = []
+    for name, rec in registry.items():
+        if name in zips:
+            continue
+        path = raw_dir / name
+        if not path.is_file():
+            missing.append(name)
+        elif isinstance(rec, dict) and not check(path, rec):
+            # pooch registries carry only a hash string, so non-Dataverse
+            # deposits are audited for presence alone — pooch itself
+            # hash-verifies each file as it downloads.
+            damaged.append(name)
+    return sorted(missing), sorted(damaged)
+
+
 def fetch_session(session: str, doi: str, stages: list[str], force: bool,
                   skip_video: bool = False) -> tuple[int, int]:
     """Fetch the requested *stages* of *session*.
@@ -237,7 +334,14 @@ def fetch_session(session: str, doi: str, stages: list[str], force: bool,
     ok = failed = 0
     for stage in stages:
         dest = DATA_ROOT / session / stage
-        if _nonempty(dest) and not force:
+        # `raw` comes from the archive file-by-file, so its completeness is
+        # verifiable against the registry — fall through and let _fetch skip the
+        # files we already have and re-pull only what's missing or truncated. A
+        # blanket "dir is non-empty -> KEEP" here would mistake an interrupted
+        # pull for a finished one and leave the session quietly incomplete.
+        # Processed stages stay coarse: a local dir there may be your own
+        # upstream output (that's the point of local-first), so we keep it.
+        if stage != "raw" and _nonempty(dest) and not force:
             print(f"  KEEP {session}/{stage}: local data present (use --force to re-download).")
             ok += 1
             continue
@@ -254,14 +358,14 @@ def fetch_session(session: str, doi: str, stages: list[str], force: bool,
                     continue
                 note = "  (timestamps/metadata only; skipping video)" if skip_video else ""
                 print(f"  GET  {session}/raw: {len(files)} files{note}")
-                _fetch(kind, ctx, registry, files, dest)
+                _fetch(kind, ctx, registry, files, dest, force=force)
             else:
                 zname = zip_for[stage]
                 if zname not in registry:
                     print(f"  SKIP {session}/{stage}: {zname} not in this deposit yet.")
                     continue
                 print(f"  GET  {session}/{stage}: {zname}")
-                _fetch(kind, ctx, registry, [zname], CACHE)
+                _fetch(kind, ctx, registry, [zname], CACHE, force=force)
                 dest.mkdir(parents=True, exist_ok=True)
                 with zipfile.ZipFile(CACHE / zname) as z:
                     _safe_extract(z, dest)

scripts/verify.py

@@ -4,11 +4,16 @@
 
     python scripts/verify.py
 
-It does not install or download anything - it just checks that each install
-step actually worked and prints one PASS / FAIL / WARN per check, with the exact
-command to fix anything that's red. Send the output (or a screenshot of an
-all-PASS run) to the organizers before the workshop so a broken setup is found
-days early, not in the room.
+It does not install anything and downloads no data - it just checks that each
+install step actually worked and prints one PASS / FAIL / WARN per check, with
+the exact command to fix anything that's red. Send the output (or a screenshot
+of an all-PASS run) to the organizers before the workshop so a broken setup is
+found days early, not in the room.
+
+One check reaches the network: the raw recording is audited against the
+archive's published file list (names and sizes only - metadata, not data), so an
+interrupted download can't pass as a finished one. With no connection that check
+reports WARN and everything else still runs.
 
 Exit code is 0 only if every required check passed.
 """
@@ -35,6 +40,9 @@
 # Teaching-notebook dirs that must hold at least one .ipynb.
 NOTEBOOK_DIRS = ["minisim", "minian", "eztrack"]
 
+# Raw video, which a deliberate `get_data.py --skip-video` pull leaves out.
+_VIDEO_EXTS = {".avi", ".mp4"}
+
 # Files the capstone needs from the prerecorded session.
 SESSION = REPO_ROOT / "data" / "sessions" / "prerecorded"
 DATA_INPUTS = [
@@ -117,6 +125,62 @@ def check_notebooks() -> None:
                "" if n else "python scripts/fetch_notebooks.py  (committed copies should already be present - check your clone).")
 
 
+def check_raw_complete() -> None:
+    """Audit the local ``raw/`` against the archive's file list.
+
+    ``check_data`` only asks whether the stage dirs are non-empty, which a
+    half-finished download satisfies — so on its own it would greenlight a
+    truncated recording and let the failure surface later, mid-notebook. This
+    compares what's on disk against what the deposit publishes (names + sizes;
+    no file contents read, no data downloaded). Offline it degrades to a WARN
+    rather than failing the gate.
+    """
+    raw = SESSION / "raw"
+    if not _nonempty_dir(raw):
+        record("WARN", "raw recording complete",
+               "No raw data yet - needed for Minian (step 2) and eztrack (step 4): "
+               "python scripts/get_data.py --what raw")
+        return
+
+    sys.path.insert(0, str(REPO_ROOT / "scripts"))
+    try:
+        import get_data
+
+        missing, damaged = get_data.raw_audit("prerecorded")
+    except Exception as exc:
+        record("WARN", "raw recording complete (unverified)",
+               f"Could not read the archive's file list ({type(exc).__name__}) - offline? "
+               f"Local files are present but completeness could not be confirmed.")
+        return
+
+    def _some(names: list[str]) -> str:
+        return ", ".join(names[:3]) + (" ..." if len(names) > 3 else "")
+
+    if damaged:
+        record("FAIL", f"raw recording complete ({len(damaged)} file(s) wrong size)",
+               f"Truncated: {_some(damaged)}. Re-run `python scripts/get_data.py --what raw` "
+               f"- it re-fetches only the damaged files.")
+        return
+
+    videos = [n for n in missing if Path(n).suffix.lower() in _VIDEO_EXTS]
+    others = [n for n in missing if n not in set(videos)]
+    if others:
+        record("FAIL", f"raw recording complete ({len(others)} file(s) missing)",
+               f"Missing: {_some(others)}. Run: python scripts/get_data.py --what raw")
+    elif videos and not any(p.suffix.lower() in _VIDEO_EXTS for p in raw.iterdir()):
+        # No video at all locally: a deliberate --skip-video pull. Fine for the
+        # capstone, so not a failure - just say what it does not cover.
+        record("PASS", "raw recording complete (timestamps only - video skipped)",
+               "Enough for the capstone. To run Minian/eztrack on real video: "
+               "python scripts/get_data.py --what raw")
+    elif videos:
+        record("WARN", f"raw recording complete ({len(videos)} video(s) missing)",
+               f"Some video present but not all - an interrupted pull. Missing: {_some(videos)}. "
+               f"Re-run: python scripts/get_data.py --what raw")
+    else:
+        record("PASS", "raw recording complete (all files match the archive)")
+
+
 def check_data() -> None:
     missing = [p for p in DATA_INPUTS
                if not (_nonempty_dir(p) if p.suffix == "" else p.exists())]
@@ -132,7 +196,7 @@ def main() -> int:
     print("Workshop install self-check\n" + "=" * 27)
     print(f"repo: {REPO_ROOT}\npython: {sys.executable}\n")
     for check in (check_python, check_venv, check_ffmpeg, check_tools,
-                  check_kernel, check_notebooks, check_data):
+                  check_kernel, check_notebooks, check_data, check_raw_complete):
         try:
             check()
         except Exception as exc:  # never let the checker itself crash the gate