diff --git a/INSTALL.md b/INSTALL.md index 73c7a96..61abf0f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -20,6 +20,25 @@ venv with no conflicts. You need three things that pip cannot install for you: **Python**, **git**, and **ffmpeg**. (pip and `venv` come bundled with Python.) +> **macOS — install Homebrew first.** Every `brew` command below needs it, and a +> fresh Mac doesn't have it. Run this in **Terminal directly** (it prompts you +> interactively, so it can't be pasted into another tool): +> +> ```bash +> /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +> ``` +> +> It asks for your login password because it needs **administrator rights**. If +> it stops with `Need sudo access on macOS` your account isn't an admin — that's +> common on lab or managed machines. Either ask whoever administers it, or skip +> Homebrew entirely and use the [python.org installer](https://www.python.org/downloads/) +> plus a prebuilt [ffmpeg binary](https://ffmpeg.org/download.html#build-mac) +> (git comes from `xcode-select --install`, which needs no admin rights). +> +> When it finishes it may print a `Next steps` block asking you to add `brew` to +> your PATH — run those two commands, then **open a new terminal** so `brew` is +> found. + ### Python 3.11–3.13 CaMAP requires Python in this range; 3.12 is recommended. **3.14 (the current diff --git a/scripts/get_data.py b/scripts/get_data.py index c5be7f1..be80ec5 100644 --- a/scripts/get_data.py +++ b/scripts/get_data.py @@ -32,6 +32,11 @@ upstream step) is kept, and only missing stages are downloaded. A stage that isn't in the deposit yet is simply skipped. +It also **resumes**: within ``raw``, each file is checked against the archive's +size and MD5, so an interrupted multi-GB pull re-fetches only what's missing or +half-written. Re-running after a dropped connection is safe and cheap — it will +not mistake a partial download for a complete one. + The **live** dataset is published *during* the workshop, so its DOI isn't baked in — pass it at fetch time with ``--doi`` (no code edit, no ``git pull``): @@ -157,6 +162,43 @@ def _human(n: float) -> str: n /= 1024 +def _md5(path: Path) -> str: + """MD5 of *path*, read in chunks (files here run to hundreds of MB).""" + h = hashlib.md5() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(_CHUNK), b""): + h.update(chunk) + return h.hexdigest() + + +def _size_ok(path: Path, rec: dict) -> bool: + """True if *path* exists and matches the size the archive published. + + Cheap (a ``stat``, no reading) and catches the truncated-by-Ctrl-C case, + which is how an interrupted download actually fails. + """ + if not path.is_file(): + return False + want_size = int(rec.get("size") or 0) + return path.stat().st_size == want_size if want_size else True + + +def _is_complete(path: Path, rec: dict) -> bool: + """True if *path* already holds the archive's copy of *rec*. + + Size first, then the MD5 the archive published — which also catches the + right-length-but-wrong-bytes case. Without this, a file left half-written by + an interrupted run is indistinguishable from a finished one. + """ + if not _size_ok(path, rec): + return False + want_md5 = rec.get("md5") + if want_md5: + return _md5(path) == want_md5 + # No hash published — size is all we can go on. + return bool(int(rec.get("size") or 0)) + + def _dataverse_download(server: str, rec: dict, dest_path: Path, label: str = "") -> None: """Stream a Dataverse datafile to *dest_path*, verifying its MD5. @@ -185,10 +227,27 @@ def _dataverse_download(server: str, rec: dict, dest_path: Path, label: str = "" print(f" {label} done ({_human(done)})") -def _fetch(kind: str, ctx: str, registry: dict, names: list[str], dest: Path) -> None: - """Download *names* (a subset of *registry*) into *dest*, verified by hash.""" +def _fetch(kind: str, ctx: str, registry: dict, names: list[str], dest: Path, + force: bool = False) -> None: + """Download *names* (a subset of *registry*) into *dest*, verified by hash. + + Files already present and matching the archive (size + MD5) are skipped, so + an interrupted pull resumes where it left off instead of starting over — + only what's missing or truncated is re-fetched. ``force`` re-downloads + everything regardless. + """ dest.mkdir(parents=True, exist_ok=True) if kind == "dataverse": + if not force: + local = [n for n in names if (dest / n).is_file()] + if local: + print(f" verifying {len(local)} local file(s) ...") + have = {n for n in local if _is_complete(dest / n, registry[n])} + if have: + print(f" {len(have)} already complete - skipping.") + names = [n for n in names if n not in have] + if not names: + return total = sum(int(registry[n].get("size") or 0) for n in names) if total: print(f" ({len(names)} files, {_human(total)} total)") @@ -215,6 +274,44 @@ def _safe_extract(z: zipfile.ZipFile, dest: Path) -> None: z.extractall(dest) +def raw_audit(session: str, doi: str | None = None, deep: bool = False, + ) -> tuple[list[str], list[str]]: + """Compare a session's local ``raw/`` against the archive's file list. + + Returns ``(missing, damaged)``: names the deposit has that aren't on disk, + and names whose local copy doesn't match what was published. The default + checks presence and size — what an interrupted download breaks — and reads + no file contents; *deep* also verifies every MD5 (reads every byte, so + roughly 10 s per 10 GB). + + Only the file *list* is fetched, never the data, so this is cheap enough for + a pre-flight check. Raises if the deposit can't be read (offline, bad DOI) + so callers can degrade to local-only checks instead of reporting a failure. + """ + doi = doi or SESSIONS.get(session) + if not doi: + raise ValueError(f"no DOI known for session {session!r}") + kind, registry, _ctx = discover(doi) + zips = {f"{st}.zip" for st in PROCESSED} + raw_dir = DATA_ROOT / session / "raw" + check = _is_complete if deep else _size_ok + + missing: list[str] = [] + damaged: list[str] = [] + for name, rec in registry.items(): + if name in zips: + continue + path = raw_dir / name + if not path.is_file(): + missing.append(name) + elif isinstance(rec, dict) and not check(path, rec): + # pooch registries carry only a hash string, so non-Dataverse + # deposits are audited for presence alone — pooch itself + # hash-verifies each file as it downloads. + damaged.append(name) + return sorted(missing), sorted(damaged) + + def fetch_session(session: str, doi: str, stages: list[str], force: bool, skip_video: bool = False) -> tuple[int, int]: """Fetch the requested *stages* of *session*. @@ -237,7 +334,14 @@ def fetch_session(session: str, doi: str, stages: list[str], force: bool, ok = failed = 0 for stage in stages: dest = DATA_ROOT / session / stage - if _nonempty(dest) and not force: + # `raw` comes from the archive file-by-file, so its completeness is + # verifiable against the registry — fall through and let _fetch skip the + # files we already have and re-pull only what's missing or truncated. A + # blanket "dir is non-empty -> KEEP" here would mistake an interrupted + # pull for a finished one and leave the session quietly incomplete. + # Processed stages stay coarse: a local dir there may be your own + # upstream output (that's the point of local-first), so we keep it. + if stage != "raw" and _nonempty(dest) and not force: print(f" KEEP {session}/{stage}: local data present (use --force to re-download).") ok += 1 continue @@ -254,14 +358,14 @@ def fetch_session(session: str, doi: str, stages: list[str], force: bool, continue note = " (timestamps/metadata only; skipping video)" if skip_video else "" print(f" GET {session}/raw: {len(files)} files{note}") - _fetch(kind, ctx, registry, files, dest) + _fetch(kind, ctx, registry, files, dest, force=force) else: zname = zip_for[stage] if zname not in registry: print(f" SKIP {session}/{stage}: {zname} not in this deposit yet.") continue print(f" GET {session}/{stage}: {zname}") - _fetch(kind, ctx, registry, [zname], CACHE) + _fetch(kind, ctx, registry, [zname], CACHE, force=force) dest.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(CACHE / zname) as z: _safe_extract(z, dest) diff --git a/scripts/verify.py b/scripts/verify.py index 86039ba..e73c99e 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -4,11 +4,16 @@ python scripts/verify.py -It does not install or download anything - it just checks that each install -step actually worked and prints one PASS / FAIL / WARN per check, with the exact -command to fix anything that's red. Send the output (or a screenshot of an -all-PASS run) to the organizers before the workshop so a broken setup is found -days early, not in the room. +It does not install anything and downloads no data - it just checks that each +install step actually worked and prints one PASS / FAIL / WARN per check, with +the exact command to fix anything that's red. Send the output (or a screenshot +of an all-PASS run) to the organizers before the workshop so a broken setup is +found days early, not in the room. + +One check reaches the network: the raw recording is audited against the +archive's published file list (names and sizes only - metadata, not data), so an +interrupted download can't pass as a finished one. With no connection that check +reports WARN and everything else still runs. Exit code is 0 only if every required check passed. """ @@ -35,6 +40,9 @@ # Teaching-notebook dirs that must hold at least one .ipynb. NOTEBOOK_DIRS = ["minisim", "minian", "eztrack"] +# Raw video, which a deliberate `get_data.py --skip-video` pull leaves out. +_VIDEO_EXTS = {".avi", ".mp4"} + # Files the capstone needs from the prerecorded session. SESSION = REPO_ROOT / "data" / "sessions" / "prerecorded" DATA_INPUTS = [ @@ -117,6 +125,62 @@ def check_notebooks() -> None: "" if n else "python scripts/fetch_notebooks.py (committed copies should already be present - check your clone).") +def check_raw_complete() -> None: + """Audit the local ``raw/`` against the archive's file list. + + ``check_data`` only asks whether the stage dirs are non-empty, which a + half-finished download satisfies — so on its own it would greenlight a + truncated recording and let the failure surface later, mid-notebook. This + compares what's on disk against what the deposit publishes (names + sizes; + no file contents read, no data downloaded). Offline it degrades to a WARN + rather than failing the gate. + """ + raw = SESSION / "raw" + if not _nonempty_dir(raw): + record("WARN", "raw recording complete", + "No raw data yet - needed for Minian (step 2) and eztrack (step 4): " + "python scripts/get_data.py --what raw") + return + + sys.path.insert(0, str(REPO_ROOT / "scripts")) + try: + import get_data + + missing, damaged = get_data.raw_audit("prerecorded") + except Exception as exc: + record("WARN", "raw recording complete (unverified)", + f"Could not read the archive's file list ({type(exc).__name__}) - offline? " + f"Local files are present but completeness could not be confirmed.") + return + + def _some(names: list[str]) -> str: + return ", ".join(names[:3]) + (" ..." if len(names) > 3 else "") + + if damaged: + record("FAIL", f"raw recording complete ({len(damaged)} file(s) wrong size)", + f"Truncated: {_some(damaged)}. Re-run `python scripts/get_data.py --what raw` " + f"- it re-fetches only the damaged files.") + return + + videos = [n for n in missing if Path(n).suffix.lower() in _VIDEO_EXTS] + others = [n for n in missing if n not in set(videos)] + if others: + record("FAIL", f"raw recording complete ({len(others)} file(s) missing)", + f"Missing: {_some(others)}. Run: python scripts/get_data.py --what raw") + elif videos and not any(p.suffix.lower() in _VIDEO_EXTS for p in raw.iterdir()): + # No video at all locally: a deliberate --skip-video pull. Fine for the + # capstone, so not a failure - just say what it does not cover. + record("PASS", "raw recording complete (timestamps only - video skipped)", + "Enough for the capstone. To run Minian/eztrack on real video: " + "python scripts/get_data.py --what raw") + elif videos: + record("WARN", f"raw recording complete ({len(videos)} video(s) missing)", + f"Some video present but not all - an interrupted pull. Missing: {_some(videos)}. " + f"Re-run: python scripts/get_data.py --what raw") + else: + record("PASS", "raw recording complete (all files match the archive)") + + def check_data() -> None: missing = [p for p in DATA_INPUTS if not (_nonempty_dir(p) if p.suffix == "" else p.exists())] @@ -132,7 +196,7 @@ def main() -> int: print("Workshop install self-check\n" + "=" * 27) print(f"repo: {REPO_ROOT}\npython: {sys.executable}\n") for check in (check_python, check_venv, check_ffmpeg, check_tools, - check_kernel, check_notebooks, check_data): + check_kernel, check_notebooks, check_data, check_raw_complete): try: check() except Exception as exc: # never let the checker itself crash the gate