diff --git a/.claude/agents/candidate-screener.md b/.claude/agents/candidate-screener.md index aae9df0d..749a78c1 100644 --- a/.claude/agents/candidate-screener.md +++ b/.claude/agents/candidate-screener.md @@ -22,7 +22,7 @@ Follow `.claude/skills/screen-candidates/SKILL.md`, steps 3 to 6, for each recor 4. Write each non-rejected record through the store API right after reloading it (`records.md`): links, metadata only where the source states it, `priority`, `AI (UNVERIFIED)` in `checked_by`, `AI-Filled (Verify)` in `tags`, the disclaimer and the screening block at the top of the comments, every existing note kept. Skip a record another - curator has claimed. + curator holds on the claims board (`claims check `; the main session claims the batch before it starts). Never commit, never build, never delete a comment, never name company code repositories, never send personal data to an outside service. Cap the threads of any Python run (`OMP_NUM_THREADS=2` and the like). diff --git a/.claude/hooks/claims_hook.py b/.claude/hooks/claims_hook.py new file mode 100644 index 00000000..d6ca30c4 --- /dev/null +++ b/.claude/hooks/claims_hook.py @@ -0,0 +1,116 @@ +"""Claude Code hook: tell the agent who holds which dataset (the live claims board), and warn before it edits one. + +Wired in ``.claude/settings.json``: + +* ``SessionStart``: fetches the board and adds it to the session's context: the other curators' claims, your own, + and yours that expire within two days. +* ``PreToolUse`` on ``Edit|Write|NotebookEdit``: when the file belongs to a dataset (``curation/records/.md`` + or ``datasets/_dev///...``) that another curator holds, shows a warning to the user and to the agent. + It never allows or denies anything: claims are advisory, and the normal permission flow goes on. + +The board is re-fetched at most every 60 s (else the cache of the last fetch is used). The hook fails open: without +the package, a curator name, the network or a board it prints nothing and exits 0. +""" + +from __future__ import annotations + +import json +import re +import sys + +PATTERNS = ( + re.compile(r"/curation/records/([^/]+)\.md$"), + re.compile(r"/datasets/_dev/[^/]+/([^/_.][^/]*)/"), +) +MAX_NAMES = 60 + + +def dataset_of(path: str) -> str | None: + """The dataset a file belongs to, or ``None``.""" + path = path.replace("\\", "/") + for pattern in PATTERNS: + match = pattern.search(path) + if match: + return match.group(1) + return None + + +def _names(items: list) -> str: + names = [c.name for c in items] + head = ", ".join(names[:MAX_NAMES]) + return head + (f" and {len(names) - MAX_NAMES} more" if len(names) > MAX_NAMES else "") + + +def session_start(claims) -> dict: + """The board as context for the agent.""" + board = claims.load(max_age=0) + me = claims.configured_curator() + lines = [ + f"Dataset claims (the live board on {board.remote}/{board.branch}, fetched at session start). A claim covers " + "a dataset's curation record and its v2 folder. Do not work on a dataset another curator holds; ask the user " + "first. Commands: `data-foundry-curation claims list|claim|release|renew|check`.", + ] + if not me: + lines.append(f"No curator name is set: `git config {claims.CURATOR_GIT_KEY} ` before claiming.") + others = {h: items for h, items in board.by_holder().items() if h != me} + mine = [c for c in board.claims.values() if c.holder == me] + if others: + lines.append("Claimed by others: " + "; ".join(f"{h} ({len(i)}): {_names(i)}" for h, i in others.items())) + else: + lines.append("No other curator holds a claim.") + if me: + lines.append(f"Yours ({len(mine)}): {_names(sorted(mine, key=lambda c: c.name))}" if mine else "Yours: none.") + soon = [c for c in mine if c.seconds_left() < 2 * 86400] + if soon: + lines.append(f"Yours expiring within 2 days (renew with `claims renew`): {_names(soon)}") + return {"hookSpecificOutput": {"hookEventName": "SessionStart", "additionalContext": "\n".join(lines)}} + + +def pre_tool_use(claims, payload: dict) -> dict | None: + """A warning when the edited file belongs to a dataset someone else holds.""" + name = dataset_of(str((payload.get("tool_input") or {}).get("file_path") or "")) + if not name: + return None + me = claims.configured_curator() + if not me: + return None + try: + board = claims.load(max_age=60) + except claims.ClaimsError: + board = claims.cached_board() + claim = board.held_by_other(name, me) if board is not None else None + if claim is None: + return None + text = f"Claim warning: {name} is claimed by {claim.describe()}." + return { + "systemMessage": text, + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": f"{text} Another curator is working on this dataset; stop and ask the user before " + "changing it further.", + }, + } + + +def main() -> int: + """Read the hook's JSON from stdin and print the answer, if any.""" + try: + payload = json.loads(sys.stdin.read() or "{}") + from data_foundry.curation import claims # noqa: PLC0415 - fail open without the package + + event = payload.get("hook_event_name") + if event == "SessionStart": + out = session_start(claims) + elif event == "PreToolUse": + out = pre_tool_use(claims, payload) + else: + out = None + except Exception: # noqa: BLE001 - a hook must never get in the way + return 0 + if out: + print(json.dumps(out)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..5a2d327f --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,27 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "test -x \"$CLAUDE_PROJECT_DIR/.venv/bin/python\" && \"$CLAUDE_PROJECT_DIR/.venv/bin/python\" \"$CLAUDE_PROJECT_DIR/.claude/hooks/claims_hook.py\" || true", + "timeout": 30 + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "Edit|Write|NotebookEdit", + "hooks": [ + { + "type": "command", + "command": "test -x \"$CLAUDE_PROJECT_DIR/.venv/bin/python\" && \"$CLAUDE_PROJECT_DIR/.venv/bin/python\" \"$CLAUDE_PROJECT_DIR/.claude/hooks/claims_hook.py\" || true", + "timeout": 30 + } + ] + } + ] + } +} diff --git a/.claude/skills/README.md b/.claude/skills/README.md index 318b1056..8546d4f6 100644 --- a/.claude/skills/README.md +++ b/.claude/skills/README.md @@ -114,6 +114,17 @@ A full panel costs on the order of a million tokens per dataset; for a re-check smaller one. In stage 1, two agents help: a screener for batches and a reject checker that argues against every quick `No`. +## Who works on what + +A claim says "I am working on this dataset now" (its record and its folder). Claims live on the branch `claims` of +the repository, pushed directly, so everyone sees them within seconds: +[the board on GitHub](https://github.com/TabArena/data-foundry/blob/claims/README.md), the Claimed by column of the +dashboard, and the agents (a hook lists the board at the start of each session and warns before an agent edits a +dataset someone else holds). The commands claim before they work and skip what someone else holds. Set your name once +with `git config data-foundry.curator `; to claim by hand, pick your name in the dashboard's Claimed by cell or +run `data-foundry-curation claims claim `. The details: +[claims.md](knowledge-data-foundry/references/claims.md). + ## Where the rules are * [knowledge-curation](knowledge-curation/SKILL.md): how we decide, independent of the tooling. Start with diff --git a/.claude/skills/add-dataset/SKILL.md b/.claude/skills/add-dataset/SKILL.md index eed7a9a1..c6e0f745 100644 --- a/.claude/skills/add-dataset/SKILL.md +++ b/.claude/skills/add-dataset/SKILL.md @@ -25,7 +25,7 @@ Two modes: | Step | Read | |---|---| | all | [principles.md](../knowledge-curation/references/principles.md): rank, strengths, the remedy ladder, the judgment record | -| 0 | [records.md](../knowledge-data-foundry/references/records.md) (claims, pointers); for a batch [agent_runs.md](../knowledge-data-foundry/references/agent_runs.md) | +| 0 | [claims.md](../knowledge-data-foundry/references/claims.md), [records.md](../knowledge-data-foundry/references/records.md) (pointers); for a batch [agent_runs.md](../knowledge-data-foundry/references/agent_runs.md) | | 1 | [use_case.md](../knowledge-curation/references/use_case.md), [provenance.md](../knowledge-curation/references/provenance.md), [metrics.md](../knowledge-curation/references/metrics.md) (MET-1 to MET-3) | | 1-3 | [definitions.md](../knowledge-data-foundry/references/definitions.md): the folder, the class, the field mapping, the recipe, the split declarations, the reference definitions | | 3 | [cleaning_features.md](../knowledge-curation/references/cleaning_features.md), [splits.md](../knowledge-curation/references/splits.md), [metrics.md](../knowledge-curation/references/metrics.md) (MET-11 to MET-13) and definitions.md, "Metrics" | @@ -52,8 +52,10 @@ errors, and `dataset build` refuses to save while any remains. Read `curation/records/.md`: the front matter and the comments, above all the screening block (the source, the use case, the regime, the risks). Verify its claims as you go instead of redoing its research; a measurement on the data supersedes what screening read from the page. No record: stop and point to -`/screen-candidates`. Never work on a record another curator has claimed; the records you take get `WIP (DF)` and the -curator's name in `claimed_by` first, and the claim goes to `main` in its own small PR. +`/screen-candidates`. Claim the records first, in the curator's name: `claims claim --stage add --note +"add-dataset"` ([claims.md](../knowledge-data-foundry/references/claims.md)); it pushes the claim to the board and +writes `WIP (DF)` and `claimed_by` into the records. Never work on a dataset another curator holds: a name the command +skips is left out and reported. A batch (several names or a dashboard filter) runs one `dataset-builder` agent per source unit for steps 0-6 ([agent_runs.md](../knowledge-data-foundry/references/agent_runs.md)); the main session then runs the panel (step 7) diff --git a/.claude/skills/knowledge-data-foundry/SKILL.md b/.claude/skills/knowledge-data-foundry/SKILL.md index 08f7273f..4f7dba09 100644 --- a/.claude/skills/knowledge-data-foundry/SKILL.md +++ b/.claude/skills/knowledge-data-foundry/SKILL.md @@ -16,6 +16,7 @@ implements. The overview for people is the [skills README](../README.md). |---|---|---| | The pipeline: the stages and commands, the record after each step, the queues, which file is the truth for what, the agents | [pipeline.md](references/pipeline.md) | finding where a record stands or what comes next | | The curation records: editing through the store API, comment lines (`CC`, `Assessment`, `Counter-argument`, `Open issues`), AI triage, fields and vocabulary, claims, duplicates and retirements, the review queue, the definition pointers | [records.md](references/records.md) | before reading or writing any record | +| Claims: the live board of who works on which dataset (a git branch), the `claims` commands, the hook, the records' mirror | [claims.md](references/claims.md) | before working on records or folders, and when a claim warning shows | | The dashboard: running and restarting it, its filters, pills, pins and links | [dashboard.md](references/dashboard.md) | starting it or explaining it to a curator | | The v2 definition: the folder, `dataset.py`, the base class, `task_summary`, field mapping, `curation_comments`, the `_clean` recipe, `Grouping` / `Temporal` / `TemporalSplits`, `_1m` versions, traps, bundle-check slugs, `_decisions`, recording decisions, reference definitions, the harness | [definitions.md](references/definitions.md) | writing or reviewing a `dataset.py` | | Running `dataset check` and the probes, and the checks only Data Foundry has (pointers, template, README) | [checks_and_probes.md](references/checks_and_probes.md) | verifying a definition | diff --git a/.claude/skills/knowledge-data-foundry/references/agent_runs.md b/.claude/skills/knowledge-data-foundry/references/agent_runs.md index a32aa710..7be36a7a 100644 --- a/.claude/skills/knowledge-data-foundry/references/agent_runs.md +++ b/.claude/skills/knowledge-data-foundry/references/agent_runs.md @@ -6,9 +6,10 @@ How a batch of records is worked by parallel agents on this machine, and the lim ## A batch of records When the curator names several records or a dashboard filter (for example "every record with `suggestion = Yes`, -`data_foundry_status` without `DF: Yes`, `claimed_by` empty, `checked_by` without `AI (UNVERIFIED)`"): +`data_foundry_status` without `DF: Yes`, unclaimed, `checked_by` without `AI (UNVERIFIED)`"): -1. List the matching records first and skip any claimed by another curator. +1. List the matching records first, then claim them in one call (`claims claim --note ""`, + [claims.md](claims.md)) and work only on the ones it claimed: a skipped name is held by another curator. 2. Group them into units, one per source (records that share a download are one unit). 3. Per unit, one agent does the work with the data in hand (download, definition, `dataset check`, the probes), a second agent verifies the result against diff --git a/.claude/skills/knowledge-data-foundry/references/claims.md b/.claude/skills/knowledge-data-foundry/references/claims.md new file mode 100644 index 00000000..d637f1b2 --- /dev/null +++ b/.claude/skills/knowledge-data-foundry/references/claims.md @@ -0,0 +1,98 @@ +# Claims: who works on which dataset + +Several curators, each running several agents, work on the same records and folders, and git shows their work only +when it reaches `main`. A claim says "I am working on this dataset now", and every curator and agent sees it within +seconds. A claim covers one dataset: its record `curation/records/.md` and its folder +`datasets/_dev/tabarena-v0pt2//` with `REVIEW.md`. Shared files (CHANGELOG, the README table, the vocabulary) +are never claimed; their conflicts are resolved when merging. + +## How it works + +* **The board** is the branch `claims` of the repository's remote. It holds one file `claims/.yaml` per claimed + dataset (holder, stage, since, until, note) and a `README.md` table. That table is the team's view on GitHub: + . +* **A claim is a commit pushed straight to that branch**, with no pull request and no CI. Two claims of the same + dataset at the same moment cannot both land: git rejects the second push, and the command fetches again, sees the + claim and skips the dataset. Nothing touches your working tree or your branch. +* **Login is the repository's.** Whoever can push to the repository can claim, and whoever can read it sees the + board. Nothing else to sign up for or host. The code (`src/data_foundry/curation/claims.py`) works with any git + remote: `$DATA_FOUNDRY_CLAIMS_REMOTE` (default `origin`) and `$DATA_FOUNDRY_CLAIMS_BRANCH` (default `claims`). +* **Stages:** + * `triage`: screening, a check, or settling an open verdict; + * `add`: building the v2 folder; + * `review`: the curator's review of an added dataset. +* **Expiry.** A claim lasts 7 days unless renewed. After that it shows as expired, and anyone may take it over with + `--steal`; the board's log names the previous holder. +* **Claims are advisory.** Nothing refuses a write. Warnings appear in three places: + * `save_record` warns (`ClaimWarning`) when it writes a record another curator holds; + * the Claude Code hook warns before an agent edits such a file; + * the dashboard shows the holder. + + A warning means: stop and ask the curator, or the holder. +* **The records mirror the board.** `claimed_by` and the WIP value (`WIP (Triage)` for triage or an open verdict, + `WIP (DF)` for work on an accepted dataset) are written by `claims claim` and `claims release`. They travel with the + work branch and reach `main` with the work. The board is the truth: the old rule of pushing the claim to `main` in + its own PR is gone. + +## Setup, once per curator + +```bash +git config data-foundry.curator # a claimed_by value of curation/vocabularies.yaml +.venv/bin/python -m data_foundry.curation.cli claims list +``` + +`claims list` works when `git fetch` from the remote works. On a machine whose ssh agent went stale, prefix the +commands with `env SSH_AUTH_SOCK=`. The git calls never prompt: a missing credential fails at once with a hint. + +## Commands + +`data-foundry-curation claims ...` (or `.venv/bin/python -m data_foundry.curation.cli claims ...`): + +| Command | What it does | +|---|---| +| `list [--mine\|--others] [--json] [--offline]` | Fetches and shows the board. | +| `check ` | Says per dataset: free, yours, or held by someone (always exit 0). | +| `claim [--stage S] [--note TEXT] [--ttl DAYS] [--steal]` | Claims what is free and skips the rest. Exit 1 when anything was skipped. Stage default: `add` for an accepted record, else `triage`. A name with no record is refused unless `--allow-unknown`. Claiming your own claim again renews it. | +| `release [--force]` | Releases your claims; `--force` releases someone else's (logged). | +| `renew [] [--ttl DAYS]` | Extends your claims (all of them by default). | +| `sync [--check]` | Makes every record's `claimed_by` and WIP value match the board. | +| `import-records` | One-time: moves the claims the records carry onto the board. | +| `init [--pages-dispatch]` | Creates the branch. `--pages-dispatch` adds `.github/workflows/claims-to-pages.yaml` to it, so each claim rebuilds the public dashboard. | + +From Python: `claims.claim(...)`, `claims.release(...)`, `claims.load(max_age=30)`, `claims.cached_board()`. + +## In the workflows + +* **Before working on records** (screening, adding, a re-check, a review round), run `claims claim --note + ""` once for the whole batch. Work only on the names it claimed. A skipped name belongs to someone else: leave + it out and report it. +* **For a long job**, `claims renew`. +* **When the work reaches `main`, or is dropped,** `claims release `. +* **Batch filters** use the board (`claims list --json`), not the record's `claimed_by`, which can lag behind. +* **Hand edits:** the dashboard's Claimed by column shows the board. Pick your own name in a cell to claim, empty it to + release. The πŸ”’ count in the bar says when the board was fetched. + +## The Claude Code hook + +`.claude/settings.json` runs `.claude/hooks/claims_hook.py` with the repository's `.venv` at two points: + +* **At session start**, it adds the board to the agent's context: the other curators' claims, yours, and yours that + expire within 2 days. +* **Before each Edit, Write or NotebookEdit** of a claimed dataset's record or folder, it shows a warning when another + curator holds the dataset. + +It never allows or denies a tool call. It re-fetches the board at most once a minute, and it does nothing without the +package, a curator name or the network. + +## The public dashboard + +The GitHub Pages workflow fetches the board before it builds, so the read-only dashboard's Claimed by column shows +the claims. With `claims init --pages-dispatch`, each push to the board dispatches the Pages workflow on `main`, and +the page follows within a minute or two. Without it, the page updates with the next push to `main`. + +## When something is off + +* **A record says `claimed_by` but the board does not** (or the reverse): `claims sync --check` lists the drift and + `claims sync` fixes the records. +* **"The claims branch kept moving"**: five pushes in a row lost a race. Run the command again. +* **The board's log** shows every claim and release: `git log origin/claims`. diff --git a/.claude/skills/knowledge-data-foundry/references/dashboard.md b/.claude/skills/knowledge-data-foundry/references/dashboard.md index 8913aa0f..6afbfeb0 100644 --- a/.claude/skills/knowledge-data-foundry/references/dashboard.md +++ b/.claude/skills/knowledge-data-foundry/references/dashboard.md @@ -37,15 +37,17 @@ commit, PR) and is not hardened for multi-user or public use. even when the AI already filled a Yes/No suggestion, and a `DF: Yes` row shows βœ“ whatever its verdict). Click the column to sort by it, or the funnel in its header to filter: each status cycles **βœ“ require, βœ• exclude, off**, so "hide everything rejected" is one click. A claimed row's - status tooltip ends with "claimed by ". + status tooltip ends with "claimed by (, until )" and the claim's note. * **Priority** (next to *Suggestion*) is the record's `priority`, 1 to 5 as a chip (the darker, the sooner a curator should act on the record: verify the screening, or review what `/add-dataset` added), set by `/screen-candidates` and again by `/add-dataset`; pick another value or *(none)* from the cell's list. Click the header to sort: records without a priority sort last in both directions. Its filter popover lists 1 to 5 and *(empty)*, so `#hf.priority=+5|+4` shows the two top priorities. -* **Claimed by** (next to *Data Foundry*) names who holds a WIP claim. Its filter on *(empty)* - (`#hf.claimed_by=+βˆ…`) shows the records nobody works on; add your name for "free or mine" - (`#hf.claimed_by=+βˆ…|+`). +* **Claimed by** (next to *Data Foundry*) shows the live claims board ([claims.md](claims.md)), which the server + fetches every 20 s; the πŸ”’ count in the bar says when, and the error if the fetch failed. Pick your own name in a + cell to claim the dataset, empty the cell to release it (both push to the board and update the record). Its filter + on *(empty)* (`#hf.claimed_by=+βˆ…`) shows the datasets nobody works on; add your name for "free or mine" + (`#hf.claimed_by=+βˆ…|+`). The server needs your name: `git config data-foundry.curator `. * **Column filters combine with AND and NOT.** Every dropdown column's header has a filter button opening a popover that lists all its values (vocabulary βˆͺ values actually present βˆͺ **(empty)**, each with a count; ⚠ marks a value missing from `vocabularies.yaml`). Clicking a diff --git a/.claude/skills/knowledge-data-foundry/references/pipeline.md b/.claude/skills/knowledge-data-foundry/references/pipeline.md index f30fee41..2fcc27aa 100644 --- a/.claude/skills/knowledge-data-foundry/references/pipeline.md +++ b/.claude/skills/knowledge-data-foundry/references/pipeline.md @@ -22,10 +22,10 @@ belong?") needs no command: answer it from knowledge-curation. | New candidate | empty | empty | empty | none | none | none | | Screened by the AI | `TBD -> Yes`, `TBD -> 2nd Tier` or `No` | 1-5 (none on `No`) | `AI (UNVERIFIED)` | none | none | the disclaimer and a `**Screening (AI, ):**` block | | Verified by the curator | `Yes` (accepted for adding), `TBD -> Yes` (later) or `No` | as the curator leaves it | the curator | none | none | `CC` lines for decisions given in chat | -| Taken for adding | `Yes` | | | `WIP (DF)` and `claimed_by` (on `main` first) | none | | +| Taken for adding | `Yes` | | | on the claims board; `WIP (DF)` and `claimed_by` mirrored | none | | | Added by the AI | `Yes` | 1-5 again, from what was measured | | kept | `v2_path`, the folder with `REVIEW.md` | `**Assessment (AI, ):**` | | Reviewed by the curator | `Yes`, or `No` / `No (Retired)` when retired | | | kept until the work reaches `main` | issues leave `REVIEW.md` as they settle; a retired folder goes | `CC` lines, `Exception to` lines | -| On `main` | `Yes` | | | `DF: Yes`, claim cleared | | | +| On `main` | `Yes` | | | `DF: Yes`, claim released | | | | Built (on request) | | | | | the README's build record, a CHANGELOG entry | | `AI (UNVERIFIED)` keeps a record in the review queue until a curator verifies it ([records.md](records.md)). The AI @@ -36,8 +36,9 @@ never sets a plain `Yes` while screening; only a curator does. * **To screen:** records without a verdict, or a filter or list the curator names. * **Screening review:** the dashboard's Screening pill (AI-unverified records whose verdict is not `No`, highest priority first); confirmed rejections are found with the status funnel on πŸ€– and βœ—. -* **To add:** `suggestion` `Yes`, no `DF: Yes`, no claim, not AI-unverified - (`#hf.suggestion=+Yes&hf.data_foundry_status=-DF: Yes&hf.claimed_by=+βˆ…&hf.checked_by=-AI (UNVERIFIED)`). +* **To add:** `suggestion` `Yes`, no `DF: Yes`, no claim on the board, not AI-unverified + (`#hf.suggestion=+Yes&hf.data_foundry_status=-DF: Yes&hf.claimed_by=+βˆ…&hf.checked_by=-AI (UNVERIFIED)`; the + dashboard's Claimed by column shows the board, [claims.md](claims.md)). * **To review:** the folders of the working copy that have a `REVIEW.md`, and records with an `**Open issues (AI)**` block ([working_copy.md](working_copy.md)); the review page lists them by priority, highest first, and within one priority the fewest open decisions first. A record that is `WIP (DF)` with neither is reviewed and waits for the PR diff --git a/.claude/skills/knowledge-data-foundry/references/records.md b/.claude/skills/knowledge-data-foundry/references/records.md index 9e6df36a..4c809fcc 100644 --- a/.claude/skills/knowledge-data-foundry/references/records.md +++ b/.claude/skills/knowledge-data-foundry/references/records.md @@ -82,7 +82,7 @@ record that already has a verdict gets an `**Assessment (AI)**` instead. | `name` | The names the dataset is published under, joined with ` / `. The body has no title heading. | | `checked_by` | The curators who reviewed it, or `AI (UNVERIFIED)`. | | `data_foundry_status` | Work state and collection membership in one list (below). | -| `claimed_by` | The curator holding a WIP claim (one name, never the AI). | +| `claimed_by` | The curator holding the claim, mirrored from the claims board ([claims.md](claims.md)); one name, never the AI. | | `suggestion` | The verdict; the one required field. | | `priority` | 1-5: how urgently a curator should act on the record at its stage (5 first): verify the screening and send it to `/add-dataset`, then review what `/add-dataset` added. Set by `/screen-candidates`, set again by `/add-dataset` from what it measured, changed by the curator; none on a rejected record. The factors are in [selection.md](../../knowledge-curation/references/selection.md), "Priority". | | `decision_markers` | Issue flags (`Duplicate`, `Trivial`, `Image`, `Too Small`, `Data Quality Issue`, ...); a clean dataset has none. | @@ -118,12 +118,12 @@ option through the dashboard's οΌ‹ header button or `save_vocabularies`. ### Claims -A WIP value is a claim: `claimed_by` names the curator who holds it, set and cleared together with the WIP value. -Leave a record claimed by someone else alone (no triage, no edits, only a note to the curator). When a curator takes -records, the claim goes to `main` on its own, ahead of the work: a branch from `main` whose one commit changes only the -claim lines (`data_foundry_status`, `claimed_by`), merged as soon as CI passes; then `main` is merged into the work -branch. The PR that brings the work to `main` sets the final state and clears the claim. The dashboard's Claimed by -filter on (empty) lists what is free. +Who works on which dataset lives on the claims board, a branch every curator pushes to directly +([claims.md](claims.md)): `claims claim ` before the work, `claims release ` when it reaches `main` or is +dropped. Leave a dataset another curator holds alone (no triage, no edits, only a note to the curator). The record +mirrors the claim: `claimed_by` names the holder and a WIP value says the stage, both written by the `claims` commands +and carried to `main` by the work's own PR, which sets the final state. `claims sync` repairs a record that drifted +from the board. ### Duplicates and retirements diff --git a/.claude/skills/review-added-datasets/SKILL.md b/.claude/skills/review-added-datasets/SKILL.md index ae68abb5..d705a3ae 100644 --- a/.claude/skills/review-added-datasets/SKILL.md +++ b/.claude/skills/review-added-datasets/SKILL.md @@ -114,15 +114,16 @@ Apply that mark only once the curator has answered; apply the others now. Never * **Record fields:** `update` sets the record's field to the definition's value; `keep` leaves the record as it is. Either way the row leaves the `REVIEW.md`. * **ship:** the definition stays; apply the item marks; the dataset joins the accepted set and is built later. The - record keeps its claim until the work reaches `main`; that PR sets `DF: Yes` and clears `WIP (DF)` and - `claimed_by`. + claim stays until the work goes to `main`: the PR that takes it there sets `DF: Yes`, and `claims release ` + ([claims.md](../knowledge-data-foundry/references/claims.md)) clears the board and the record's `WIP (DF)` and + `claimed_by` in that PR. * **changes:** apply the changes, run `dataset check` and the probes they touch ([verification.md](../knowledge-curation/references/verification.md)), and put the record back to the curator with what changed: the issues that stay open stay in the `REVIEW.md`. * **dont:** retire it as [records.md](../knowledge-data-foundry/references/records.md) says (duplicates and retirements): the record gets `No` (or `No (Retired)` for a shipped dataset) with the reason as a marker and a CC - line, its claim is released, and the folder, its `REVIEW.md` with it, leaves the working copy (`v2_path` cleared, - `sync-notebooks --check` clean). + line, its claim is released (`claims release `), and the folder, its `REVIEW.md` with it, leaves the working + copy (`v2_path` cleared, `sync-notebooks --check` clean). * **discuss:** answer on the page and wait for the curator's next mark. * **Cross-dataset items:** `ok` means go ahead as the item proposes, `leave` means no change. While the page is in use, close a settled item in the working copy's `REVIEW.md` with a `**Resolved ():**` label instead of diff --git a/.claude/skills/screen-candidates/SKILL.md b/.claude/skills/screen-candidates/SKILL.md index 7e856f4e..26772306 100644 --- a/.claude/skills/screen-candidates/SKILL.md +++ b/.claude/skills/screen-candidates/SKILL.md @@ -31,8 +31,10 @@ $ARGUMENTS ## Step 1: List the records Resolve the input to records under `curation/records/`; create a record for a new link (`unique_name` in snake_case, -`name`, the link in `source_links`) through the store API. Skip every record claimed by another curator. A record that -already has a screening block is screened again only when the curator asks. +`name`, the link in `source_links`) through the store API. Claim the records for the run (`claims claim +--stage triage --note "screening"`, [claims.md](../knowledge-data-foundry/references/claims.md)) and skip every one +the command skips: another curator holds it. Release them when the screening is written (`claims release `). +A record that already has a screening block is screened again only when the curator asks. A batch runs on the `candidate-screener` agent, about 10 records per agent, all in one message; the main session keeps the coverage snapshot, the reject check and the report. Mind the limits in diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml index 73beec68..4851965a 100644 --- a/.github/workflows/pages.yaml +++ b/.github/workflows/pages.yaml @@ -5,7 +5,10 @@ name: Pages # The site is regenerated from curation/records (the source of truth) on every # push to main that touches the backlog, the viewer, or the guideline skills # (rendered by guidelines.html), then deployed. Nothing generated is committed -# back to the repo. +# back to the repo. Its Claimed by column shows the live claims board (the +# `claims` branch): every push there runs `claims-to-pages.yaml`, a workflow that +# lives on that branch only, which dispatches this one on main +# (`data-foundry-curation claims init --pages-dispatch` puts it there). # # One-time setup: Settings -> Pages -> "Build and deployment" -> Source = # "GitHub Actions". The repository must be public (or on a plan that allows @@ -53,6 +56,10 @@ jobs: - name: Install package run: uv sync + - name: Fetch the claims board + # Caches the board in .git for build-site; a missing branch is an empty board. + run: uv run data-foundry-curation claims list > /dev/null + - name: Build read-only static site env: DATA_FOUNDRY_CURATION_DIR: ${{ github.workspace }}/curation diff --git a/CLAUDE.md b/CLAUDE.md index fd0953d3..5b1c2881 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,6 +32,10 @@ gotchas that look like blockers but aren't β€” lives in * the curator reviews what was added β€” `/review-added-datasets` ([`.claude/skills/review-added-datasets/SKILL.md`](.claude/skills/review-added-datasets/SKILL.md)): the review page from the `REVIEW.md` files; the curator's decisions are checked for consistency and applied. +* **Claims (who works on what):** a live board on the remote's `claims` branch, pushed directly + (`data-foundry-curation claims list|claim|release`, + [`claims.md`](.claude/skills/knowledge-data-foundry/references/claims.md)). Claim records before working on them + and leave the ones another curator holds alone; the SessionStart hook lists the board, a PreToolUse hook warns. * **Builds** happen only on the curator's ask ([`rebuild.md`](.claude/skills/knowledge-data-foundry/references/rebuild.md)). Otherwise you touch code to extend the package (`src/data_foundry/`) and update examples (`examples/`) when an API changes. diff --git a/curation/vocabularies.yaml b/curation/vocabularies.yaml index 2050e7ca..76ee535a 100644 --- a/curation/vocabularies.yaml +++ b/curation/vocabularies.yaml @@ -8,9 +8,6 @@ checked_by: claimed_by: - Lennart - Andrej -- Mustafa -- Alex -- Gioia data_foundry_status: - 'DF: Yes' - WIP (DF) diff --git a/datasets/_dev/tabarena-v0pt2/CHANGELOG.md b/datasets/_dev/tabarena-v0pt2/CHANGELOG.md index afb7e6f9..a47e4856 100644 --- a/datasets/_dev/tabarena-v0pt2/CHANGELOG.md +++ b/datasets/_dev/tabarena-v0pt2/CHANGELOG.md @@ -4,6 +4,15 @@ Every change to this folder gets an entry here, newest first: edited definitions datasets, and re-runs that produce a new container (give the new UUID). Say what changed and why, and link the record, audit or PR that motivated it. +## 2026-10-23 (claims move to a live board) + +- `GETTING_STARTED.md` (section 7) describes the new claims: a branch `claims` of the repository that every curator + pushes to directly, one file per claimed dataset, so a claim shows within seconds instead of after a claim PR to + `main`. The tooling is `data-foundry-curation claims` (`src/data_foundry/curation/claims.py`), the dashboard's + Claimed by column, and a Claude Code hook that warns before an agent edits a dataset another curator holds; see + `.claude/skills/knowledge-data-foundry/references/claims.md`. The records keep `claimed_by` and the WIP value as a + mirror of the board. + ## 2026-10-22 (numerai28_6 added as forced IID) - New folder `numerai28_6` (Lennart, 2026-10-22: add as forced IID). It is Numerai's 2016 Kaggle training sample, CC0: 96,320 rows, 21 encrypted features and a binary target. diff --git a/datasets/_dev/tabarena-v0pt2/GETTING_STARTED.md b/datasets/_dev/tabarena-v0pt2/GETTING_STARTED.md index 872039c6..b26fc987 100644 --- a/datasets/_dev/tabarena-v0pt2/GETTING_STARTED.md +++ b/datasets/_dev/tabarena-v0pt2/GETTING_STARTED.md @@ -141,8 +141,8 @@ approximate). The steps are in [`CONTRIBUTING_DATASETS.md`](../../../CONTRIBUTIN **Recommended: Claude adds a batch, you review it.** -1. Claim the records first (section 7), then in Claude Code run `/add-dataset` on a dashboard filter, for example - "every record with `suggestion = Yes`, `data_foundry_status` without `DF: Yes`, claimed by me, `checked_by` without +1. In Claude Code run `/add-dataset` on a dashboard filter (it claims the records first, section 7), for example + "every record with `suggestion = Yes`, `data_foundry_status` without `DF: Yes`, free or claimed by me, `checked_by` without `AI (UNVERIFIED)`" (only records you have checked). Per source one agent downloads the data, writes the definition, runs `dataset check` and the leak, task and group probes and the verification checklist; then a review panel of agents attacks the result (a claim auditor, column @@ -185,18 +185,23 @@ definition (Workflow B). Each record's comments say what is open. Pick one, or a batch, and claim it before you start, so nobody else works on it: -1. **Find what is free.** The *Claimed by* filter on *(empty)* shows the records nobody holds - ([the same view locally](http://127.0.0.1:8765/#hf.claimed_by=+%E2%88%85)). Leave a record claimed by someone +0. **Once:** name yourself, `git config data-foundry.curator ` (a name of `claimed_by` in + `curation/vocabularies.yaml`). +1. **Find what is free.** Claims live on a branch of the repository that everyone pushes to directly, so they show + within seconds: [the board on GitHub](https://github.com/TabArena/data-foundry/blob/claims/README.md), + `.venv/bin/python -m data_foundry.curation.cli claims list`, or the dashboard's *Claimed by* column, whose filter + on *(empty)* shows the datasets nobody holds + ([the same view locally](http://127.0.0.1:8765/#hf.claimed_by=+%E2%88%85)). Leave a dataset claimed by someone else alone, or ask them. -2. **Claim on `main`, ahead of the work.** Branch from `main`, set `WIP (Triage)` (verdict still open) or `WIP (DF)` - (a final `Yes`, definition to write) and your name in `claimed_by` on the records you take, and nothing else. - Open a small PR and merge it as soon as CI passes: the published log then shows your claim. Claude makes this - claim branch on request. -3. **Work on your own branch.** Merge `main` into it after the claim PR (the claim lines are identical on both - sides, so nothing conflicts) and do the work there, as long as it takes. -4. **Release.** The PR that brings the work to `main` sets the final state (`DF: Yes`, or the settled suggestion) - and clears `claimed_by`. Giving a record up without the work is another small PR on `main`. - -A WIP state without a name, or a name without a WIP state, puts the record in the Review queue. The record's +2. **Claim before you work.** `claims claim --note ""`, or pick your name in the dashboard's + *Claimed by* cell. It pushes the claim to the board (no PR) and writes `claimed_by` and the WIP value into the + records. The Claude commands claim their batch themselves, and a hook warns an agent before it edits a dataset + someone else holds. A claim lasts 7 days; `claims renew` extends yours. +3. **Work on your own branch**, as long as it takes. +4. **Release.** `claims release ` in the PR that brings the work to `main` (it sets the final state, + `DF: Yes` or the settled suggestion), or as soon as you give a dataset up. + +How it works: [claims.md](../../../.claude/skills/knowledge-data-foundry/references/claims.md). A WIP state without a +name, or a name without a WIP state, puts the record in the Review queue. The record's comments, the Guidelines tab and the definitions in this folder answer most questions; for the rest, ask the team (Lennart). diff --git a/pyproject.toml b/pyproject.toml index 0c99a800..8abfd81c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -223,6 +223,8 @@ exclude = [ "INP001", # skill script folders are not packages "S603", "S607", # subprocess calls with fixed commands ] +# Claude Code hooks (`.claude/hooks/`): scripts the agent harness runs, not a package. +".claude/hooks/*.py" = ["INP001"] # v2 dataset definitions (`datasets/**/dataset.py`): one class of declarations and a few hooks. Metadata text # and column lists are long, commented-out columns document decisions, and optional dependencies (polars, # matplotlib) are imported inside the hook that needs them. diff --git a/src/data_foundry/curation/app.py b/src/data_foundry/curation/app.py index 9a934783..0696e69a 100644 --- a/src/data_foundry/curation/app.py +++ b/src/data_foundry/curation/app.py @@ -17,6 +17,11 @@ The server is meant to run locally per curator (edit -> git commit -> PR); it is not hardened for multi-user or public exposure. + +Claims come from the live claims board (:mod:`data_foundry.curation.claims`), not from the records: a background +thread fetches the board every :data:`CLAIMS_REFRESH_S` seconds, the records payload shows each record's live holder +in ``claimed_by`` (with the details in ``live_claim``), and editing the Claimed by cell claims (your own name) or +releases (empty) through ``POST /api/claims/``, which also mirrors the claim into the record. """ from __future__ import annotations @@ -24,16 +29,21 @@ import json import os import re +import threading +import time +from dataclasses import asdict from functools import partial from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from urllib.parse import unquote, urlparse +from data_foundry.curation import claims from data_foundry.curation._paths import records_dir, resolve_curation_root, skills_dir, vocabularies_path from data_foundry.curation.notebooks import resolve_notebook -from data_foundry.curation.record import FIELDS, load_vocabularies, save_vocabularies +from data_foundry.curation.record import ACCEPTED_SUGGESTIONS, FIELDS, load_vocabularies, save_vocabularies from data_foundry.curation.store import ( load_all, + load_record, record_from_dict, record_to_dict, save_record, @@ -60,6 +70,9 @@ START_REPO_PATH: str = ".claude/skills/README.md" """Where the Start here page lives in the repo; its relative links are resolved from here.""" +CLAIMS_REFRESH_S: float = 20.0 +"""How often the dashboard server fetches the claims board.""" + _FRONT_MATTER = re.compile(r"\Aο»Ώ?---[ \t]*\r?\n(?:.*?\r?\n)?(?:---|\.\.\.)[ \t]*(?:\r?\n|\Z)", re.DOTALL) _FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})") _H1 = re.compile(r"^ {0,3}#[ \t]+(.+?)(?:[ \t]+#+)?[ \t]*$") @@ -74,6 +87,57 @@ def _schema_payload() -> dict[str, object]: } +class _ClaimsPoller: + """Fetches the claims board every :data:`CLAIMS_REFRESH_S` seconds in a daemon thread; keeps the last error.""" + + def __init__(self, interval: float = CLAIMS_REFRESH_S) -> None: + self.interval = interval + self.error: str | None = None + + def start(self) -> None: + """Run the loop in a daemon thread (it dies with the server).""" + threading.Thread(target=self._loop, name="claims-poller", daemon=True).start() + + def _loop(self) -> None: + while True: + try: + claims.fetch() + self.error = None + except claims.ClaimsError as exc: + self.error = str(exc).splitlines()[0] + time.sleep(self.interval) + + +_POLLER: _ClaimsPoller | None = None + + +def _claims_payload() -> dict[str, object]: + """The state of the claims board for the page: who you are, when it was fetched, the last error.""" + board = claims.cached_board() + return { + "me": claims.configured_curator(), + "remote": claims.claims_remote(), + "branch": claims.claims_branch(), + "tip": board.tip if board else None, + "fetched_at": board.fetched_at if board else None, + "count": len(board.claims) if board else 0, + "error": _POLLER.error if _POLLER is not None else None, + } + + +def _overlay_claim(row: dict[str, object], board: claims.Board | None) -> dict[str, object]: + """Show the live holder in ``claimed_by`` and the claim in ``live_claim``. + + Without a board (none fetched, or no claims branch yet) the row keeps the record's own value. + """ + if board is None or board.tip is None: + return row + claim = board.claims.get(str(row["unique_name"])) + row["claimed_by"] = claim.holder if claim else None + row["live_claim"] = {**asdict(claim), "expired": claim.expired()} if claim else None + return row + + def _records_payload(directory: Path | None) -> list[dict[str, object]]: """Serialise every record, augmented with read-only *derived* link fields. @@ -84,13 +148,15 @@ def _records_payload(directory: Path | None) -> list[dict[str, object]]: pointing at its own markdown file (``curation/records/.md``), so a record can be referenced directly from the UI. The ``*_url`` keys are not part of the record schema and are never written back β€” the - dashboard's per-row πŸ““ / πŸ“„ buttons read them (live and on the static site). + dashboard's per-row πŸ““ / πŸ“„ buttons read them (live and on the static site). With a cached + claims board, ``claimed_by`` is the live holder and ``live_claim`` the claim (:func:`_overlay_claim`). """ repo_root = resolve_curation_root().parent.resolve() records_root = (Path(directory) if directory is not None else records_dir()).resolve() + board = claims.cached_board() payload: list[dict[str, object]] = [] for r in load_all(directory): - d = record_to_dict(r) + d = _overlay_claim(record_to_dict(r), board) rel = r.notebook_path or resolve_notebook(r) if rel: d["notebook_path"] = rel @@ -128,7 +194,9 @@ def _version_payload(directory: Path | None) -> dict[str, object]: vocab = Path(vocabularies_path()) if vocab.exists(): newest = max(newest, vocab.stat().st_mtime_ns) - return {"version": f"{newest}-{count}", "records": count} + board = claims.cached_board() # a claim pushed by anyone moves the token too + tip = (board.tip or "none")[:12] if board is not None else "-" + return {"version": f"{newest}-{count}-{tip}", "records": count} def start_readme_path() -> Path: @@ -306,6 +374,8 @@ def do_GET(self) -> None: self._send_json(_version_payload(self._records_dir)) elif route == "/api/docs": self._send_json(_docs_payload()) + elif route == "/api/claims": + self._send_json(_claims_payload()) elif route.startswith("/docs/"): self._send_doc(unquote(route[len("/docs/") :])) else: @@ -326,11 +396,13 @@ def do_PUT(self) -> None: self._send_json({"error": str(exc)}, status=500) def do_POST(self) -> None: - """Add a new dropdown option to the vocabulary file.""" + """Add a new dropdown option to the vocabulary file, or claim / release a dataset.""" route = urlparse(self.path).path try: if route == "/api/vocab": self._add_vocab_option() + elif route.startswith("/api/claims/"): + self._claim_action(unquote(route[len("/api/claims/") :])) else: self._send_json({"error": "not found"}, status=404) except Exception as exc: # noqa: BLE001 @@ -352,6 +424,36 @@ def _update_record(self, unique_name: str) -> None: save_record(record, self._records_dir) self._send_json(record_to_dict(record)) + def _claim_action(self, unique_name: str) -> None: + """Claim (``{"action": "claim"}``) or release a dataset on the board, mirror it into the record, return the row. + + The stage follows the record: ``add`` for an accepted dataset, else ``triage``. A dataset another curator + holds, a missing curator name or an unreachable remote answer 409 with the reason. + """ + body = self._read_json_body() + path = Path(self._records_dir) / f"{unique_name}.md" + if not path.exists(): + self._send_json({"error": f"unknown record: {unique_name}"}, status=404) + return + action = body.get("action") + try: + if action == "claim": + stage = "add" if load_record(path).suggestion in ACCEPTED_SUGGESTIONS else "triage" + outcome = claims.claim([unique_name], stage=stage, note=str(body.get("note") or "")) + elif action == "release": + outcome = claims.release([unique_name]) + else: + self._send_json({"error": "action must be claim or release"}, status=400) + return + except claims.ClaimsError as exc: + self._send_json({"error": str(exc)}, status=409) + return + if unique_name in outcome.skipped: + self._send_json({"error": f"{unique_name}: {outcome.skipped[unique_name]}"}, status=409) + return + claims.mirror_to_records(outcome.board, [unique_name], self._records_dir) + self._send_json(_overlay_claim(record_to_dict(load_record(path)), outcome.board)) + def _add_vocab_option(self) -> None: body = self._read_json_body() field, value = body.get("field"), body.get("value") @@ -368,7 +470,10 @@ def _add_vocab_option(self) -> None: def serve(host: str = "127.0.0.1", port: int = 8765, directory: str | Path | None = None) -> None: """Run the dashboard server until interrupted.""" + global _POLLER # noqa: PLW0603 - one poller per server process target = Path(directory) if directory is not None else records_dir() + _POLLER = _ClaimsPoller() + _POLLER.start() handler = partial(CurationHandler, directory=target) httpd = ThreadingHTTPServer((host, port), handler) n = len([p for p in target.glob("*.md") if not p.name.startswith("_")]) if target.exists() else 0 diff --git a/src/data_foundry/curation/claims.py b/src/data_foundry/curation/claims.py new file mode 100644 index 00000000..e3e168eb --- /dev/null +++ b/src/data_foundry/curation/claims.py @@ -0,0 +1,772 @@ +"""Live dataset claims: who works on which dataset right now, shared through one git branch. + +A claim says that one curator is working on one dataset: its curation record and its v2 folder (with ``REVIEW.md``). +Claims do not travel with the work. They live on a branch of their own (``claims`` on ``origin`` by default): one +small YAML file per claimed dataset under ``claims/``, and a generated ``README.md`` table that GitHub shows as the +team's board. A claim is pushed straight to that branch, with no pull request, so the other curators and their agents +see it within seconds. + +Atomicity comes from git. Every change is a commit on top of the tip just fetched, and the remote rejects a push that +is not a fast-forward. When two curators change the board at the same moment, one push lands; the other fetches +again, checks again (the dataset may now be taken) and retries. Nothing here touches the working tree, the index or +the checked-out branch: the commits are built with git plumbing in a temporary index. + +Login is the remote's own: whoever can push to the repository can claim, whoever can read it sees the board. Any git +remote works; ``$DATA_FOUNDRY_CLAIMS_REMOTE`` and ``$DATA_FOUNDRY_CLAIMS_BRANCH`` pick another remote or branch. + +The curator is named by ``$DATA_FOUNDRY_CURATOR`` or ``git config data-foundry.curator`` and must be one of the +``claimed_by`` values of ``vocabularies.yaml``. Each fetch caches the board in the git directory +(``data-foundry-claims.json``), so the warnings (``save_record``, the Claude Code hook, the dashboard) need no network. + +Claims are advisory: nothing refuses a write. A claim expires after ``ttl_days`` (7 by default) unless renewed; an +expired claim can be taken over with ``steal``. The records keep ``claimed_by`` and the WIP value as a mirror of the +board (:func:`mirror_to_records`), written in the work branch and merged with the work. +""" + +from __future__ import annotations + +import datetime as dt +import json +import os +import subprocess +import tempfile +import time +import warnings +from collections.abc import Callable, Iterable +from dataclasses import asdict, dataclass, replace +from functools import cache +from pathlib import Path + +import yaml + +from data_foundry.curation._paths import records_dir, resolve_curation_root + +CLAIMS_REMOTE_ENV = "DATA_FOUNDRY_CLAIMS_REMOTE" +CLAIMS_BRANCH_ENV = "DATA_FOUNDRY_CLAIMS_BRANCH" +CURATOR_ENV = "DATA_FOUNDRY_CURATOR" +CURATOR_GIT_KEY = "data-foundry.curator" +DEFAULT_REMOTE = "origin" +DEFAULT_BRANCH = "claims" +LOCAL_REF = "refs/data-foundry/claims" +"""Where the fetched board is kept locally: a private ref, never a branch anyone checks out.""" +CACHE_NAME = "data-foundry-claims.json" +CLAIMS_DIR = "claims" +STAGES: tuple[str, ...] = ("triage", "add", "review") +"""``triage``: the verdict is open (screening, a check); ``add``: the v2 folder is being built; ``review``: the +curator's review of an added dataset.""" +DEFAULT_TTL_DAYS = 7 +MAX_ATTEMPTS = 5 + +PAGES_DISPATCH_PATH = ".github/workflows/claims-to-pages.yaml" +PAGES_DISPATCH_WORKFLOW = """\ +# Lives on the claims branch only: every push of a claim asks the Pages workflow on main to rebuild the +# read-only dashboard, so its Claimed by column follows the board. Written by `data-foundry-curation claims init`. +name: Claims to Pages +on: + push: + branches: [claims] +permissions: + actions: write +jobs: + dispatch: + runs-on: ubuntu-latest + steps: + - run: gh workflow run pages.yaml --repo "$GITHUB_REPOSITORY" --ref main + env: + GH_TOKEN: ${{ github.token }} +""" + + +class ClaimsError(RuntimeError): + """A claims operation could not be done (no curator name, no access to the remote, a busy branch).""" + + +class ClaimWarning(UserWarning): + """A dataset is being changed while another curator holds its claim.""" + + +def _utcnow() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc).replace(microsecond=0) + + +def _iso(when: dt.datetime) -> str: + return when.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _parse_time(value: object) -> dt.datetime: + if isinstance(value, dt.datetime): # YAML read an unquoted timestamp + return value if value.tzinfo else value.replace(tzinfo=dt.timezone.utc) + return dt.datetime.strptime(str(value), "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=dt.timezone.utc) + + +@dataclass(frozen=True) +class Claim: + """One curator's claim on one dataset.""" + + name: str + """The dataset's ``unique_name`` (the record's file stem and the v2 folder name).""" + holder: str + """The curator, a ``claimed_by`` value of the vocabulary.""" + stage: str + """One of :data:`STAGES`.""" + claimed_at: str + """UTC, ``YYYY-MM-DDTHH:MM:SSZ``; kept when the holder renews.""" + expires_at: str + """UTC; after it, anyone may take the claim over (``steal``).""" + note: str = "" + """Free text: what is being done, e.g. "add-dataset batch 3".""" + + def expired(self, now: dt.datetime | None = None) -> bool: + """Whether the claim is past its expiry.""" + return self.seconds_left(now) <= 0 + + def seconds_left(self, now: dt.datetime | None = None) -> float: + """Seconds until the claim expires (negative once it has).""" + return (_parse_time(self.expires_at) - (now or _utcnow())).total_seconds() + + def describe(self) -> str: + """One line for messages: holder, stage, dates and note.""" + state = "expired" if self.expired() else "until" + text = f"{self.holder} ({self.stage}) since {self.claimed_at[:10]}, {state} {self.expires_at[:10]}" + return f"{text}: {self.note}" if self.note else text + + def to_yaml(self) -> str: + """The file stored on the claims branch.""" + return yaml.safe_dump(asdict(self), sort_keys=False, allow_unicode=True, width=4096) + + @classmethod + def from_mapping(cls, data: dict[str, object]) -> Claim: + """Read a claim from a parsed YAML file or the cache, tolerating timestamps YAML parsed itself.""" + return cls( + name=str(data["name"]), + holder=str(data["holder"]), + stage=str(data.get("stage") or "triage"), + claimed_at=_iso(_parse_time(data["claimed_at"])), + expires_at=_iso(_parse_time(data["expires_at"])), + note=str(data.get("note") or ""), + ) + + +@dataclass(frozen=True) +class Board: + """Every claim on the claims branch at one commit.""" + + claims: dict[str, Claim] + tip: str | None + """The branch's commit, or ``None`` when the branch does not exist yet.""" + fetched_at: float + """When the board was read from the remote (epoch seconds).""" + remote: str + branch: str + + def age(self) -> float: + """Seconds since the board was fetched.""" + return time.time() - self.fetched_at + + def held_by_other(self, name: str, me: str | None) -> Claim | None: + """The live claim on ``name`` when someone other than ``me`` holds it, else ``None``.""" + claim = self.claims.get(name) + if claim is None or claim.holder == me or claim.expired(): + return None + return claim + + def by_holder(self) -> dict[str, list[Claim]]: + """The claims grouped by holder, holders and names sorted.""" + out: dict[str, list[Claim]] = {} + for claim in sorted(self.claims.values(), key=lambda c: (c.holder, c.name)): + out.setdefault(claim.holder, []).append(claim) + return out + + +@dataclass(frozen=True) +class Outcome: + """What a change of the board did.""" + + changed: list[str] + """The datasets written: claimed, renewed or released.""" + skipped: dict[str, str] + """The datasets left alone, each with the reason.""" + board: Board + """The board after the change.""" + + +# -- git ------------------------------------------------------------------------------------------------------------ + + +def repo_root() -> Path: + """The repository the claims are read from and pushed through: the parent of the curation root.""" + return resolve_curation_root().parent + + +def claims_remote() -> str: + """The remote that holds the claims branch (``$DATA_FOUNDRY_CLAIMS_REMOTE``, default ``origin``).""" + return os.environ.get(CLAIMS_REMOTE_ENV) or DEFAULT_REMOTE + + +def claims_branch() -> str: + """The claims branch (``$DATA_FOUNDRY_CLAIMS_BRANCH``, default ``claims``).""" + return os.environ.get(CLAIMS_BRANCH_ENV) or DEFAULT_BRANCH + + +def _run_git( + repo: Path, *args: str, stdin: str | bytes | None = None, env: dict[str, str] | None = None, binary: bool = False +) -> subprocess.CompletedProcess: + """Run git without a terminal: a credential or passphrase prompt fails at once instead of hanging a hook.""" + env = {**(env if env is not None else os.environ), "GIT_TERMINAL_PROMPT": "0"} + feed = {"input": stdin} if stdin is not None else {"stdin": subprocess.DEVNULL} + try: + return subprocess.run( # noqa: S603 - fixed git subcommands; names are passed as data, never to a shell + ["git", "-C", str(repo), *args], # noqa: S607 - git from PATH + **feed, + capture_output=True, + text=not binary, + encoding=None if binary else "utf-8", + env=env, + timeout=60, + start_new_session=True, # no controlling terminal, so ssh cannot stop to ask for a passphrase + check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ClaimsError(f"git {args[0]} failed: {exc}") from exc + + +def _git(repo: Path, *args: str, stdin: str | None = None, env: dict[str, str] | None = None) -> str: + proc = _run_git(repo, *args, stdin=stdin, env=env) + if proc.returncode: + raise ClaimsError(f"git {' '.join(args[:2])} failed: {proc.stderr.strip()}") + return proc.stdout.strip() + + +_AUTH_HINT = ( + "The claims use the same remote and credentials as `git push`; check that `git ls-remote {remote}` works here " + "(an ssh agent that went stale can be bypassed with `env SSH_AUTH_SOCK= `)." +) + + +def _remote_error(action: str, stderr: str, remote: str) -> ClaimsError: + return ClaimsError(f"{action} on '{remote}' failed: {stderr.strip()}\n{_AUTH_HINT.format(remote=remote)}") + + +@cache +def _git_dir(repo: str) -> Path | None: + """The repository's common git directory (shared by its worktrees), or ``None`` outside a git repository.""" + try: + proc = _run_git(Path(repo), "rev-parse", "--git-common-dir") + except ClaimsError: + return None + if proc.returncode: + return None + path = Path(proc.stdout.strip()) + return path if path.is_absolute() else (Path(repo) / path).resolve() + + +# -- reading the board ------------------------------------------------------------------------------------------------ + + +def _read_tree(repo: Path, tip: str) -> dict[str, Claim]: + listing = _run_git(repo, "ls-tree", "-r", "-z", tip, "--", f"{CLAIMS_DIR}/", binary=True) + if listing.returncode: + raise ClaimsError(f"git ls-tree failed: {listing.stderr.decode(errors='replace').strip()}") + shas = [] + for entry in listing.stdout.split(b"\0"): + if not entry: + continue + meta, _, path = entry.partition(b"\t") + if path.endswith(b".yaml"): + shas.append(meta.split()[2]) + if not shas: + return {} + batch = _run_git(repo, "cat-file", "--batch", stdin=b"\n".join(shas) + b"\n", binary=True) + if batch.returncode: + raise ClaimsError(f"git cat-file failed: {batch.stderr.decode(errors='replace').strip()}") + out, data, pos = {}, batch.stdout, 0 + for _ in shas: + header_end = data.index(b"\n", pos) + size = int(data[pos:header_end].split()[2]) + body = data[header_end + 1 : header_end + 1 + size] + pos = header_end + 1 + size + 1 + claim = Claim.from_mapping(yaml.safe_load(body.decode("utf-8")) or {}) + out[claim.name] = claim + return out + + +def fetch(repo: str | Path | None = None) -> Board: + """Fetch the claims branch from the remote, cache it and return the board (an empty one when there is no branch). + + Raises: + ClaimsError: the remote cannot be reached or read. + """ + repo = Path(repo) if repo is not None else repo_root() + remote, branch = claims_remote(), claims_branch() + proc = _run_git( + repo, "fetch", "--quiet", "--no-tags", "--no-write-fetch-head", remote, f"+refs/heads/{branch}:{LOCAL_REF}" + ) + if proc.returncode: + if "couldn't find remote ref" not in proc.stderr: + raise _remote_error("Fetching the claims", proc.stderr, remote) + _run_git(repo, "update-ref", "-d", LOCAL_REF) # the branch is gone (or was never made): an empty board + board = Board({}, None, time.time(), remote, branch) + else: + tip = _git(repo, "rev-parse", LOCAL_REF) + board = Board(_read_tree(repo, tip), tip, time.time(), remote, branch) + _write_cache(repo, board) + return board + + +def _cache_path(repo: Path) -> Path | None: + git_dir = _git_dir(str(Path(repo).resolve())) + return git_dir / CACHE_NAME if git_dir is not None else None + + +def _write_cache(repo: Path, board: Board) -> None: + path = _cache_path(repo) + if path is None: + return + payload = { + "remote": board.remote, + "branch": board.branch, + "tip": board.tip, + "fetched_at": board.fetched_at, + "claims": [asdict(c) for c in board.claims.values()], + } + with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent, suffix=".tmp", delete=False) as tmp: + json.dump(payload, tmp, ensure_ascii=False, indent=1) + Path(tmp.name).replace(path) # atomic: a reader never sees half a file + + +_CACHE_MEMO: dict[Path, tuple[int, Board]] = {} + + +def cached_board(repo: str | Path | None = None) -> Board | None: + """The board of the last fetch, from the cache; ``None`` when there is none (never touches the network).""" + try: + path = _cache_path(Path(repo) if repo is not None else repo_root()) + if path is None or not path.exists(): + return None + mtime = path.stat().st_mtime_ns + memo = _CACHE_MEMO.get(path) + if memo is not None and memo[0] == mtime: + board = memo[1] + else: + data = json.loads(path.read_text(encoding="utf-8")) + claims = {c["name"]: Claim.from_mapping(c) for c in data.get("claims", [])} + board = Board(claims, data.get("tip"), float(data["fetched_at"]), data["remote"], data["branch"]) + _CACHE_MEMO[path] = (mtime, board) + except (OSError, ValueError, KeyError, TypeError): + return None + if (board.remote, board.branch) != (claims_remote(), claims_branch()): + return None + return board + + +def load(repo: str | Path | None = None, max_age: float = 30.0) -> Board: + """The cached board when it is at most ``max_age`` seconds old, else a fresh :func:`fetch`.""" + board = cached_board(repo) + if board is not None and board.age() <= max_age: + return board + return fetch(repo) + + +# -- who am I --------------------------------------------------------------------------------------------------------- + + +def configured_curator(repo: str | Path | None = None) -> str | None: + """The curator name from ``$DATA_FOUNDRY_CURATOR`` or ``git config data-foundry.curator``, unchecked.""" + name = os.environ.get(CURATOR_ENV) + if name: + return name.strip() + try: + proc = _run_git(Path(repo) if repo is not None else repo_root(), "config", "--get", CURATOR_GIT_KEY) + except ClaimsError: + return None + if proc.returncode: + return None + return proc.stdout.strip() or None + + +def current_curator(repo: str | Path | None = None) -> str: + """The configured curator name, checked against the ``claimed_by`` vocabulary. + + Raises: + ClaimsError: no name is configured, or it is not in the vocabulary. + """ + from data_foundry.curation.record import load_vocabularies # noqa: PLC0415 - record imports nothing from here + + allowed = load_vocabularies().get("claimed_by", []) + name = configured_curator(repo) + hint = f"one of: {', '.join(allowed)}" if allowed else "a name from vocabularies.yaml (claimed_by)" + if not name: + raise ClaimsError(f"Name yourself first: `git config {CURATOR_GIT_KEY} ` (or ${CURATOR_ENV}), {hint}.") + if allowed and name not in allowed: + raise ClaimsError(f"Curator '{name}' is not in vocabularies.yaml (claimed_by); use {hint}.") + return name + + +# -- changing the board ----------------------------------------------------------------------------------------------- + + +def render_readme(claims: dict[str, Claim], now: dt.datetime | None = None) -> str: + """The board as the markdown table GitHub shows on the claims branch, one section per holder.""" + now = now or _utcnow() + groups: dict[str, list[Claim]] = {} + for claim in sorted(claims.values(), key=lambda c: (c.holder, c.name)): + groups.setdefault(claim.holder, []).append(claim) + counts = ", ".join(f"{holder} {len(items)}" for holder, items in groups.items()) + lines = [ + "# Dataset claims", + "", + "Who is working on which dataset right now. A claim covers the dataset's curation record and its dataset " + "folder. One file per claim under `claims/`, written by `data-foundry-curation claims` (claim, release, " + "renew); do not edit by hand. An expired claim can be taken over.", + "", + f"As of {now.strftime('%Y-%m-%d %H:%M')} UTC: {len(claims)} claim(s)" + (f" ({counts})." if counts else "."), + ] + for holder, items in groups.items(): + lines += ["", f"## {holder}", "", "| Dataset | Stage | Since | Until | Note |", "|---|---|---|---|---|"] + for c in items: + until = c.expires_at[:10] + (" (expired)" if c.expired(now) else "") + note = c.note.replace("|", "\\|").replace("\n", " ") + lines.append(f"| `{c.name}` | {c.stage} | {c.claimed_at[:10]} | {until} | {note} |") + return "\n".join(lines) + "\n" + + +def _commit( + repo: Path, + tip: str | None, + changes: dict[str, Claim | None], + claims: dict[str, Claim], + message: str, + extra: dict[str, str] | None = None, +) -> str: + """A commit on ``tip`` that writes ``changes`` (``None`` removes a claim) and the README, built off the worktree.""" + with tempfile.TemporaryDirectory() as tmp: + env = {**os.environ, "GIT_INDEX_FILE": str(Path(tmp) / "index")} + _git(repo, "read-tree", *([tip] if tip else ["--empty"]), env=env) + files: dict[str, str | None] = { + f"{CLAIMS_DIR}/{n}.yaml": (c.to_yaml() if c else None) for n, c in changes.items() + } + files["README.md"] = render_readme(claims) + files.update(extra or {}) + for path, text in files.items(): + if text is None: + _git(repo, "update-index", "--force-remove", "--", path, env=env) + else: + blob = _git(repo, "hash-object", "-w", "--stdin", stdin=text) + _git(repo, "update-index", "--add", "--cacheinfo", f"100644,{blob},{path}", env=env) + tree = _git(repo, "write-tree", env=env) + return _git(repo, "commit-tree", tree, "-m", message, *(["-p", tip] if tip else [])) + + +def _rejected(stderr: str) -> bool: + """Whether a push failed because the branch moved (retry), not because of access or the network.""" + return any(s in stderr for s in ("[rejected]", "non-fast-forward", "fetch first", "cannot lock ref")) + + +Decision = tuple[dict[str, "Claim | None"], dict[str, str], str] + + +def _update( + repo: str | Path | None, decide: Callable[[Board], Decision], extra: dict[str, str] | None = None +) -> Outcome: + """Fetch, decide, commit and push; when the push loses a race, do it all again on the new tip.""" + repo = Path(repo) if repo is not None else repo_root() + remote, branch = claims_remote(), claims_branch() + for _ in range(MAX_ATTEMPTS): + board = fetch(repo) + changes, skipped, message = decide(board) + if not changes and extra is None: + return Outcome([], skipped, board) + claims = dict(board.claims) + for name, claim in changes.items(): + if claim is None: + claims.pop(name, None) + else: + claims[name] = claim + commit = _commit(repo, board.tip, changes, claims, message, extra) + push = _run_git(repo, "push", "--quiet", remote, f"{commit}:refs/heads/{branch}") + if push.returncode == 0: + _git(repo, "update-ref", LOCAL_REF, commit) + new = Board(claims, commit, time.time(), remote, branch) + _write_cache(repo, new) + return Outcome(sorted(changes), skipped, new) + if not _rejected(push.stderr): + raise _remote_error("Pushing the claims", push.stderr, remote) + raise ClaimsError(f"The claims branch kept moving ({MAX_ATTEMPTS} attempts); run the command again.") + + +def _names(names: Iterable[str]) -> list[str]: + out = list(dict.fromkeys(n.strip() for n in names if n and n.strip())) + for name in out: + if "/" in name or name.startswith("."): + raise ClaimsError(f"Not a dataset name: {name!r}") + return out + + +def _list_for_message(names: Iterable[str], limit: int = 8) -> str: + names = list(names) + head = ", ".join(names[:limit]) + return head + (f" and {len(names) - limit} more" if len(names) > limit else "") + + +def claim( + names: Iterable[str], + *, + stage: str, + note: str = "", + ttl_days: float = DEFAULT_TTL_DAYS, + steal: bool = False, + holder: str | None = None, + repo: str | Path | None = None, +) -> Outcome: + """Claim datasets for the current curator; renew the ones they already hold. + + A dataset someone else holds is skipped with the reason; an expired one is taken over only with ``steal`` (the + commit message names the previous holder). Renewing keeps ``claimed_at`` and, without a new ``note``, the note. + + Args: + names: The datasets' unique names. + stage: One of :data:`STAGES`. + note: What is being done (optional). + ttl_days: Days until the claim expires. + steal: Take over expired claims of others. + holder: Claim in this curator's name (only for importing the records' claims); defaults to the current + curator. + repo: The repository (defaults to the one of the curation root). + + Returns: + The datasets claimed, the ones skipped with the reason, and the new board. + """ + if stage not in STAGES: + raise ClaimsError(f"Unknown stage {stage!r}; use one of {', '.join(STAGES)}.") + me = holder or current_curator(repo) + wanted = _names(names) + + def decide(board: Board) -> Decision: + now = _utcnow() + expires = _iso(now + dt.timedelta(days=ttl_days)) + changes: dict[str, Claim | None] = {} + skipped: dict[str, str] = {} + taken: list[str] = [] + for name in wanted: + old = board.claims.get(name) + mine = old is not None and old.holder == me + if old is not None and not mine: + if not old.expired(now): + skipped[name] = f"held by {old.describe()}" + continue + if not steal: + skipped[name] = f"held by {old.describe()}; take it over with --steal" + continue + taken.append(f"{name} from {old.holder}") + changes[name] = Claim( + name=name, + holder=me, + stage=stage, + claimed_at=old.claimed_at if mine else _iso(now), + expires_at=expires, + note=note or (old.note if mine else ""), + ) + message = f"claim: {_list_for_message(changes)} ({me}, {stage})" + if taken: + message += f"; taken over: {', '.join(taken)}" + return changes, skipped, message + + return _update(repo, decide) + + +def release( + names: Iterable[str], *, force: bool = False, holder: str | None = None, repo: str | Path | None = None +) -> Outcome: + """Release datasets the current curator holds; another curator's claim only with ``force`` (named in the log).""" + me = holder or current_curator(repo) + wanted = _names(names) + + def decide(board: Board) -> Decision: + changes: dict[str, Claim | None] = {} + skipped: dict[str, str] = {} + forced: list[str] = [] + for name in wanted: + old = board.claims.get(name) + if old is None: + skipped[name] = "not claimed" + elif old.holder != me and not force: + skipped[name] = f"held by {old.describe()}; release another curator's claim with --force" + else: + changes[name] = None + if old.holder != me: + forced.append(f"{name} of {old.holder}") + message = f"release: {_list_for_message(changes)} ({me})" + if forced: + message += f"; forced: {', '.join(forced)}" + return changes, skipped, message + + return _update(repo, decide) + + +def renew( + names: Iterable[str] | None = None, + *, + ttl_days: float = DEFAULT_TTL_DAYS, + holder: str | None = None, + repo: str | Path | None = None, +) -> Outcome: + """Push the expiry of the current curator's claims (all of them, or ``names``) ``ttl_days`` from now.""" + me = holder or current_curator(repo) + wanted = _names(names) if names is not None else None + + def decide(board: Board) -> Decision: + expires = _iso(_utcnow() + dt.timedelta(days=ttl_days)) + targets = wanted if wanted is not None else [c.name for c in board.claims.values() if c.holder == me] + changes: dict[str, Claim | None] = {} + skipped: dict[str, str] = {} + for name in targets: + old = board.claims.get(name) + if old is None or old.holder != me: + skipped[name] = "not yours" if old else "not claimed" + else: + changes[name] = replace(old, expires_at=expires) + return changes, skipped, f"renew: {_list_for_message(changes)} ({me}, until {expires[:10]})" + + return _update(repo, decide) + + +def init_branch(repo: str | Path | None = None, *, pages_dispatch: bool = False) -> Outcome: + """Create the claims branch when it is missing (README only), optionally with the Pages dispatch workflow. + + The workflow (:data:`PAGES_DISPATCH_WORKFLOW`) runs on GitHub on every push to the claims branch and asks the + ``pages.yaml`` workflow of ``main`` to rebuild the read-only dashboard, so its Claimed by column follows the board. + """ + if not pages_dispatch: + board = fetch(repo) + if board.tip is not None: + return Outcome([], {}, board) + + def decide(_board: Board) -> Decision: + return {}, {}, "init: the claims board" + (" and the Pages dispatch" if pages_dispatch else "") + + return _update(repo, decide, {PAGES_DISPATCH_PATH: PAGES_DISPATCH_WORKFLOW} if pages_dispatch else {}) + + +# -- the records' mirror ---------------------------------------------------------------------------------------------- + + +def wip_status(stage: str, suggestion: str | None) -> str: + """The record's WIP value for a claim: ``WIP (DF)`` for work on an accepted dataset, else ``WIP (Triage)``.""" + from data_foundry.curation.record import ACCEPTED_SUGGESTIONS # noqa: PLC0415 + + return "WIP (DF)" if stage != "triage" and suggestion in ACCEPTED_SUGGESTIONS else "WIP (Triage)" + + +def mirror_to_records( + board: Board, + names: Iterable[str] | None = None, + directory: str | Path | None = None, + *, + write: bool = True, +) -> dict[str, tuple[str | None, str | None]]: + """Make the records' ``claimed_by`` and WIP value match the board. + + A record with a live or expired claim gets the holder and the WIP value of :func:`wip_status`; a record that + carries a claim or a WIP value the board does not have loses both. Records are reloaded right before each write. + + Args: + board: The board to mirror. + names: Only these records (default: every record). + directory: The records directory (defaults to the standard one). + write: ``False`` only reports what would change. + + Returns: + ``{name: (claimed_by before, claimed_by after)}`` for every record that differs (or differed). + """ + from data_foundry.curation.record import WIP_STATUSES, load_vocabularies # noqa: PLC0415 + from data_foundry.curation.store import load_record, save_record # noqa: PLC0415 - store imports this module + + target = Path(directory) if directory is not None else records_dir() + if names is None: + paths = [p for p in sorted(target.glob("*.md")) if not p.name.startswith("_")] + else: + paths = [target / f"{n}.md" for n in _names(names)] + vocab = load_vocabularies() if write else {} + changed: dict[str, tuple[str | None, str | None]] = {} + for path in paths: + if not path.exists(): + continue + record = load_record(path) + claim = board.claims.get(record.unique_name) + status = list(record.data_foundry_status or []) + has_wip = any(s in WIP_STATUSES for s in status) + if claim is None and not record.claimed_by and not has_wip: + continue + kept = [s for s in status if s not in WIP_STATUSES] + if claim is None: + want_status, want_holder = kept, None + else: + wip = wip_status(claim.stage, record.suggestion) + at = next((i for i, s in enumerate(status) if s in WIP_STATUSES), len(kept)) + want_status, want_holder = [*kept[:at], wip, *kept[at:]], claim.holder + if want_status == status and want_holder == record.claimed_by: + continue + changed[record.unique_name] = (record.claimed_by, want_holder) + if write: + record.data_foundry_status = want_status + record.claimed_by = want_holder + record.needs_review = record.review_reasons(vocab) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ClaimWarning) + save_record(record, target) + return changed + + +def import_from_records( + directory: str | Path | None = None, + *, + ttl_days: float = DEFAULT_TTL_DAYS, + repo: str | Path | None = None, +) -> Outcome: + """Put every claim the records carry (``claimed_by``) on the board, in its holder's name; existing ones stay.""" + from data_foundry.curation.store import load_all # noqa: PLC0415 + + held = [r for r in load_all(directory) if r.claimed_by] + + def decide(board: Board) -> Decision: + now = _utcnow() + expires = _iso(now + dt.timedelta(days=ttl_days)) + changes: dict[str, Claim | None] = {} + skipped: dict[str, str] = {} + for record in held: + old = board.claims.get(record.unique_name) + if old is not None: + skipped[record.unique_name] = f"already on the board: {old.describe()}" + continue + stage = "triage" if "WIP (Triage)" in (record.data_foundry_status or []) else "add" + changes[record.unique_name] = Claim(record.unique_name, record.claimed_by, stage, _iso(now), expires, "") + holders = sorted({c.holder for c in changes.values() if c}) + return changes, skipped, f"import: {len(changes)} claim(s) from the records ({', '.join(holders)})" + + return _update(repo, decide) + + +# -- warnings --------------------------------------------------------------------------------------------------------- + + +def warn_if_claimed(name: str, repo: str | Path | None = None) -> Claim | None: + """Warn (:class:`ClaimWarning`) when the cached board shows ``name`` held by another curator. + + Reads only the cache of the last fetch, never the network, and stays silent when there is no cache or no curator + name. Returns the other curator's claim, if any. + """ + try: + board = cached_board(repo) + claim = board.claims.get(name) if board is not None else None + if claim is None or claim.expired(): + return None + me = configured_curator(repo) + if me is None or claim.holder == me: + return None + except Exception: # noqa: BLE001 - a warning must never break a write + return None + warnings.warn( + ClaimWarning(f"{name} is claimed by {claim.describe()}; leave it to them or talk to them first."), + stacklevel=3, + ) + return claim diff --git a/src/data_foundry/curation/cli.py b/src/data_foundry/curation/cli.py index 1b607838..396a716c 100644 --- a/src/data_foundry/curation/cli.py +++ b/src/data_foundry/curation/cli.py @@ -9,6 +9,7 @@ data-foundry-curation export --format csv OUT.csv # flat snapshot (csv|parquet) data-foundry-curation export --format gsheet --spreadsheet data-foundry-curation dataset {list,check,build,new} # v2 dataset folders (see data_foundry.v2.cli) + data-foundry-curation claims {list,claim,release,renew,check,sync,import-records,init} # live claims board Run ``python -m data_foundry.curation.cli -h`` for details. """ @@ -18,14 +19,16 @@ import argparse import json import sys +from collections.abc import Callable +from dataclasses import asdict -from data_foundry.curation import exporter +from data_foundry.curation import claims, exporter from data_foundry.curation._paths import records_dir, skills_dir from data_foundry.curation.app import serve from data_foundry.curation.notebooks import sync_notebook_paths, sync_v2_paths -from data_foundry.curation.record import load_vocabularies +from data_foundry.curation.record import ACCEPTED_SUGGESTIONS, load_vocabularies from data_foundry.curation.site import build_site -from data_foundry.curation.store import load_all +from data_foundry.curation.store import load_all, load_index from data_foundry.v2.cli import add_dataset_parser @@ -101,6 +104,218 @@ def _cmd_export(args: argparse.Namespace) -> int: return 0 +def _print_claims(items: list[claims.Claim]) -> None: + if not items: + return + width = max(len(c.name) for c in items) + for c in items: + until = c.expires_at[:10] + (" EXPIRED" if c.expired() else "") + print(f" {c.name:<{width}} {c.holder:<8} {c.stage:<7} {c.claimed_at[:10]} -> {until:<18} {c.note}".rstrip()) + + +def _board_age(board: claims.Board) -> str: + age = board.age() + return "just now" if age < 5 else f"{age:.0f} s ago" if age < 120 else f"{age / 60:.0f} min ago" + + +def _load_board(args: argparse.Namespace) -> claims.Board: + if getattr(args, "offline", False): + board = claims.cached_board() + if board is None: + raise claims.ClaimsError("No cached board yet; run once without --offline.") + return board + return claims.load(max_age=getattr(args, "max_age", 0.0)) + + +def _cmd_claims_list(args: argparse.Namespace) -> int: + board = _load_board(args) + me = claims.configured_curator() + items = sorted(board.claims.values(), key=lambda c: (c.holder, c.name)) + if args.mine: + items = [c for c in items if c.holder == me] + elif args.others: + items = [c for c in items if c.holder != me] + if args.json: + print(json.dumps([asdict(c) | {"expired": c.expired()} for c in items], ensure_ascii=False, indent=1)) + return 0 + tip = board.tip[:8] if board.tip else "no branch yet" + print(f"{len(items)} claim(s) on {board.remote}/{board.branch} ({tip}, fetched {_board_age(board)}):") + _print_claims(items) + return 0 + + +def _stage_for(record: object, stage: str | None) -> str: + if stage: + return stage + return "add" if getattr(record, "suggestion", None) in ACCEPTED_SUGGESTIONS else "triage" + + +def _report(outcome: claims.Outcome, verb: str) -> None: + if outcome.changed: + print(f"{verb} {len(outcome.changed)}: {', '.join(outcome.changed)}") + for name, reason in outcome.skipped.items(): + print(f"skipped {name}: {reason}") + + +def _sync_names(args: argparse.Namespace, outcome: claims.Outcome) -> None: + if args.no_sync or not outcome.changed: + return + changed = claims.mirror_to_records(outcome.board, outcome.changed, args.records_dir) + if changed: + print(f"records updated (claimed_by, WIP): {', '.join(changed)}") + + +def _cmd_claims_claim(args: argparse.Namespace) -> int: + index = load_index(args.records_dir) + unknown = [n for n in args.names if n not in index] + if unknown and not args.allow_unknown: + print(f"error: no record for {', '.join(unknown)} (create the record first, or pass --allow-unknown)") + return 2 + groups: dict[str, list[str]] = {} + for name in args.names: + groups.setdefault(_stage_for(index.get(name), args.stage), []).append(name) + complete = True + for stage, names in groups.items(): + outcome = claims.claim(names, stage=stage, note=args.note, ttl_days=args.ttl, steal=args.steal) + _report(outcome, f"claimed ({stage})") + _sync_names(args, outcome) + complete &= not outcome.skipped + return 0 if complete else 1 + + +def _cmd_claims_release(args: argparse.Namespace) -> int: + outcome = claims.release(args.names, force=args.force) + _report(outcome, "released") + _sync_names(args, outcome) + return 0 if not outcome.skipped else 1 + + +def _cmd_claims_renew(args: argparse.Namespace) -> int: + outcome = claims.renew(args.names or None, ttl_days=args.ttl) + _report(outcome, f"renewed for {args.ttl:g} days") + return 0 if not outcome.skipped else 1 + + +def _cmd_claims_check(args: argparse.Namespace) -> int: + board = _load_board(args) + me = claims.configured_curator() + for name in args.names: + c = board.claims.get(name) + if c is None: + print(f"{name}: free") + elif c.holder == me: + print(f"{name}: yours ({c.describe()})") + else: + print( + f"WARNING {name}: claimed by {c.describe()}" + (" - can be taken over (--steal)" if c.expired() else "") + ) + return 0 # advisory: the check warns, it never blocks + + +def _cmd_claims_sync(args: argparse.Namespace) -> int: + board = _load_board(args) + changed = claims.mirror_to_records(board, None, args.records_dir, write=not args.check) + if not changed: + print("Every record's claimed_by and WIP value match the claims board. βœ“") + return 0 + verb = "would change" if args.check else "updated" + print(f"{len(changed)} record(s) {verb} (claimed_by before -> after):") + for name, (before, after) in changed.items(): + print(f" {name}: {before or '(none)'} -> {after or '(none)'}") + return 1 if args.check else 0 + + +def _cmd_claims_import(args: argparse.Namespace) -> int: + outcome = claims.import_from_records(args.records_dir, ttl_days=args.ttl) + _report(outcome, "imported") + return 0 + + +def _cmd_claims_init(args: argparse.Namespace) -> int: + outcome = claims.init_branch(pages_dispatch=args.pages_dispatch) + board = outcome.board + print(f"Claims board: {board.remote}/{board.branch} at {board.tip[:8] if board.tip else '-'}.") + return 0 + + +def _run_claims(func: Callable[[argparse.Namespace], int]) -> Callable[[argparse.Namespace], int]: + """Turn a :class:`~data_foundry.curation.claims.ClaimsError` into one line on stderr and exit code 2.""" + + def run(args: argparse.Namespace) -> int: + try: + return func(args) + except claims.ClaimsError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + return run + + +def _add_claims_parser(sub: argparse._SubParsersAction) -> None: + p_claims = sub.add_parser( + "claims", help="The live claims board: who works on which dataset (a branch of the remote, pushed directly)." + ) + csub = p_claims.add_subparsers(dest="claims_command", required=True) + + def board_options(p: argparse.ArgumentParser) -> None: + p.add_argument("--offline", action="store_true", help="Use the cached board of the last fetch.") + p.add_argument("--max-age", type=float, default=0.0, help="Reuse a cached board up to this many seconds old.") + + p = csub.add_parser("list", help="Show the claims (fetches the board).") + board_options(p) + who = p.add_mutually_exclusive_group() + who.add_argument("--mine", action="store_true") + who.add_argument("--others", action="store_true") + p.add_argument("--json", action="store_true") + p.set_defaults(func=_run_claims(_cmd_claims_list)) + + ttl = {"type": float, "default": claims.DEFAULT_TTL_DAYS, "help": "Days until the claim expires."} + no_sync = {"action": "store_true", "help": "Leave the records' claimed_by and WIP value alone."} + + p = csub.add_parser("claim", help="Claim datasets (skips the ones someone else holds; exit 1 then).") + p.add_argument("names", nargs="+") + p.add_argument("--stage", choices=claims.STAGES, help="Default: add for an accepted record, else triage.") + p.add_argument("--note", default="", help="What you are doing, shown on the board.") + p.add_argument("--ttl", **ttl) + p.add_argument("--steal", action="store_true", help="Take over expired claims of others.") + p.add_argument("--allow-unknown", action="store_true", help="Claim names that have no record yet.") + p.add_argument("--no-sync", **no_sync) + p.set_defaults(func=_run_claims(_cmd_claims_claim)) + + p = csub.add_parser("release", help="Release your claims.") + p.add_argument("names", nargs="+") + p.add_argument("--force", action="store_true", help="Release another curator's claim (logged).") + p.add_argument("--no-sync", **no_sync) + p.set_defaults(func=_run_claims(_cmd_claims_release)) + + p = csub.add_parser("renew", help="Extend your claims (all, or the names given).") + p.add_argument("names", nargs="*") + p.add_argument("--ttl", **ttl) + p.set_defaults(func=_run_claims(_cmd_claims_renew)) + + p = csub.add_parser("check", help="Say for each dataset whether it is free, yours or someone else's (exit 0).") + p.add_argument("names", nargs="+") + board_options(p) + p.set_defaults(func=_run_claims(_cmd_claims_check)) + + p = csub.add_parser("sync", help="Mirror the board into the records' claimed_by and WIP values.") + p.add_argument("--check", action="store_true", help="Report drift and exit non-zero instead of writing.") + board_options(p) + p.set_defaults(func=_run_claims(_cmd_claims_sync)) + + p = csub.add_parser("import-records", help="Once: put the claims the records carry on the board.") + p.add_argument("--ttl", **ttl) + p.set_defaults(func=_run_claims(_cmd_claims_import)) + + p = csub.add_parser("init", help="Create the claims branch if missing.") + p.add_argument( + "--pages-dispatch", + action="store_true", + help="Also put the workflow on the branch that rebuilds the GitHub Pages dashboard on every claim.", + ) + p.set_defaults(func=_run_claims(_cmd_claims_init)) + + def build_parser() -> argparse.ArgumentParser: """Construct the ``data-foundry-curation`` argument parser.""" parser = argparse.ArgumentParser(prog="data-foundry-curation", description=__doc__.splitlines()[0]) @@ -135,6 +350,7 @@ def build_parser() -> argparse.ArgumentParser: p_export.set_defaults(func=_cmd_export) add_dataset_parser(sub) + _add_claims_parser(sub) return parser diff --git a/src/data_foundry/curation/record.py b/src/data_foundry/curation/record.py index d2e8c803..eceb17d5 100644 --- a/src/data_foundry/curation/record.py +++ b/src/data_foundry/curation/record.py @@ -142,7 +142,8 @@ def is_list(self) -> bool: COLLECTION_TAGS: tuple[str, ...] = ("TabArena (v0.1)", "BeyondArena") """``data_foundry_status`` values denoting membership in one of *our* shipped collections.""" WIP_STATUSES: tuple[str, ...] = ("WIP (DF)", "WIP (Triage)") -"""``data_foundry_status`` values that claim a record: someone is working on it, named in ``claimed_by``.""" +"""``data_foundry_status`` values that claim a record: someone is working on it, named in ``claimed_by`` (a mirror of +the claims board, :mod:`data_foundry.curation.claims`).""" ACCEPTED_SUGGESTIONS: tuple[str, ...] = ("Yes", "Yes (Disagreement)") """``suggestion`` values that count as "accepted for inclusion". @@ -180,7 +181,8 @@ class CurationRecord: data_foundry_status: list[str] = pydantic.Field(default_factory=list) claimed_by: str | None = None """The curator who holds the record's WIP claim (:data:`WIP_STATUSES`). Set and cleared together with the WIP - state; it reaches the main branch on its own, ahead of the work, so other curators see the claim.""" + state, as a mirror of the live claims board (:mod:`data_foundry.curation.claims`), which is where other curators + see the claim at once; the record carries it to ``main`` with the work.""" suggestion: str | None = None priority: int | None = pydantic.Field(default=None, ge=1, le=5) """How urgently a curator should verify the screening and send the record to ``/add-dataset``: 5 first, 1 last. diff --git a/src/data_foundry/curation/site.py b/src/data_foundry/curation/site.py index bf3910ad..44ce844f 100644 --- a/src/data_foundry/curation/site.py +++ b/src/data_foundry/curation/site.py @@ -26,6 +26,7 @@ from data_foundry.curation._paths import records_dir from data_foundry.curation.app import ( STATIC_DIR, + _claims_payload, _docs_payload, _records_payload, _schema_payload, @@ -41,7 +42,8 @@ def build_site( ) -> Path: """Write a read-only static site. - The site is ``index.html`` + ``schema.json`` + ``records.json`` for the grid, the other + The site is ``index.html`` + ``schema.json`` + ``records.json`` (with the claims of the cached claims board, + see :func:`~data_foundry.curation.app._records_payload`) + ``claims.json`` for the grid, the other static pages (``guidelines.html``), and ``docs/manifest.json`` with a copy of every guideline markdown file under ``docs//.md`` (the Start here README as ``docs/start/README.md``). @@ -62,6 +64,7 @@ def build_site( records = records_dir() if directory is None else Path(directory) _write_json(out / "schema.json", _schema_payload()) _write_json(out / "records.json", _records_payload(records)) + _write_json(out / "claims.json", _claims_payload()) # the board the records' Claimed by column shows # Every page gets the read-only flag: index.html drops its write affordances, guidelines.html reads # the frozen docs/manifest.json. Cross-links between the pages are relative, so they resolve whether diff --git a/src/data_foundry/curation/static/index.html b/src/data_foundry/curation/static/index.html index 511db9be..6ce246e0 100644 --- a/src/data_foundry/curation/static/index.html +++ b/src/data_foundry/curation/static/index.html @@ -301,6 +301,7 @@
+ βœ• Clear filters πŸ”— Copy link
@@ -687,7 +688,13 @@

About β€” Data Foundry curation

// the (AI-supplied) suggestion as if it were settled. function rowStatus(d) { const st = rowStatusOf(d); - // A WIP claim names its holder in the tooltip: others leave the record alone until it is released. + // A claim names its holder in the tooltip: others leave the dataset alone until it is released. The live + // board's claim (live_claim) carries the stage, the expiry and the note. + const lc = d.live_claim; + if (lc) { + const until = lc.expired ? `expired ${lc.expires_at.slice(0, 10)}` : `until ${lc.expires_at.slice(0, 10)}`; + return { ...st, t: `${st.t} Β· claimed by ${lc.holder} (${lc.stage}, ${until})` + (lc.note ? `: ${lc.note}` : "") }; + } return d.claimed_by ? { ...st, t: st.t + " Β· claimed by " + d.claimed_by } : st; } function rowStatusOf(d) { @@ -1364,6 +1371,7 @@

About β€” Data Foundry curation

} finally { refreshInFlight = false; } } async function pollVersion() { + refreshClaimsInfo(); try { const v = await fetchVersion(); if (lastVersion === null) lastVersion = v; @@ -1503,6 +1511,7 @@

About β€” Data Foundry curation

cell.restoreOldValue(); return; } + if (f === "claimed_by") return claimEdit(cell, value); // a claim goes to the live board, not only the record try { if (spec.kind !== "number") await registerNew(spec.vocab, asArray(value)); // a number has no vocabulary const updated = await putRecord(cell.getRow().getData().unique_name, { [f]: value }); @@ -1515,6 +1524,54 @@

About β€” Data Foundry curation

} catch (e) { toast("Save failed: " + e.message, true); cell.restoreOldValue(); } } +// ---- claims (the live board) -------------------------------------------------- +// The Claimed by column shows the live claims board (a git branch every curator pushes to), refreshed by the +// server every 20 s. Picking your own name claims the dataset, emptying the cell releases it; the server pushes the +// change to the board and mirrors it into the record. Someone else's name cannot be picked: they claim for themselves. +let CLAIMS = { me: null }; +async function refreshClaimsInfo() { + try { CLAIMS = await (await fetch(STATIC ? "./claims.json" : "/api/claims", { cache: "no-store" })).json(); } + catch (e) { return; } + const el = document.getElementById("claimsInfo"); + const where = `${CLAIMS.remote}/${CLAIMS.branch}`; + if (!CLAIMS.fetched_at) { + el.textContent = "πŸ”’ claims: no board"; + el.title = `The claims board (${where}) has not been fetched yet.` + (CLAIMS.error ? `\nLast fetch failed: ${CLAIMS.error}` : ""); + return; + } + const age = Math.max(0, Math.round(Date.now() / 1000 - CLAIMS.fetched_at)); + const ago = age < 120 ? `${age} s` : age < 7200 ? `${Math.round(age / 60)} min` : `${Math.round(age / 3600)} h`; + el.textContent = `πŸ”’ ${CLAIMS.count} claims` + (CLAIMS.error ? " ⚠" : ""); + el.title = `Claimed by shows the live claims board (${where}), fetched ${ago} ago.` + + (STATIC ? "" : CLAIMS.me ? `\nYou are ${CLAIMS.me}: pick your name in a Claimed by cell to claim, empty it to release.` + : "\nSet your curator name to claim here: git config data-foundry.curator , then restart the server.") + + (CLAIMS.error ? `\nLast fetch failed: ${CLAIMS.error}` : ""); +} +async function claimEdit(cell, value) { + const id = cell.getRow().getData().unique_name; + let action; + if (!value) action = "release"; + else if (value === CLAIMS.me) action = "claim"; + else { + cell.restoreOldValue(); + toast(CLAIMS.me ? `You claim as ${CLAIMS.me}; ${value} claims for themselves.` + : "Set your curator name first: git config data-foundry.curator ", true); + return; + } + try { + const res = await fetch("/api/claims/" + encodeURIComponent(id), { + method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ action }), + }); + const data = await res.json(); + if (data.error) throw new Error(data.error); + cell.getRow().update(toRow(data)); + cell.getRow().reformat(); + syncVersion(); // our own push and record write moved the token + updateCount(); applyFilters(); refreshClaimsInfo(); + toast(action === "claim" ? `Claimed ${id}: on the board for everyone` : `Released ${id}`); + } catch (e) { cell.restoreOldValue(); toast((action === "claim" ? "Claim" : "Release") + " failed: " + e.message, true); } +} + // ---- modal (extra fields) -------------------------------------------------- let modalId = null; function openModal(id) { @@ -1837,6 +1894,7 @@

${escapeHtml(spec.label)} ${multi ? "(multi)" : "(single)"}

table.on("dataSorted", () => syncShareLink()); // a sort alone makes the view worth a link table.on("tableBuilt", () => { applyFiltersFromHash(); applyFilters(); updateCount(); forwardFrozenRowScroll(); }); startLiveRefresh(); // pick up records written outside this tab (agent, other tab, git pull) + refreshClaimsInfo(); watchEditLists(); document.getElementById("search").addEventListener("input", applyFilters); diff --git a/src/data_foundry/curation/store.py b/src/data_foundry/curation/store.py index c3629ba1..f99c19e2 100644 --- a/src/data_foundry/curation/store.py +++ b/src/data_foundry/curation/store.py @@ -34,6 +34,7 @@ import yaml from data_foundry.curation._paths import records_dir +from data_foundry.curation.claims import warn_if_claimed from data_foundry.curation.record import BODY_FIELDS, FIELDS, CurationRecord BODY_SECTION_TITLES: dict[str, str] = {"comments": "Comments", "reference": "Reference"} @@ -96,8 +97,15 @@ def markdown_to_record(text: str) -> CurationRecord: def save_record(record: CurationRecord, directory: str | Path | None = None) -> Path: - """Write ``record`` to ``/.md`` and return the path.""" + """Write ``record`` to ``/.md`` and return the path. + + Writing a record of the standard records directory while another curator holds its claim (the cached claims board, + :mod:`data_foundry.curation.claims`) emits a :class:`~data_foundry.curation.claims.ClaimWarning`; the write still + happens, since claims are advisory. + """ target_dir = Path(directory) if directory is not None else records_dir() + if target_dir.resolve() == records_dir().resolve(): + warn_if_claimed(record.unique_name) target_dir.mkdir(parents=True, exist_ok=True) path = target_dir / f"{record.unique_name}.md" path.write_text(record_to_markdown(record)) diff --git a/tests/test_claims.py b/tests/test_claims.py new file mode 100644 index 00000000..6e064d99 --- /dev/null +++ b/tests/test_claims.py @@ -0,0 +1,353 @@ +"""Tests of the live claims board: a local bare repository plays the remote, two clones play two curators.""" + +from __future__ import annotations + +import datetime as dt +import http.client +import json +import os +import subprocess +import sys +import threading +import warnings +from functools import partial +from http.server import ThreadingHTTPServer +from pathlib import Path + +import pytest +import yaml +from data_foundry.curation import claims +from data_foundry.curation._paths import DATA_FOUNDRY_CURATION_ENV +from data_foundry.curation.app import CurationHandler +from data_foundry.curation.cli import main as cli_main +from data_foundry.curation.record import CurationRecord +from data_foundry.curation.store import load_record, save_record + +HOOK = Path(__file__).resolve().parents[1] / ".claude" / "hooks" / "claims_hook.py" + + +def git(repo: Path, *args: str) -> str: + """Run git in ``repo`` and return its output.""" + out = subprocess.run(["git", "-C", str(repo), *args], capture_output=True, text=True, check=True) # noqa: S603, S607 + return out.stdout.strip() + + +@pytest.fixture +def env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Path]: + """A bare remote, two clones (Lennart's and Andrej's) and a curation root inside Lennart's clone.""" + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(tmp_path / "gitconfig")) + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + for key in ("GIT_AUTHOR_NAME", "GIT_COMMITTER_NAME"): + monkeypatch.setenv(key, "Tester") + for key in ("GIT_AUTHOR_EMAIL", "GIT_COMMITTER_EMAIL"): + monkeypatch.setenv(key, "tester@example.org") + for key in (claims.CURATOR_ENV, claims.CLAIMS_REMOTE_ENV, claims.CLAIMS_BRANCH_ENV): + monkeypatch.delenv(key, raising=False) + remote = tmp_path / "remote.git" + subprocess.run(["git", "init", "--bare", "-q", str(remote)], check=True) # noqa: S603, S607 + repos = {} + for who in ("lennart", "andrej"): + repo = tmp_path / who + repo.mkdir() + git(repo, "init", "-q") + git(repo, "remote", "add", "origin", str(remote)) + repos[who] = repo + root = repos["lennart"] / "curation" + (root / "records").mkdir(parents=True) + (root / "vocabularies.yaml").write_text(yaml.safe_dump({"claimed_by": ["Lennart", "Andrej"]})) + monkeypatch.setenv(DATA_FOUNDRY_CURATION_ENV, str(root)) + claims._CACHE_MEMO.clear() + return {"remote": remote, "root": root, **repos} + + +def _record(root: Path, name: str, **fields: object) -> None: + save_record(CurationRecord(unique_name=name, name=name, **fields), root / "records") + + +def test_claim_list_release_round_trip(env: dict[str, Path]) -> None: + """A claim lands on the branch with its file and README row; a release removes it.""" + out = claims.claim(["a", "b"], stage="add", note="batch 1", holder="Lennart", repo=env["lennart"]) + assert out.changed == ["a", "b"] + assert not out.skipped + board = claims.fetch(env["andrej"]) + assert set(board.claims) == {"a", "b"} + assert board.claims["a"].holder == "Lennart" + assert board.claims["a"].note == "batch 1" + readme = git(env["remote"], "show", "claims:README.md") + assert "## Lennart" in readme + assert "| `a` | add |" in readme + assert "batch 1" in readme + assert claims.release(["a"], holder="Lennart", repo=env["lennart"]).changed == ["a"] + assert set(claims.fetch(env["andrej"]).claims) == {"b"} + assert "release: a (Lennart)" in git(env["remote"], "log", "-1", "--format=%s", "claims") + + +def test_claim_held_by_another_is_skipped_and_release_needs_force(env: dict[str, Path]) -> None: + """Another curator's claim is skipped, and released only with force (logged).""" + claims.claim(["a"], stage="triage", holder="Andrej", repo=env["andrej"]) + out = claims.claim(["a", "b"], stage="add", holder="Lennart", repo=env["lennart"]) + assert out.changed == ["b"] + assert "Andrej (triage)" in out.skipped["a"] + assert claims.release(["a"], holder="Lennart", repo=env["lennart"]).changed == [] + forced = claims.release(["a"], force=True, holder="Lennart", repo=env["lennart"]) + assert forced.changed == ["a"] + assert "forced: a of Andrej" in git(env["remote"], "log", "-1", "--format=%s", "claims") + + +def test_renewing_keeps_since_and_note(env: dict[str, Path]) -> None: + """Claiming again renews: the start date and note stay, stage and expiry move.""" + claims.claim(["a"], stage="add", note="first", ttl_days=1, holder="Lennart", repo=env["lennart"]) + before = claims.fetch(env["lennart"]).claims["a"] + claims.claim(["a"], stage="review", ttl_days=10, holder="Lennart", repo=env["lennart"]) + after = claims.fetch(env["lennart"]).claims["a"] + assert after.claimed_at == before.claimed_at + assert after.note == "first" + assert after.stage == "review" + assert after.expires_at > before.expires_at + renewed = claims.renew(ttl_days=20, holder="Lennart", repo=env["lennart"]) + assert renewed.changed == ["a"] + + +def test_expired_claim_is_taken_only_with_steal(env: dict[str, Path], monkeypatch: pytest.MonkeyPatch) -> None: + """An expired claim of another curator is taken over only with steal, and the log names them.""" + claims.claim(["a"], stage="add", ttl_days=1, holder="Andrej", repo=env["andrej"]) + later = claims._utcnow() + dt.timedelta(days=2) + monkeypatch.setattr(claims, "_utcnow", lambda: later) + refused = claims.claim(["a"], stage="add", holder="Lennart", repo=env["lennart"]) + assert refused.changed == [] + assert "--steal" in refused.skipped["a"] + taken = claims.claim(["a"], stage="add", steal=True, holder="Lennart", repo=env["lennart"]) + assert taken.changed == ["a"] + assert claims.fetch(env["andrej"]).claims["a"].holder == "Lennart" + assert "taken over: a from Andrej" in git(env["remote"], "log", "-1", "--format=%s", "claims") + + +def test_race_on_the_same_dataset_has_one_winner(env: dict[str, Path], monkeypatch: pytest.MonkeyPatch) -> None: + """Both curators read the same tip; Andrej's push lands first, Lennart's is rejected and sees the claim.""" + real_fetch = claims.fetch + raced = [] + + def racing_fetch(repo: Path | None = None) -> claims.Board: + board = real_fetch(repo) + if Path(repo) == env["lennart"] and not raced: + raced.append(True) + claims.claim(["x"], stage="add", holder="Andrej", repo=env["andrej"]) + return board # Lennart decides on the stale board + + monkeypatch.setattr(claims, "fetch", racing_fetch) + out = claims.claim(["x"], stage="add", holder="Lennart", repo=env["lennart"]) + assert out.changed == [] + assert "Andrej" in out.skipped["x"] + assert real_fetch(env["andrej"]).claims["x"].holder == "Andrej" + + +def test_race_on_different_datasets_both_land(env: dict[str, Path], monkeypatch: pytest.MonkeyPatch) -> None: + """A push that lost the race to a claim on another dataset retries and lands.""" + real_fetch = claims.fetch + raced = [] + + def racing_fetch(repo: Path | None = None) -> claims.Board: + board = real_fetch(repo) + if Path(repo) == env["lennart"] and not raced: + raced.append(True) + claims.claim(["x"], stage="add", holder="Andrej", repo=env["andrej"]) + return board + + monkeypatch.setattr(claims, "fetch", racing_fetch) + out = claims.claim(["y"], stage="add", holder="Lennart", repo=env["lennart"]) + assert out.changed == ["y"] + board = real_fetch(env["andrej"]) + assert {n: c.holder for n, c in board.claims.items()} == {"x": "Andrej", "y": "Lennart"} + + +def test_board_is_cached_and_offline_reads_need_no_remote(env: dict[str, Path]) -> None: + """Each fetch caches the board in the git directory; nothing touches the working tree.""" + claims.claim(["a"], stage="add", holder="Lennart", repo=env["lennart"]) + assert claims.cached_board(env["lennart"]).claims["a"].holder == "Lennart" + assert claims.cached_board(env["andrej"]) is None # Andrej has not fetched yet + assert claims.load(env["andrej"], max_age=60).claims["a"].holder == "Lennart" + assert claims.cached_board(env["andrej"]) is not None + cache = json.loads((env["lennart"] / ".git" / claims.CACHE_NAME).read_text()) + assert cache["branch"] == "claims" + # The working tree and the checked-out branch are never touched. + assert git(env["lennart"], "status", "--porcelain") == "?? curation/" + + +def test_missing_branch_is_an_empty_board_and_init_creates_it(env: dict[str, Path]) -> None: + """No branch reads as an empty board; init creates it, and its extra files survive later commits.""" + assert claims.fetch(env["lennart"]).tip is None + claims.init_branch(env["lennart"]) + assert claims.fetch(env["andrej"]).tip is not None + claims.init_branch(env["lennart"], pages_dispatch=True) + workflow = git(env["remote"], "show", f"claims:{claims.PAGES_DISPATCH_PATH}") + assert "gh workflow run pages.yaml" in workflow + claims.claim(["a"], stage="add", holder="Lennart", repo=env["lennart"]) + assert git(env["remote"], "show", f"claims:{claims.PAGES_DISPATCH_PATH}") == workflow # kept by later commits + + +def test_current_curator_needs_a_known_name(env: dict[str, Path], monkeypatch: pytest.MonkeyPatch) -> None: + """The curator name comes from git config or the environment and must be in the vocabulary.""" + with pytest.raises(claims.ClaimsError, match="Name yourself"): + claims.current_curator(env["lennart"]) + git(env["lennart"], "config", claims.CURATOR_GIT_KEY, "Lennart") + assert claims.current_curator(env["lennart"]) == "Lennart" + monkeypatch.setenv(claims.CURATOR_ENV, "Mallory") + with pytest.raises(claims.ClaimsError, match="not in vocabularies"): + claims.current_curator(env["lennart"]) + + +def test_mirror_sets_and_clears_the_records_claims(env: dict[str, Path]) -> None: + """The records' claimed_by and WIP value follow the board, both ways.""" + root = env["root"] + _record(root, "a", suggestion="Yes") + _record(root, "b", suggestion="TBD -> Yes") + _record(root, "c", suggestion="Yes", claimed_by="Andrej", data_foundry_status=["WIP (DF)", "BeyondArena"]) + claims.claim(["a", "b"], stage="add", holder="Lennart", repo=env["lennart"]) + board = claims.fetch(env["lennart"]) + assert set(claims.mirror_to_records(board, write=False)) == {"a", "b", "c"} + changed = claims.mirror_to_records(board) + assert changed == {"a": (None, "Lennart"), "b": (None, "Lennart"), "c": ("Andrej", None)} + a, b, c = (load_record(root / "records" / f"{n}.md") for n in "abc") + assert (a.claimed_by, a.data_foundry_status) == ("Lennart", ["WIP (DF)"]) + assert b.data_foundry_status == ["WIP (Triage)"] # work on a record whose verdict is still open + assert (c.claimed_by, c.data_foundry_status) == (None, ["BeyondArena"]) + assert not {"wip_unclaimed", "claim_without_wip"} & set(a.needs_review + c.needs_review) + assert claims.mirror_to_records(board) == {} + + +def test_import_from_records(env: dict[str, Path]) -> None: + """The records' claims move onto the board in their holders' names, once.""" + root = env["root"] + _record(root, "a", suggestion="Yes", claimed_by="Lennart", data_foundry_status=["WIP (DF)"]) + _record(root, "b", suggestion="TBD -> Yes", claimed_by="Andrej", data_foundry_status=["WIP (Triage)"]) + out = claims.import_from_records(repo=env["lennart"]) + assert out.changed == ["a", "b"] + board = claims.fetch(env["andrej"]) + assert (board.claims["a"].holder, board.claims["a"].stage) == ("Lennart", "add") + assert (board.claims["b"].holder, board.claims["b"].stage) == ("Andrej", "triage") + assert claims.import_from_records(repo=env["lennart"]).changed == [] + + +def test_save_record_warns_only_on_another_curators_claim( + env: dict[str, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + """save_record warns on another curator's claim, writes anyway, and is silent otherwise.""" + root = env["root"] + claims.claim(["a"], stage="add", holder="Andrej", repo=env["lennart"]) + claims.claim(["b"], stage="add", holder="Lennart", repo=env["lennart"]) + monkeypatch.setenv(claims.CURATOR_ENV, "Lennart") + with pytest.warns(claims.ClaimWarning, match="a is claimed by Andrej"): + _record(root, "a") + with warnings.catch_warnings(): + warnings.simplefilter("error", claims.ClaimWarning) + _record(root, "b") # mine + _record(root, "c") # free + save_record(CurationRecord(unique_name="a", name="a"), env["andrej"]) # not the records directory + assert (root / "records" / "a.md").exists() # advisory: the write happened + + +def _hook(payload: dict[str, object]) -> dict[str, object] | None: + proc = subprocess.run( # noqa: S603 + [sys.executable, str(HOOK)], + input=json.dumps(payload), + capture_output=True, + text=True, + check=True, + env=os.environ, + ) + return json.loads(proc.stdout) if proc.stdout.strip() else None + + +def test_hook_warns_before_editing_a_dataset_someone_else_holds( + env: dict[str, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + """The hook warns (never decides) on another curator's dataset and lists the board at session start.""" + repo = env["lennart"] + claims.claim(["a"], stage="add", holder="Andrej", repo=repo) + claims.claim(["b"], stage="add", holder="Lennart", repo=repo) + monkeypatch.setenv(claims.CURATOR_ENV, "Lennart") + + def edit(path: str) -> dict[str, object] | None: + payload = {"hook_event_name": "PreToolUse", "tool_name": "Edit", "cwd": str(repo)} + return _hook({**payload, "tool_input": {"file_path": str(repo / path)}}) + + out = edit("curation/records/a.md") + assert "claimed by Andrej" in out["systemMessage"] + assert "permissionDecision" not in out["hookSpecificOutput"] # warn only, never allow or deny + assert "claimed by Andrej" in out["hookSpecificOutput"]["additionalContext"] + assert "Andrej" in edit("datasets/_dev/tabarena-v0pt2/a/dataset.py")["systemMessage"] + assert edit("curation/records/b.md") is None + assert edit("datasets/_dev/tabarena-v0pt2/c/REVIEW.md") is None + assert edit("src/data_foundry/curation/claims.py") is None + + start = _hook({"hook_event_name": "SessionStart", "cwd": str(repo)}) + context = start["hookSpecificOutput"]["additionalContext"] + assert "Andrej (1): a" in context + assert "Yours (1): b" in context + + +def test_hook_fails_open(tmp_path: Path) -> None: + """Without a board or a known event the hook prints nothing.""" + payload = {"hook_event_name": "PreToolUse", "tool_name": "Edit", "cwd": str(tmp_path)} + assert _hook({**payload, "tool_input": {"file_path": str(tmp_path / "curation/records/a.md")}}) is None + assert _hook({"hook_event_name": "Nonsense"}) is None + + +def test_cli_claim_check_and_sync( + env: dict[str, Path], monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """The CLI claims, checks, syncs and releases, and turns a missing name into one error line.""" + root, repo = env["root"], env["lennart"] + _record(root, "a", suggestion="Yes") + monkeypatch.setenv(claims.CURATOR_ENV, "Lennart") + monkeypatch.setattr(claims, "repo_root", lambda: repo) + assert cli_main(["claims", "claim", "a", "--note", "testing"]) == 0 + assert load_record(root / "records" / "a.md").claimed_by == "Lennart" + assert cli_main(["claims", "claim", "nope"]) == 2 # no record + assert cli_main(["claims", "check", "a", "z"]) == 0 + out = capsys.readouterr().out + assert "a: yours" in out + assert "z: free" in out + assert cli_main(["claims", "sync", "--check"]) == 0 + assert cli_main(["claims", "release", "a"]) == 0 + assert load_record(root / "records" / "a.md").claimed_by is None + monkeypatch.delenv(claims.CURATOR_ENV) + assert cli_main(["claims", "claim", "a"]) == 2 # no curator name: one error line, no traceback + assert "Name yourself" in capsys.readouterr().err + + +def test_dashboard_claims_and_releases_through_the_board(env: dict[str, Path], monkeypatch: pytest.MonkeyPatch) -> None: + """The dashboard shows the board's holder, claims and releases through it, and refuses another's dataset.""" + root, repo = env["root"], env["lennart"] + _record(root, "a", suggestion="Yes") + _record(root, "b", suggestion="TBD -> Yes") + claims.claim(["b"], stage="triage", holder="Andrej", repo=env["andrej"]) + claims.fetch(repo) + monkeypatch.setenv(claims.CURATOR_ENV, "Lennart") + server = ThreadingHTTPServer(("127.0.0.1", 0), partial(CurationHandler, directory=root / "records")) + threading.Thread(target=server.serve_forever, daemon=True).start() + + def call(method: str, path: str, body: dict[str, object] | None = None) -> tuple[int, object]: + conn = http.client.HTTPConnection("127.0.0.1", server.server_address[1], timeout=30) + conn.request(method, path, body=json.dumps(body) if body is not None else None) + res = conn.getresponse() + return res.status, json.loads(res.read()) + + try: + rows = {r["unique_name"]: r for r in call("GET", "/api/records")[1]} + assert rows["b"]["claimed_by"] == "Andrej" # from the board, not the record + assert rows["b"]["live_claim"]["stage"] == "triage" + assert rows["a"]["claimed_by"] is None + assert call("GET", "/api/claims")[1]["me"] == "Lennart" + status, row = call("POST", "/api/claims/a", {"action": "claim"}) + assert status == 200 + assert (row["claimed_by"], row["live_claim"]["stage"]) == ("Lennart", "add") + assert load_record(root / "records" / "a.md").data_foundry_status == ["WIP (DF)"] + assert claims.fetch(env["andrej"]).claims["a"].holder == "Lennart" + status, err = call("POST", "/api/claims/b", {"action": "claim"}) + assert status == 409 + assert "Andrej" in err["error"] + assert call("POST", "/api/claims/a", {"action": "release"})[1]["claimed_by"] is None + assert load_record(root / "records" / "a.md").claimed_by is None + finally: + server.shutdown()