diff --git a/.claude/agents/fsgg-critic-best.md b/.claude/agents/fsgg-critic-best.md index 94ec4e47d..5c13c3c02 100644 --- a/.claude/agents/fsgg-critic-best.md +++ b/.claude/agents/fsgg-critic-best.md @@ -31,3 +31,14 @@ nonmaterial observations remain in the review. No routine minted identity, marke critic chain or repair-phase round reset is required. Only explicit human selection loads `pnext-item/references/independent-review.md` and its scoped heavyweight contract. Use this role only where actual runtime/model routing and delegation are authorized. + +When the review is posted as a PR comment or a submitted PR review, end it with exactly one final line, +alone, with nothing after it: + +`VERDICT: APPROVE @ <40-hex head>` or `VERDICT: REQUEST_CHANGES @ <40-hex head>` + +The head is the full 40-character SHA of the exact candidate you reviewed, the same head the review text +names. A rejection is `REQUEST_CHANGES`. One review comment carries one verdict line. A re-review at a new +head is a new comment with its own line; never edit an earlier comment to change it. The line is a +machine-readable convenience for census and delivery tooling, not an authority: it grants no merge or +publication authority and does not replace the findings or any strict review-decision record. diff --git a/.claude/agents/fsgg-critic-normal.md b/.claude/agents/fsgg-critic-normal.md index 5f187d1f4..1fad87803 100644 --- a/.claude/agents/fsgg-critic-normal.md +++ b/.claude/agents/fsgg-critic-normal.md @@ -31,3 +31,14 @@ nonmaterial observations remain in the review. No routine minted identity, marke critic chain or repair-phase round reset is required. Only explicit human selection loads `pnext-item/references/independent-review.md` and its scoped heavyweight contract. Use this role only where actual runtime/model routing and delegation are authorized. + +When the review is posted as a PR comment or a submitted PR review, end it with exactly one final line, +alone, with nothing after it: + +`VERDICT: APPROVE @ <40-hex head>` or `VERDICT: REQUEST_CHANGES @ <40-hex head>` + +The head is the full 40-character SHA of the exact candidate you reviewed, the same head the review text +names. A rejection is `REQUEST_CHANGES`. One review comment carries one verdict line. A re-review at a new +head is a new comment with its own line; never edit an earlier comment to change it. The line is a +machine-readable convenience for census and delivery tooling, not an authority: it grants no merge or +publication authority and does not replace the findings or any strict review-decision record. diff --git a/.github/workflows/usage-census-selftest.yml b/.github/workflows/usage-census-selftest.yml new file mode 100644 index 000000000..019b6be12 --- /dev/null +++ b/.github/workflows/usage-census-selftest.yml @@ -0,0 +1,38 @@ +# Offline smoke test of the usage and review census tools (tools/usage-census). +# +# The tools produced the #4823 and #4639 census reports from one worker's scratch directory. They are +# published so the next extension window is a documented run (tools/usage-census/README.md). This suite +# builds a synthetic work directory and drives every offline tool through its real entry point, so that +# a change to the run-assignment, review-scoring, deduplication, redaction or evidence-cap rules cannot +# land unobserved. It reads no credential and calls no network: stdlib only. +# +# The two `paths:` lists are identical on purpose; `check-paths-coherence.py` rule (a) reds a +# divergence. +name: usage-census-selftest + +on: + pull_request: + paths: + - "tools/usage-census/**" + - "tests/usage-census/**" + - ".github/workflows/usage-census-selftest.yml" + push: + branches: [main] + paths: + - "tools/usage-census/**" + - "tests/usage-census/**" + - ".github/workflows/usage-census-selftest.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + fixture: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + + - name: Run the usage and review census smoke test + run: python3 tests/usage-census/run.py diff --git a/tests/usage-census/run.py b/tests/usage-census/run.py new file mode 100755 index 000000000..d7d49d213 --- /dev/null +++ b/tests/usage-census/run.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Offline smoke test of tools/usage-census. + +Builds a tiny synthetic work directory outside the repository (two member PRs, one in a private +repository; five runs; two transcripts) and runs every offline tool over it in order: match, reqcheck, +review, joinkeys, usage_summary, assemble, export and report. The network tools (population, project, +prdetail, required_rules, required, issues, runs, jobs) and the reader build are only compiled here: +they need GitHub and a .NET build, which a smoke test must not. + +Properties asserted: exclusive run assignment, push-only merge runs, the force-pushed-head run kept +out of the totals, the zero-second rule for a job that ends before it starts, V/unsampled scoring from +the adjudication table, unknown (never pass) for an unreadable repository, deduplication of a resumed +session's response, the branch join, redaction of the private member in both published files, and the +evidence size cap. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TOOLS = ROOT / "tools" / "usage-census" +sys.dont_write_bytecode = True + +for tool in sorted(TOOLS.glob("*.py")): + compile(tool.read_text(encoding="utf-8"), str(tool), "exec") + +HEAD_A, HEAD_B = "a" * 40, "b" * 40 +MERGE_A, MERGE_B = "c" * 40, "d" * 40 +OLD_A, GONE = "e" * 40, "f" * 40 + + +def check(condition: bool, message: str) -> None: + if not condition: + raise SystemExit("FAIL " + message) + print("ok " + message) + + +def tool(work: Path, name: str, *args: str) -> str: + env = dict(os.environ, CENSUS_WORK=str(work), PYTHONDONTWRITEBYTECODE="1", PYTHONPATH=str(TOOLS)) + proc = subprocess.run([sys.executable, str(TOOLS / name), *args], env=env, capture_output=True, text=True) + if proc.returncode: + raise SystemExit(f"FAIL {name} exited {proc.returncode}\n{proc.stdout}\n{proc.stderr}") + return proc.stdout + + +def write(path: Path, value) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value) if not isinstance(value, str) else value, encoding="utf-8") + + +def pr(repo, number, head, merge, branch, created, merged, body): + return {"repo": repo, "number": number, "title": f"title {number}", "url": "u", "createdAt": created, "mergedAt": merged, "updatedAt": merged, + "headRefName": branch, "headRefOid": head, "baseRefName": "main", "mergeCommit": {"oid": merge}, "body": body} + + +def run(i, repo, event, sha, branch, created): + return {"id": i, "name": "w", "event": event, "status": "completed", "conclusion": "success", "head_sha": sha, "head_branch": branch, + "run_attempt": 1, "created_at": created, "updated_at": created, "run_started_at": created, "path": "w.yml", "repo": repo, "prs": []} + + +def job(start, end): + return {"id": 1, "run_attempt": 1, "name": "j", "status": "completed", "conclusion": "success", "started_at": start, "completed_at": end} + + +def csv_row(ts, session, response, total, model="claude-opus-5-5"): + return f"{ts},t,{session},{session},turn,{response},Anthropic,{model},high,1,61526e94,unknown,unknown,1,{total},0,0,0,0,{total}" + + +CSV_HEAD = "timestamp,task,session_id,thread_id,turn_id,response_id,provider,model,effort,runtime_version,coordination_version,sdd_version,contracts_version,ledger_schema,input,cached_input,cache_write_input,output,reasoning,total" + +with tempfile.TemporaryDirectory(prefix="usage-census-smoke-") as tmp: + work = Path(tmp) / "work" + data = work / "data" + marker = lambda head: f"" + prs = [pr("FS-GG/.github", 1, HEAD_A, MERGE_A, "routine/a", "2026-10-23T01:00:00Z", "2026-10-23T02:00:00Z", "body " + marker(HEAD_A)), + pr("FS-GG/Private.Repo", 2, HEAD_B, MERGE_B, "routine/b", "2026-10-23T02:30:00Z", "2026-10-23T03:00:00Z", "body")] + write(work / "config.json", {"org": "FS-GG", "project": 3, "issue": "FS-GG/.github#0", "task": "t", "lo": "2026-10-23T00:00:00Z", "hi": "2026-10-24T00:00:00Z", + "coord_version": "61526e94", "reader": "unused", "receipt_store": str(Path(tmp) / "rs"), "claude_home": "unused", "codex_home": "unused"}) + write(data / "prs.json", {"window": ["a", "b"], "repos": [".github", "Private.Repo"], "prs": prs}) + write(data / "repos.json", {"private": ["FS-GG/Private.Repo"]}) + no_review = {"reviews": {"totalCount": 0, "nodes": []}, "closingIssuesReferences": {"nodes": []}} + write(data / "prdetail.json", { + "FS-GG/.github#1": {"commits": {"totalCount": 2, "nodes": [{"commit": {"oid": OLD_A}}, {"commit": {"oid": HEAD_A}}]}, + "comments": {"totalCount": 1, "nodes": [{"databaseId": 7, "author": {"login": "x"}, "createdAt": "t", "updatedAt": "t", + "body": f"Independent review (fsgg-critic-normal) of exact head {HEAD_A}: APPROVE\n\nVERDICT: APPROVE @ {HEAD_A}"}]}, **no_review}, + "FS-GG/Private.Repo#2": {"commits": {"totalCount": 1, "nodes": [{"commit": {"oid": HEAD_B}}]}, "comments": {"totalCount": 1, "nodes": [{"databaseId": 8, "author": {"login": "x"}, "createdAt": "t", "updatedAt": "t", "body": f"earlier head\n\nVERDICT: REQUEST_CHANGES @ {OLD_A}"}]}, **no_review}}) + runs = [run(1, "FS-GG/.github", "pull_request", HEAD_A, "routine/a", "2026-10-23T01:10:00Z"), + run(2, "FS-GG/.github", "push", MERGE_A, "main", "2026-10-23T02:01:00Z"), + run(3, "FS-GG/.github", "schedule", MERGE_A, "main", "2026-10-23T02:02:00Z"), + run(4, "FS-GG/.github", "pull_request", GONE, "routine/a", "2026-10-23T01:30:00Z"), + run(5, "FS-GG/Private.Repo", "pull_request", HEAD_B, "routine/b", "2026-10-23T02:40:00Z"), + run(6, "FS-GG/.github", "pull_request", HEAD_A, "routine/a", "2026-10-23T02:30:00Z")] # created after A merged: excluded + write(data / "runs.json", runs) + out = tool(work, "match.py") + match = json.loads((data / "match.json").read_text()) + extra = json.loads((data / "match_extra.json").read_text()) + check(match["FS-GG/.github#1"] == {"pre": [1], "post": [2]}, "PR-phase run and push-on-merge run assigned; the scheduled run on the merge SHA excluded") + check(match["FS-GG/Private.Repo#2"]["pre"] == [5], "the private member's run is assigned") + check(6 not in match["FS-GG/.github#1"]["pre"] and 6 not in [i for v in extra.values() for i in v["pre"]], "a run created after the member's merge is excluded") + check(extra == {"FS-GG/.github#1": {"pre": [4], "post": []}}, "a run on a force-pushed head is kept out of the match") + + t = "2026-10-23T01:" + jobs = {1: [job(t + "10:00Z", t + "20:00Z"), job(t + "20:00Z", t + "25:00Z"), job(t + "30:00Z", t + "29:00Z")], 2: [job(t + "10:00Z", t + "11:40Z")], 5: [job(t + "10:00Z", t + "13:20Z")]} + write(data / "jobs.main.0.jsonl", "".join(json.dumps({"run_id": i, "total": len(j), "jobs": j}) + "\n" for i, j in jobs.items())) + write(data / "jobs.extra.0.jsonl", json.dumps({"run_id": 4, "total": 1, "jobs": [job(t + "10:00Z", t + "10:50Z")]}) + "\n") + + write(data / "required_rules.json", {"FS-GG/.github": {"readable": True, "contexts": ["gate"], "notes": []}, "FS-GG/Private.Repo": {"readable": False, "contexts": [], "notes": ["403"]}}) + gate = {"statusCheckRollup": {"contexts": {"pageInfo": {"hasNextPage": False}, "nodes": [{"__typename": "CheckRun", "name": "gate", "conclusion": "SUCCESS", "status": "COMPLETED", "completedAt": "2026-10-23T01:59:00Z"}]}}} + write(data / "required.json", {"FS-GG/.github#1": {"oid": HEAD_A, **gate}, "FS-GG/Private.Repo#2": {"oid": HEAD_B, "error": "403", "statusCheckRollup": None}}) + tool(work, "reqcheck.py") + req = json.loads((data / "reqcheck.json").read_text()) + check(req["FS-GG/.github#1"]["result"] == "pass" and req["FS-GG/Private.Repo#2"]["result"] == "unknown", "required checks: pass where readable, unknown (never pass) where not") + + out = tool(work, "review.py") + raw = json.loads((data / "review_raw.json").read_text()) + check(bool(raw["FS-GG/.github#1"]["found"]) and not raw["FS-GG/Private.Repo#2"]["found"], "review finder proposes the member with a head-bound verdict only") + check(raw["FS-GG/.github#1"]["verdict_lines"] == [{"source": "C7", "verdict": "APPROVE", "head": HEAD_A, "final_head": True}], "a final VERDICT line is recorded exactly and bound to the final head") + check(raw["FS-GG/Private.Repo#2"]["verdict_lines"] == [{"source": "C8", "verdict": "REQUEST_CHANGES", "head": OLD_A, "final_head": False}], "a VERDICT line naming another head is not bound to the final head") + write(data / "adjudication.json", {"V": {"FS-GG/.github#1": "APPROVE (C7)"}, "E": {}, "A": [], "NOTE": {}}) + + claude = Path(tmp) / "claude" + transcript = claude / "s1.jsonl" + transcript.parent.mkdir(parents=True) + lines = [{"type": "assistant", "sessionId": "S", "gitBranch": b, "message": {"id": m}} for m, b in (("m1", "routine/a"), ("m2", "main"), ("m3", "main"), ("m4", "routine/a"), ("m5", "routine/b"))] + transcript.write_text("".join(json.dumps(x) + "\n" for x in lines), encoding="utf-8") + ud = data / "usage" + (ud / "claude").mkdir(parents=True) + rows = [csv_row("2026-10-23T01:20:00.000Z", "S", "m1", 100), csv_row("2026-10-23T01:21:00.000Z", "S", "m2", 50), csv_row("2026-10-23T01:22:00.000Z", "S", "m3", 25), + csv_row("2026-10-23T02:30:00.000Z", "S", "m4", 10), csv_row("2026-10-23T02:45:00.000Z", "S", "m5", 7)] + (ud / "claude" / "aaaa.csv").write_text(CSV_HEAD + "\n" + "\n".join(rows) + "\n") + # A resumed session copies m2 into a second transcript: it must be counted once. + (ud / "claude" / "bbbb.csv").write_text(CSV_HEAD + "\n" + rows[1] + "\n") + refused = "fsgg-coord-engine: telemetry usage: no Claude responses matched the requested turn/time window" + (ud / "index.tsv").write_text(f"claude\taaaa\t0\t{transcript}\t\nclaude\tbbbb\t0\t{claude}/s2.jsonl\t\nclaude\tcccc\t1\t{claude}/subagents/s3.jsonl\t{refused}\n") + (claude / "s2.jsonl").write_text(json.dumps({"type": "assistant", "sessionId": "S", "gitBranch": "main", "message": {"id": "m2"}}) + "\n") + tool(work, "joinkeys.py") + tool(work, "usage_summary.py") + usage = json.loads((data / "usage_summary.json").read_text()) + check(usage["totals"]["claude"]["total"] == 192 and usage["dupes"] == 1, "usage totals deduplicate the copied response by provider response ID") + check({a[1]: a[3]["total"] for a in usage["attributed"]} == {"FS-GG/.github#1": 100, "FS-GG/Private.Repo#2": 7}, "only branch-joined responses at or before the merge are attributed (a later response on the same branch is not)") + + write(data / "project3.json", {"total": 5, "fetched": 5, "closed": [{"content": {"__typename": "Issue", "number": 9, "repository": {"nameWithOwner": "FS-GG/.github"}, "closedAt": "2026-10-23T05:00:00Z"}, "fieldValueByName": {"name": "Done"}}]}) + write(data / "issues.json", [{"repo": "FS-GG/.github", "number": 9, "title": "t", "closedAt": "2026-10-23T05:00:00Z", "stateReason": "COMPLETED", "projectStatus": "Done", "timelineItems": {"nodes": []}}]) + tool(work, "assemble.py") + census = json.loads((data / "census.json").read_text()) + a = census["actions"] + check(a["pr_phase_runs"] == 2 and a["pr_phase_job_seconds"] == 600 + 300 + 200 and a["merge_commit_job_seconds"] == 100, "Actions totals sum job wall time over matched runs") + check(a["jobs_with_completed_before_started_counted_as_zero"]["pr_phase"] == 1, "a job that ends before it starts counts as 0 s and is counted") + check(a["force_pushed_head_pr_runs_not_in_totals"]["runs"] == 1 and a["force_pushed_head_pr_runs_not_in_totals"]["job_seconds"] == 50, "the force-pushed-head run is disclosed outside the totals") + classes = {r["member"]: r["review_class"] for r in census["prs"]} + check(classes["FS-GG/.github#1"] == "V" and classes["private-repository-member-1"] == "N", "review scoring follows the adjudication table") + ev = Path(tmp) / "evidence.json" + tool(work, "export.py", str(ev)) + md = Path(tmp) / "report.md" + tool(work, "report.py", str(md)) + text = md.read_text() + ev.read_text() + text += (data / "census.json").read_text() + check(all(s not in text for s in ("Private.Repo", HEAD_B[:8], MERGE_B[:8], "title 2")), "the private member is redacted in the report, the evidence and the census document (attribution included)") + check("private-repository-member-1" in json.dumps(census["usage"]["attributed"]), "a private member's token attribution is kept under its placeholder") + capped = subprocess.run([sys.executable, str(TOOLS / "export.py"), str(Path(tmp) / "capped.json")], env=dict(os.environ, CENSUS_WORK=str(work), CENSUS_EVIDENCE_CAP="100"), capture_output=True, text=True) + check(capped.returncode != 0 and not (Path(tmp) / "capped.json").exists(), "the evidence cap refuses an oversized document") + inside = subprocess.run([sys.executable, str(TOOLS / "match.py")], env=dict(os.environ, CENSUS_WORK=str(ROOT)), capture_output=True, text=True) + check(inside.returncode != 0 and "inside a git work tree" in inside.stderr + inside.stdout, "a work directory inside a repository is refused") + check(len(ev.read_bytes()) < 65536 and json.loads(ev.read_text())["prs"]["rows"][0][0] == ".github#1", "evidence JSON is valid, tabular and under the 64 KiB cap") + check("| .github#1 |" in md.read_text() and "private-repository-member-1" in md.read_text(), "the member tables are rendered, private members last") + # The continuation template formats with --previous (here: this segment's own evidence stands in). + md2 = Path(tmp) / "report2.md" + tool(work, "report.py", str(md2), "--previous", str(ev), "--template", str(TOOLS / "continuation-report.example.md")) + text2 = md2.read_text() + check("## Coverage of the settled population" in text2 and "{n_" not in text2 and "{all_" not in text2, "the continuation template formats against the computed figures") +print("usage-census smoke: all checks passed") diff --git a/tools/usage-census/README.md b/tools/usage-census/README.md new file mode 100644 index 000000000..5b222efcf --- /dev/null +++ b/tools/usage-census/README.md @@ -0,0 +1,136 @@ +# Usage and review census tools + +The scripts behind the settled usage and review census reports +([#4823](../../docs/reports/2026-10-23-usage-census-4823.md), +[#4639 continuation](../../docs/reports/2026-10-24-usage-census-4639.md)), published so the next +extension window does not depend on one worker's scratch directory. They are read-only against +GitHub and the local transcripts. They open no receiver, telemetry store or UTEL database, and they +add no store of their own: everything they write goes under one work directory. + +What a census records per window `[lo, hi)`: every PR merged in the organization and every Project 3 +item closed; per PR the route marker, the review evidence (an exact-head independent verdict, or +**unsampled** with a reason), the required-check conclusion, Actions job wall time (PR phase and +merge-commit pushes) and open-to-merge elapsed time (labelled as including overlapping queues); and +native Claude Code and Codex token counters from the coordinator host through the strict reader. +Billed cost, other hosts, per-reviewer tokens and anything the provider does not support stay +**Unknown**, never zero. No attribution, baseline or cost is invented, and no transcript is recaptured. + +## Requirements + +- `gh` authenticated for the organization (the census credential), `python3` 3.10 or later (standard + library only), and `dotnet` with the SDK in `global.json` to build the reader. +- The two-CPU budget: every tool is single-threaded except `prdetail.py` and `required.py` (two threads). + Run them under `taskset -c 0,1` on a shared host. +- GraphQL and REST are the shared account budget (5,000 points and 5,000 calls an hour). `jobs.py` + throttles while the remaining REST quota is below `--floor`; leave headroom for other agents. +- A **work directory** outside every repository checkout and outside the temporary directory. It holds + raw per-response usage rows and must never be committed or published. + +## Build the reader once + +``` +tools/usage-census/build-reader.sh "$HOME/.cache/usage-census/reader" +``` + +This builds the CLI at the pinned commit (`61526e94`, PR #4802), outside the repository. A built +reader is not checked in: it is a 67 MB .NET output tied to one commit. A newer commit may be passed +as the second argument; record it in the report, because the counters come from the reader alone. +The script was run for this publication: the rebuilt reader's `fsgg-coord-engine.dll` differs byte-wise from +the census-time build (build metadata), and on a sample transcript its counter columns are identical to the +census-time output (only the `source` transcript hash differs, because the live transcript had grown). + +## Run the next extension window + +Choose the new window so that it starts exactly where the last report ended (`lo` equals the previous +`hi`) and ends at a fixed cutoff `hi` far enough in the past that the last merge's Actions runs have +completed. Then: + +``` +export WORK="$HOME/.cache/usage-census/2026-10-25" # new directory +export CENSUS_WORK="$WORK" +T=tools/usage-census +mkdir -p "$WORK" && cp $T/config.example.json "$WORK/config.json" # edit: issue, task, lo, hi, paths + +# 1. Freeze the population (member lists are digested; print the digests into the report) +python3 $T/population.py # data/prs.json, data/repos.json +python3 $T/project.py # data/project3.json +python3 $T/prdetail.py # commits, comments, submitted reviews, closing references +python3 $T/issues.py # closers of the closed Project 3 items + +# 2. Required checks (rules are read now; changes during the window are Unknown) +python3 $T/required_rules.py +python3 $T/required.py +python3 $T/reqcheck.py + +# 3. Actions machine time +python3 $T/runs.py # all runs from the earliest member's creation day to the end day +python3 $T/match.py # exclusive run-to-member assignment; writes match_extra.json +python3 $T/jobs.py --pace 0.8 --shard 0 --of 2 & # two shards at most +python3 $T/jobs.py --pace 0.8 --shard 1 --of 2 & +wait +python3 $T/jobs.py --extra --pace 0.8 # force-pushed-head runs, disclosed outside the totals + +# 4. Native usage (read-only; the receipt store is outside the work directory and every repository) +python3 $T/collect.py +python3 $T/joinkeys.py +python3 $T/usage_summary.py + +# 5. Review evidence: adjudicate BY HAND from the full text +python3 $T/review.py --dump > "$WORK/review-dump.txt" +$EDITOR "$WORK/data/adjudication.json" # format: adjudication.example.json + +# 6. Assemble, then publish +python3 $T/assemble.py +python3 $T/export.py "docs/reports/evidence/-usage-census-.json" +python3 $T/report.py "docs/reports/-usage-census-.md" \ + --previous docs/reports/evidence/2026-10-24-usage-census-4639.json \ + --template $T/continuation-report.example.md +``` + +`config.json` keys: `org`, `project` (Project 3 number), `issue` (label written into the evidence), +`task` (reader task label), `lo` and `hi` (ISO UTC, `hi` exclusive), `coord_version`, `reader` (the +built `fsgg-coord-engine`; `CENSUS_READER` overrides it), `receipt_store`, `claude_home`, `codex_home`. +A step that failed is re-run alone; `jobs.py` resumes and skips runs it already holds. + +### Review adjudication + +`review.py` only proposes candidates. A PR scores **V** only when one review statement or one dedicated +review comment (a submitted GitHub review counts) names an independent review or critic, gives an +explicit verdict, and binds it to the **final** head SHA in the same statement. Everything else is +unsampled: **A** (verdict stated, head not named), **E** (verdict at an earlier head), or **N** (no +verdict in the record; a `NOTE` records a review mention without one). Read the whole comment, not an +excerpt: a second statement in the same comment can carry the verdict. A critic comment ending in the +machine-readable line `VERDICT: APPROVE @ ` can be matched exactly; older records cannot. + +### Before publishing + +- Run `python3 scripts/check-telemetry-git-safety.py --base origin/main`. Evidence under a `usage` or + `telemetry` path is capped at 64 KiB; `export.py` refuses to write more. +- Private-repository members are redacted by `assemble.py` and listed last. Do not publish their + numbers, titles, SHAs, dates or review comment ids. +- `--previous` covers one earlier segment. For a third segment, combine the earlier figures by hand in the + prose. Re-read the prose of the template against the data. The figures are computed; the sentences about the + window, the retention boundary and the coordinator-runtime changes are not. +- One PR with one independent review and green required CI, merged through `tools/routine-delivery.py`. + Link the owning issue without closing it (`Refs`). + +## Smoke test + +``` +python3 tests/usage-census/run.py +``` + +It builds a synthetic work directory and runs every offline tool, asserting exclusive run assignment, +push-only merge runs, the force-pushed-head disclosure, the zero-second job rule, review scoring, +unknown (never pass) for an unreadable repository, response deduplication, the branch join, private +redaction and the evidence cap. The network tools and the reader build are only compiled there. + +## Known limits + +- Job time is Jobs API wall time, not billed minutes. A job that completes before it starts counts as 0 s. +- Runs on force-pushed heads are not in the totals (`match_extra.json`, disclosed separately). +- Tokens join a member only through the `gitBranch` on a Claude Code response record; coordinator-run + Codex usage and anything on `main` is unattributed. +- Retained transcripts bound the native usage: Codex rollouts exist from 2026-10-19 and Claude Code + transcripts from 2026-10-22 on the coordinator host. Pruned transcripts are Unknown. +- Review independence is as the record asserts it: one account posts every record. diff --git a/tools/usage-census/adjudication.example.json b/tools/usage-census/adjudication.example.json new file mode 100644 index 000000000..7ba989d9d --- /dev/null +++ b/tools/usage-census/adjudication.example.json @@ -0,0 +1,16 @@ +{ + "V": { + "FS-GG/.github#4854": "APPROVE (C6385548575)", + "FS-GG/.github#4855": "APPROVE (submitted review, state COMMENTED)", + "FS-GG/FS.GG.Coordination#1055": "approved (PR body)" + }, + "E": { + "FS-GG/.github#4825": "REJECT recorded at an earlier head; no verdict at the final head" + }, + "A": [ + "FS-GG/FS.GG.Templates#762" + ], + "NOTE": { + "FS-GG/FS.GG.Coordination#1057": "independent exact-source review stated; no explicit verdict" + } +} diff --git a/tools/usage-census/assemble.py b/tools/usage-census/assemble.py new file mode 100755 index 000000000..735bdd295 --- /dev/null +++ b/tools/usage-census/assemble.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Join every input into one census document (data/census.json) with private members redacted. + +Inputs under data/: prs, prdetail, match, match_extra, runs, jobs.main.*.jsonl, jobs.extra.*.jsonl, +reqcheck, issues, project3, usage_summary, repos, adjudication (hand-made, see adjudication.example.json). +""" +import collections +import glob +import hashlib +import json +import os +import re + +from census_common import DATA, cfg, dump, load, pr_digest, ts + +C = cfg() +pop, det, match, extra = load("prs.json"), load("prdetail.json"), load("match.json"), load("match_extra.json") +reqc, issues, pj, usage = load("reqcheck.json"), load("issues.json"), load("project3.json"), load("usage_summary.json") +adj = load("adjudication.json") +PRIVATE = set(load("repos.json")["private"]) + + +def read_jobs(family): + jobs = {} + for fn in sorted(glob.glob(os.path.join(DATA, f"jobs.{family}.*.jsonl"))): + for line in open(fn, encoding="utf-8"): + o = json.loads(line) + if "error" not in o or o["run_id"] not in jobs: + jobs[o["run_id"]] = o + return jobs + + +jobs = read_jobs("main") +YES, EARLIER, UNBOUND, NOTE = dict(adj["V"]), dict(adj["E"]), list(adj["A"]), dict(adj["NOTE"]) +for k, v in EARLIER.items(): + NOTE[k] = v +for k in UNBOUND: + NOTE[k] = "independent verdict stated; head not named" + + +def jobsecs(ids, source=None): + source = source or jobs + s, n, miss, incomplete, negative = 0.0, 0, 0, 0, 0 + for i in ids: + j = source.get(i) + if not j or "error" in j: + miss += 1 + continue + for x in j["jobs"]: + if x["started_at"] and x["completed_at"]: + d = (ts(x["completed_at"]) - ts(x["started_at"])).total_seconds() + if d > 0: + s += d + elif d < 0: + negative += 1 # counted as 0 s; the report states the rule + n += 1 + else: + incomplete += 1 + return s, n, miss, incomplete, negative + + +attr = {a[1]: a for a in usage["attributed"]} +rows = [] +for p in pop["prs"]: + k = f'{p["repo"]}#{p["number"]}' + m = match[k] + marker = re.findall(r"", p["body"] or "") + rv = "independent verdict at exact head: " + YES[k] if k in YES else "unsampled (" + NOTE.get(k, "no independent verdict in PR record") + ")" + rcat = "V" if k in YES else ("E" if k in EARLIER else ("A" if k in UNBOUND else "N")) + pre, post = jobsecs(m["pre"]), jobsecs(m["post"]) + el = (ts(p["mergedAt"]) - ts(p["createdAt"])).total_seconds() + a = attr.get(k) + route = (f"routine-development/v1 operation={marker[0][1]} head={'final' if marker[0][0] == p['headRefOid'] else 'stale'}" if marker + else (", ".join(sorted(set(re.findall(r"", "", t or "", flags=re.S) + + +def units(text): + return [u for u in re.split(r"\n\s*\n|\n(?=[-*] )|\n(?=#)", text) if u.strip()] + + +def nearest(u, pos): + best = None + for m in VERD.finditer(u): + d = abs(m.start() - pos) + if best is None or d < best[0]: + best = (d, m.group(1)) + return best[1] if best else None + + +VLINE = re.compile(r"^VERDICT: (APPROVE|REQUEST_CHANGES) @ ([0-9a-f]{40})\s*$", re.M) +out = {} +for p in pop["prs"]: + key = f'{p["repo"]}#{p["number"]}' + head = p["headRefOid"].lower() + pref = head[:7] + d = det[key] + sources = [("C%s" % c["databaseId"], strip(c["body"])) for c in d["comments"]["nodes"]] + [("body", strip(p["body"]))] + sources += [("R%s" % r["submittedAt"], strip(r["body"])) for r in d["reviews"]["nodes"]] + found = [] + for kind, text in sources: + for u in units(text): + i = u.lower().find(pref) + if i >= 0 and REV.search(u) and VERD.search(u): + found.append({"source": kind, "verdict": nearest(u, i), "excerpt": re.sub(r"\s+", " ", u)[:260]}) + # Machine-readable verdict lines (critic convention): exact, and bound to the final head or not. + vlines = [{"source": kind, "verdict": m.group(1), "head": m.group(2), "final_head": m.group(2) == head} + for kind, text in sources for m in VLINE.finditer(text)] + mention = bool(re.search(r"independent[^.\n]{0,80}review|fsgg-critic|critic", (p["body"] or "") + "\n".join(c["body"] or "" for c in d["comments"]["nodes"]), re.I)) + out[key] = {"found": found, "review_mentioned": mention, "submitted_reviews": d["reviews"]["totalCount"], "verdict_lines": vlines} +dump("review_raw.json", out, indent=1) +print(collections.Counter(("hit" if v["found"] else ("mention" if v["review_mentioned"] else "none")) for v in out.values())) +print("members with submitted GitHub reviews:", [k for k, v in out.items() if v["submitted_reviews"]]) +if "--dump" in sys.argv[1:]: + for p in pop["prs"]: + key = f'{p["repo"]}#{p["number"]}' + d = det[key] + print("=====", key, "final head", p["headRefOid"]) + for r in d["reviews"]["nodes"]: + print(f' [submitted review {r["state"]} {r["submittedAt"]} commit {(r["commit"] or {}).get("oid")}] {r["body"]}') + for kind, text in [("body", strip(p["body"]))] + [("C%s" % c["databaseId"], strip(c["body"])) for c in d["comments"]["nodes"]]: + for u in units(text): + if KW.search(u): + print(f" [{kind}] {u.strip()}") diff --git a/tools/usage-census/runs.py b/tools/usage-census/runs.py new file mode 100755 index 000000000..d6ce4cfd9 --- /dev/null +++ b/tools/usage-census/runs.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""List every workflow run created from the earliest member's creation day to the window end day. + +Only repositories that have members are listed. The runs listing caps a filtered query at 1000 rows, +so a day at the cap is re-listed in hourly slices, which must each stay below the cap. +""" +import datetime + +from census_common import cfg, dump, gh_json, load + +C = cfg() +pop = load("prs.json") +repos = sorted({p["repo"] for p in pop["prs"]}) +first = min(p["createdAt"] for p in pop["prs"])[:10] +last = C["hi"][:10] +if C.get("run_from"): + first = C["run_from"] + + +def listing(repo, rng): + runs, page = [], 1 + while True: + d = gh_json("api", f"repos/{repo}/actions/runs?created={rng}&per_page=100&page={page}") + runs += d["workflow_runs"] + if len(d["workflow_runs"]) < 100: + return d["total_count"], runs + page += 1 + + +allruns = {} +for repo in repos: + day = datetime.date.fromisoformat(first) + end = datetime.date.fromisoformat(last) + while day <= end: + total, found = listing(repo, f"{day}T00:00:00Z..{day}T23:59:59Z") + if total >= 1000 or len(found) < total or len(found) >= 1000: + found = [] + for hour in range(24): + t2, r2 = listing(repo, f"{day}T{hour:02d}:00:00Z..{day}T{hour:02d}:59:59Z") + assert len(r2) == t2 and t2 < 1000, (repo, day, hour, t2, len(r2)) + found += r2 + for x in found: + allruns[x["id"]] = {k: x[k] for k in ("id", "name", "event", "status", "conclusion", "head_sha", "head_branch", "run_attempt", "created_at", "updated_at", "run_started_at", "path")} | {"repo": repo, "prs": [p["number"] for p in x.get("pull_requests") or []]} + day += datetime.timedelta(days=1) +dump("runs.json", list(allruns.values())) +print(len(allruns)) diff --git a/tools/usage-census/usage_summary.py b/tools/usage-census/usage_summary.py new file mode 100755 index 000000000..fe4924a41 --- /dev/null +++ b/tools/usage-census/usage_summary.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Deduplicate reader output by provider response ID and join Claude Code responses to members. + +A resumed Claude Code session copies earlier records into its new transcript, so totals must be +deduplicated by provider response ID across files. A response joins a member only when the `gitBranch` +on its own record equals the member's unique head branch and its timestamp is no later than the merge. +Everything else is unattributed. +""" +import collections +import csv +import json + +from census_common import SESSION_COL, data_path, dump, load + +idx = [line.rstrip("\n").split("\t") for line in open(data_path("usage/index.tsv"), encoding="utf-8")] +jk, pop = load("joinkeys.json"), load("prs.json") +bybranch = {} +for p in pop["prs"]: + bybranch.setdefault(p["headRefName"], []).append(p) +F = ["input", "cached_input", "cache_write_input", "output", "reasoning", "total"] +files, refusals = collections.Counter(), collections.Counter() +tot, attr = collections.defaultdict(collections.Counter), collections.defaultdict(collections.Counter) +models = collections.defaultdict(collections.Counter) +resp, attrresp, sessions = collections.Counter(), collections.Counter(), collections.defaultdict(set) +seen, dupes = set(), 0 +for rt, fid, rc, path, err in idx: + cls = "subagent" if "/subagents/" in path else "main" + files[(rt, cls, "ok" if rc == "0" else "refused")] += 1 + if rc != "0": + e = err.split("fsgg-coord-engine: telemetry usage: ")[1] if "usage: " in err else err + e = "no completed usage record (totals withheld)" if "no completed usage record" in e else e + refusals[(rt, cls, e.strip())] += 1 + continue + for r in csv.DictReader(open(data_path(f"usage/{rt}/{fid}.csv"), encoding="utf-8")): + key = (rt, r["response_id"]) + if key in seen: + dupes += 1 + continue + seen.add(key) + sessions[rt].add(r[SESSION_COL]) + resp[rt] += 1 + for f in F: + tot[rt][f] += int(r[f] or 0) + models[(rt, r["model"])][f] += int(r[f] or 0) + models[(rt, r["model"])]["responses"] += 1 + target = None + if rt == "claude": + m = jk[fid]["msgs"].get(r["response_id"]) + if m and m["branch"] in bybranch and len(bybranch[m["branch"]]) == 1: + p = bybranch[m["branch"]][0] + if r["timestamp"] <= p["mergedAt"]: + target = f'{p["repo"]}#{p["number"]}' + if target: + attrresp[(rt, target)] += 1 + for f in F: + attr[(rt, target)][f] += int(r[f] or 0) +res = {"files": [[*k, v] for k, v in sorted(files.items())], "refusals": [[*k, v] for k, v in sorted(refusals.items())], + "dupes": dupes, "responses": dict(resp), "sessions": {k: len(v) for k, v in sessions.items()}, "totals": {k: dict(v) for k, v in tot.items()}, + "models": [[*k, dict(v)] for k, v in sorted(models.items())], + "attributed": [[*k, attrresp[k], dict(v)] for k, v in sorted(attr.items())]} +dump("usage_summary.json", res, indent=1) +print(json.dumps({k: res[k] for k in ("dupes", "responses", "sessions", "totals")}, indent=1))