diff --git a/CLAUDE.md b/CLAUDE.md index 9107ce9..96443ab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,8 +1,15 @@ # CLAUDE.md — perfdigest -Guidance for Claude Code working in this repo. Read `docs/INIT_PROMPT.md` first — it is the -**canonical spec** with all locked decisions and rationale. This file is the short operational -contract; the init prompt is the source of truth. +Guidance for Claude Code working in this repo. **This file is the live contract** — +where it and any other document disagree about the current state, this one wins. + +`docs/INIT_PROMPT.md` is the original kickoff spec, kept deliberately FROZEN as the +record of why the locked design decisions were made (the absence convention, the +digest/capture split, mandatory `format`, mapping-not-inventing). Read it for +rationale, never for current facts: it predates the release and its concrete +numbers are superseded — it says Python 3.11+ (the floor is now 3.10) and describes +three tier-1 tools (there are five, plus two capture-advisory ones), and its claim +that `ncu_report` is not on PyPI stopped being true. Those are history, not bugs. ## What this project is @@ -52,7 +59,7 @@ a folder under `adapters/` with a reader + `mapping.py` + a `backend.py` that `r 5. **Convention** — `server/prompts.py` (usage convention/vocabulary) + `report_store/discovery.py`. 6. Then **`csv_reader.py`** as a second reader on the same contract (extension-ready proof). -Current state: **v1.2.0 — the Development Observatory release (pre-release, in review).** +Current state: **v1.2.0 — the Development Observatory release (RELEASED, on PyPI).** The digest matrix now covers the four feedback channels of the dev loop — RepoState (what changed) -> BuildDigest (does it build) -> CIDigest (does it pass) -> PerfDigest (how fast) — through the SAME seven tools; a backend is a registry row, never a new API surface, and every @@ -112,8 +119,12 @@ machinery via hooks), `adapters/criterion/` (directory-shaped report refs). ## Environment & commands -- **Python 3.11+**, packaged with **uv** (src-layout). Entry points: `perfdigest-mcp` - / `perfdigest` = `perfdigest.server.app:main`. Repo root **is** the package now. +- **Python 3.10+** (the floor `pyproject.toml` declares and CI tests), packaged with + **uv** (src-layout). CI runs 3.10/3.11/3.12/3.13/3.14 and free-threaded 3.14t on + Linux, macOS and Windows; the one known gap is Windows + 3.14t, where `mcp`'s + `pywin32` dependency has no cp314t wheel yet (that cell is an allowed failure). + Entry points: `perfdigest-mcp` / `perfdigest` = `perfdigest.server.app:main`. + Repo root **is** the package now. ```bash uv sync --extra dev # base + pytest; all pure-Python readers work with no GPU diff --git a/docs/clients.md b/docs/clients.md index 15c34a0..3ff1ed5 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -35,8 +35,10 @@ claude mcp add perfdigest --scope user -- uvx perfdigest-mcp ``` Then inject the workflow convention once per session via the MCP prompt -`perfdigest_usage`, and call `/mcp` to confirm the three digest tools + -`platform_capabilities` + `suggest_profile_command` are listed. +`perfdigest_usage`, and call `/mcp` to confirm all seven tools are listed: the +five digest tools (`summarize_report`, `list_kernels`, `get_metrics`, +`compare_metrics`, `expand`) plus `platform_capabilities` and +`suggest_profile_command`. ## OpenAI Codex diff --git a/src/perfdigest/core/compare.py b/src/perfdigest/core/compare.py index bc2c98f..43b5e58 100644 --- a/src/perfdigest/core/compare.py +++ b/src/perfdigest/core/compare.py @@ -60,7 +60,15 @@ def build_comparison( "index_a": unit_a.index, "index_b": unit_b.index, "format": format, + # Both sides carry a domain, and they can legitimately differ: one + # trace holds framework_op AND gpu_kernel units, and comparing two + # units of one report is a supported flow. Reporting only A's domain + # would assert something untrue of half the payload and drop a fact + # the reader already knows. ``domain`` stays as A's value so existing + # callers keep working, with the honest pair alongside it. "domain": unit_a.domain, + "domain_a": unit_a.domain, + "domain_b": unit_b.domain, "raw_ref_a": unit_a.raw_ref, "raw_ref_b": unit_b.raw_ref, "sign_convention": "delta = b - a", diff --git a/src/perfdigest/server/tools.py b/src/perfdigest/server/tools.py index 161eea0..9c6c6f5 100644 --- a/src/perfdigest/server/tools.py +++ b/src/perfdigest/server/tools.py @@ -1,8 +1,9 @@ """The MCP tools — a thin shell over registry + core + platform, no business logic. Two tiers (see ``server/prompts.py``): - * Tier 1 — read/digest: ``list_kernels`` / ``get_metrics`` / ``expand`` work for - ANY registered ``format`` on ANY host. A report's origin is irrelevant. + * Tier 1 — read/digest: ``summarize_report`` / ``list_kernels`` / ``get_metrics`` + / ``compare_metrics`` / ``expand`` work for ANY registered ``format`` on ANY + host. A report's origin is irrelevant. * Tier 2 — capture advisory: ``platform_capabilities`` / ``suggest_profile_command`` are platform-verified and refuse a capture that cannot run here. diff --git a/tests/test_measured_findings.py b/tests/test_measured_findings.py index 78e3943..ee9e3de 100644 --- a/tests/test_measured_findings.py +++ b/tests/test_measured_findings.py @@ -173,6 +173,69 @@ def test_coverage_uses_exact_summation(): assert summary["coverage_pct_of_total_duration"] == pytest.approx(100.0) +# --- #31: a comparison must not hide the B-side domain ---------------------- + + +def test_compare_reports_both_domains_when_they_differ(fixtures_dir): + """One torch trace holds framework_op AND gpu_kernel units, and comparing + two units of one report is supported — so the payload must carry both. + """ + report = str(fixtures_dir / "torch_sample.torch-trace.json") + units = tools.list_kernels(report, FMT) + framework = next(u for u in units if u["domain"] == "framework_op") + kernel = next(u for u in units if u["domain"] == "gpu_kernel") + + out = tools.compare_metrics( + report, report, FMT, kernel=f"#{framework['index']}", kernel_b=f"#{kernel['index']}" + ) + + assert out["domain_a"] == "framework_op" + assert out["domain_b"] == "gpu_kernel" + assert out["domain"] == out["domain_a"] # retained alias, A-side as before + + +def test_compare_domain_alias_holds_for_same_domain_units(fixtures_dir): + report = str(fixtures_dir / "torch_sample.torch-trace.json") + kernels = [u for u in tools.list_kernels(report, FMT) if u["domain"] == "gpu_kernel"] + out = tools.compare_metrics( + report, report, FMT, kernel=f"#{kernels[0]['index']}", kernel_b=f"#{kernels[1]['index']}" + ) + + assert out["domain"] == out["domain_a"] == out["domain_b"] == "gpu_kernel" + + +# --- #30: live guidance must describe the shipped contract ------------------ + + +def test_live_guidance_matches_the_shipped_surface(): + """The docs an agent reads must not describe an older contract.""" + import re + from pathlib import Path + + from perfdigest.server.app import mcp + + repo = Path(__file__).resolve().parents[1] + claude_md = (repo / "CLAUDE.md").read_text(encoding="utf-8") + clients_md = (repo / "docs" / "clients.md").read_text(encoding="utf-8") + pyproject = (repo / "pyproject.toml").read_text(encoding="utf-8") + + assert "pre-release, in review" not in claude_md + + # The runtime floor the live guide states must be the one the package + # declares. Asserted by extraction rather than by a literal, because the + # guide legitimately QUOTES the frozen init prompt's superseded floor while + # explaining that it is history. + floor = re.search(r'requires-python\s*=\s*"[>=<]*\s*(\d+\.\d+)"', pyproject).group(1) + assert f"**Python {floor}+**" in claude_md, f"CLAUDE.md does not state the {floor} floor" + + # Every shipped tool name should appear in the client setup guide, so a + # reader verifying their install checks for the surface that exists. + import asyncio + + for name in sorted(t.name for t in asyncio.run(mcp.list_tools())): + assert name in clients_md, f"{name} missing from docs/clients.md" + + # --- #25 / #27: caveats must actually reach the agent -----------------------