From b66abdcdf55e41c8410a50809c7cbc76eb1da610 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Sat, 20 Jun 2026 18:51:23 +0000 Subject: [PATCH 01/51] feat(ingest): add a real RFC distractor pool for the N-curve Give the RFC scenarios a domain-matched real distractor pool, mirroring the PEP pool (ADR-0004). `ingest.rfcs pool build` scans a pinned RFC range (default 1-400, ~371 RFCs) into RAC-native decision artifacts; provenance.json (the pin: exact set + per-RFC sha256 + supersedes edges) is committed and the bulky corpus is rebuilt on demand and gitignored. - scenarios.loader.load_pool now reads either a PEP (`peps`) or RFC (`rfcs`) provenance, so the crossover can scale N over either pool. - crossover dataset label generalised to `real-decision-pool` (PEP or RFC). Validated offline: the RFC pool (371 decisions, 22 real supersedes edges) drives the curve on the RFC scenarios. A skip-if-absent test loads each committed pool when its corpus has been built. --- decisiongrounding/.github/workflows/ci.yml | 42 + decisiongrounding/.gitignore | 6 + decisiongrounding/CONTRIBUTING.md | 106 + decisiongrounding/EXPLAINER.md | 128 + decisiongrounding/LICENSE | 201 + decisiongrounding/Makefile | 36 + decisiongrounding/NOTICE | 4 + decisiongrounding/README.md | 279 + decisiongrounding/conftest.py | 6 + .../decisions/ADR-0001-harness-foundation.md | 160 + .../ADR-0002-real-corpus-pilot-peps.md | 133 + .../ADR-0003-rac-arm-pep-integration.md | 120 + .../ADR-0004-real-distractor-curve.md | 114 + decisiongrounding/decisions/ADR-template.md | 65 + decisiongrounding/ingest/__init__.py | 8 + decisiongrounding/ingest/peps.py | 352 + decisiongrounding/ingest/rfcs.py | 364 + decisiongrounding/providers/__init__.py | 96 + decisiongrounding/providers/answering.py | 282 + decisiongrounding/providers/base.py | 113 + decisiongrounding/providers/context_dump.py | 25 + decisiongrounding/providers/embedding.py | 163 + .../providers/grounding_format.py | 29 + .../providers/memory_provider.py | 34 + decisiongrounding/providers/naive_rag.py | 97 + decisiongrounding/providers/no_grounding.py | 24 + decisiongrounding/providers/rac.py | 219 + decisiongrounding/pyproject.toml | 39 + decisiongrounding/results/.gitignore | 9 + decisiongrounding/results/.gitkeep | 0 decisiongrounding/results/README.md | 14 + decisiongrounding/runner/__init__.py | 1 + decisiongrounding/runner/cli.py | 387 + decisiongrounding/scenarios/__init__.py | 23 + .../corpus/DG-ADR-RETRY-001.md | 29 + .../corpus/DG-ADR-RETRY-002.md | 34 + .../conflicting_scoped_retry/scenario.json | 27 + decisiongrounding/scenarios/loader.py | 185 + .../corpus/DG-ADR-RL-001.md | 27 + .../corpus/DG-ADR-SEC-001.md | 28 + .../negative_control_cache_ttl/scenario.json | 25 + .../corpus/DG-ADR-LANG-001.md | 35 + .../scenario.json | 28 + .../corpus/DG-ADR-LOG-001.md | 33 + .../simple_adherence_logging/scenario.json | 24 + .../corpus/DG-ADR-DBA-001.md | 33 + .../corpus/DG-ADR-DBA-002.md | 36 + .../superseded_decision/scenario.json | 36 + .../corpus/PEP-0563.md | 715 ++ .../corpus/PEP-0649.md | 1517 +++ .../provenance.json | 40 + .../scenario.json | 40 + .../peps_enum_supersession/corpus/PEP-0354.md | 277 + .../peps_enum_supersession/corpus/PEP-0435.md | 646 ++ .../peps_enum_supersession/provenance.json | 39 + .../peps_enum_supersession/scenario.json | 40 + .../corpus/PEP-0440.md | 1691 ++++ .../provenance.json | 20 + .../scenario.json | 32 + .../corpus/PEP-0513.md | 659 ++ .../corpus/PEP-0571.md | 403 + .../corpus/PEP-0599.md | 334 + .../corpus/PEP-0600.md | 546 + .../provenance.json | 77 + .../peps_manylinux_supersession/scenario.json | 52 + .../corpus/PEP-0345.md | 596 ++ .../corpus/PEP-0566.md | 212 + .../provenance.json | 41 + .../peps_metadata_supersession/scenario.json | 40 + .../corpus/PEP-0622.md | 2308 +++++ .../corpus/PEP-0634.md | 717 ++ .../provenance.json | 39 + .../scenario.json | 40 + .../scenarios_real/peps_pool/.gitignore | 6 + .../scenarios_real/peps_pool/provenance.json | 5970 +++++++++++ .../corpus/PEP-0431.md | 364 + .../corpus/PEP-0615.md | 985 ++ .../provenance.json | 39 + .../peps_timezone_supersession/scenario.json | 40 + .../corpus/PEP-0386.md | 541 + .../corpus/PEP-0440.md | 1695 ++++ .../peps_version_supersession/provenance.json | 39 + .../peps_version_supersession/scenario.json | 40 + .../rfc_rc4_prohibition/corpus/RFC-7465.md | 371 + .../rfc_rc4_prohibition/provenance.json | 18 + .../rfc_rc4_prohibition/scenario.json | 32 + .../corpus/RFC-5246.md | 5863 +++++++++++ .../corpus/RFC-8446.md | 8999 +++++++++++++++++ .../provenance.json | 45 + .../scenario.json | 40 + .../scenarios_real/rfcs_pool/.gitignore | 7 + .../scenarios_real/rfcs_pool/provenance.json | 4281 ++++++++ .../schema/run_result.schema.json | 164 + decisiongrounding/schema/scenario.schema.json | 158 + decisiongrounding/scoring/__init__.py | 13 + decisiongrounding/scoring/crossover.py | 281 + decisiongrounding/scoring/metrics.py | 71 + decisiongrounding/scoring/scorer.py | 107 + decisiongrounding/spec/scenario-taxonomy.md | 70 + decisiongrounding/spec/scoring-rubric.md | 62 + decisiongrounding/tests/test_arms_smoke.py | 98 + decisiongrounding/tests/test_hardening.py | 99 + decisiongrounding/tests/test_ingest_peps.py | 101 + decisiongrounding/tests/test_ingest_rfcs.py | 59 + decisiongrounding/tests/test_real_backends.py | 127 + decisiongrounding/tests/test_real_curve.py | 93 + decisiongrounding/tests/test_real_pilots.py | 59 + .../tests/test_real_scenarios.py | 123 + decisiongrounding/tests/test_schema.py | 68 + decisiongrounding/tests/test_scorers.py | 87 + 110 files changed, 46595 insertions(+) create mode 100644 decisiongrounding/.github/workflows/ci.yml create mode 100644 decisiongrounding/.gitignore create mode 100644 decisiongrounding/CONTRIBUTING.md create mode 100644 decisiongrounding/EXPLAINER.md create mode 100644 decisiongrounding/LICENSE create mode 100644 decisiongrounding/Makefile create mode 100644 decisiongrounding/NOTICE create mode 100644 decisiongrounding/README.md create mode 100644 decisiongrounding/conftest.py create mode 100644 decisiongrounding/decisions/ADR-0001-harness-foundation.md create mode 100644 decisiongrounding/decisions/ADR-0002-real-corpus-pilot-peps.md create mode 100644 decisiongrounding/decisions/ADR-0003-rac-arm-pep-integration.md create mode 100644 decisiongrounding/decisions/ADR-0004-real-distractor-curve.md create mode 100644 decisiongrounding/decisions/ADR-template.md create mode 100644 decisiongrounding/ingest/__init__.py create mode 100644 decisiongrounding/ingest/peps.py create mode 100644 decisiongrounding/ingest/rfcs.py create mode 100644 decisiongrounding/providers/__init__.py create mode 100644 decisiongrounding/providers/answering.py create mode 100644 decisiongrounding/providers/base.py create mode 100644 decisiongrounding/providers/context_dump.py create mode 100644 decisiongrounding/providers/embedding.py create mode 100644 decisiongrounding/providers/grounding_format.py create mode 100644 decisiongrounding/providers/memory_provider.py create mode 100644 decisiongrounding/providers/naive_rag.py create mode 100644 decisiongrounding/providers/no_grounding.py create mode 100644 decisiongrounding/providers/rac.py create mode 100644 decisiongrounding/pyproject.toml create mode 100644 decisiongrounding/results/.gitignore create mode 100644 decisiongrounding/results/.gitkeep create mode 100644 decisiongrounding/results/README.md create mode 100644 decisiongrounding/runner/__init__.py create mode 100644 decisiongrounding/runner/cli.py create mode 100644 decisiongrounding/scenarios/__init__.py create mode 100644 decisiongrounding/scenarios/conflicting_scoped_retry/corpus/DG-ADR-RETRY-001.md create mode 100644 decisiongrounding/scenarios/conflicting_scoped_retry/corpus/DG-ADR-RETRY-002.md create mode 100644 decisiongrounding/scenarios/conflicting_scoped_retry/scenario.json create mode 100644 decisiongrounding/scenarios/loader.py create mode 100644 decisiongrounding/scenarios/negative_control_cache_ttl/corpus/DG-ADR-RL-001.md create mode 100644 decisiongrounding/scenarios/negative_control_cache_ttl/corpus/DG-ADR-SEC-001.md create mode 100644 decisiongrounding/scenarios/negative_control_cache_ttl/scenario.json create mode 100644 decisiongrounding/scenarios/prohibition_language_migration/corpus/DG-ADR-LANG-001.md create mode 100644 decisiongrounding/scenarios/prohibition_language_migration/scenario.json create mode 100644 decisiongrounding/scenarios/simple_adherence_logging/corpus/DG-ADR-LOG-001.md create mode 100644 decisiongrounding/scenarios/simple_adherence_logging/scenario.json create mode 100644 decisiongrounding/scenarios/superseded_decision/corpus/DG-ADR-DBA-001.md create mode 100644 decisiongrounding/scenarios/superseded_decision/corpus/DG-ADR-DBA-002.md create mode 100644 decisiongrounding/scenarios/superseded_decision/scenario.json create mode 100644 decisiongrounding/scenarios_real/peps_annotations_supersession/corpus/PEP-0563.md create mode 100644 decisiongrounding/scenarios_real/peps_annotations_supersession/corpus/PEP-0649.md create mode 100644 decisiongrounding/scenarios_real/peps_annotations_supersession/provenance.json create mode 100644 decisiongrounding/scenarios_real/peps_annotations_supersession/scenario.json create mode 100644 decisiongrounding/scenarios_real/peps_enum_supersession/corpus/PEP-0354.md create mode 100644 decisiongrounding/scenarios_real/peps_enum_supersession/corpus/PEP-0435.md create mode 100644 decisiongrounding/scenarios_real/peps_enum_supersession/provenance.json create mode 100644 decisiongrounding/scenarios_real/peps_enum_supersession/scenario.json create mode 100644 decisiongrounding/scenarios_real/peps_local_version_prohibition/corpus/PEP-0440.md create mode 100644 decisiongrounding/scenarios_real/peps_local_version_prohibition/provenance.json create mode 100644 decisiongrounding/scenarios_real/peps_local_version_prohibition/scenario.json create mode 100644 decisiongrounding/scenarios_real/peps_manylinux_supersession/corpus/PEP-0513.md create mode 100644 decisiongrounding/scenarios_real/peps_manylinux_supersession/corpus/PEP-0571.md create mode 100644 decisiongrounding/scenarios_real/peps_manylinux_supersession/corpus/PEP-0599.md create mode 100644 decisiongrounding/scenarios_real/peps_manylinux_supersession/corpus/PEP-0600.md create mode 100644 decisiongrounding/scenarios_real/peps_manylinux_supersession/provenance.json create mode 100644 decisiongrounding/scenarios_real/peps_manylinux_supersession/scenario.json create mode 100644 decisiongrounding/scenarios_real/peps_metadata_supersession/corpus/PEP-0345.md create mode 100644 decisiongrounding/scenarios_real/peps_metadata_supersession/corpus/PEP-0566.md create mode 100644 decisiongrounding/scenarios_real/peps_metadata_supersession/provenance.json create mode 100644 decisiongrounding/scenarios_real/peps_metadata_supersession/scenario.json create mode 100644 decisiongrounding/scenarios_real/peps_pattern_matching_supersession/corpus/PEP-0622.md create mode 100644 decisiongrounding/scenarios_real/peps_pattern_matching_supersession/corpus/PEP-0634.md create mode 100644 decisiongrounding/scenarios_real/peps_pattern_matching_supersession/provenance.json create mode 100644 decisiongrounding/scenarios_real/peps_pattern_matching_supersession/scenario.json create mode 100644 decisiongrounding/scenarios_real/peps_pool/.gitignore create mode 100644 decisiongrounding/scenarios_real/peps_pool/provenance.json create mode 100644 decisiongrounding/scenarios_real/peps_timezone_supersession/corpus/PEP-0431.md create mode 100644 decisiongrounding/scenarios_real/peps_timezone_supersession/corpus/PEP-0615.md create mode 100644 decisiongrounding/scenarios_real/peps_timezone_supersession/provenance.json create mode 100644 decisiongrounding/scenarios_real/peps_timezone_supersession/scenario.json create mode 100644 decisiongrounding/scenarios_real/peps_version_supersession/corpus/PEP-0386.md create mode 100644 decisiongrounding/scenarios_real/peps_version_supersession/corpus/PEP-0440.md create mode 100644 decisiongrounding/scenarios_real/peps_version_supersession/provenance.json create mode 100644 decisiongrounding/scenarios_real/peps_version_supersession/scenario.json create mode 100644 decisiongrounding/scenarios_real/rfc_rc4_prohibition/corpus/RFC-7465.md create mode 100644 decisiongrounding/scenarios_real/rfc_rc4_prohibition/provenance.json create mode 100644 decisiongrounding/scenarios_real/rfc_rc4_prohibition/scenario.json create mode 100644 decisiongrounding/scenarios_real/rfc_tls_version_supersession/corpus/RFC-5246.md create mode 100644 decisiongrounding/scenarios_real/rfc_tls_version_supersession/corpus/RFC-8446.md create mode 100644 decisiongrounding/scenarios_real/rfc_tls_version_supersession/provenance.json create mode 100644 decisiongrounding/scenarios_real/rfc_tls_version_supersession/scenario.json create mode 100644 decisiongrounding/scenarios_real/rfcs_pool/.gitignore create mode 100644 decisiongrounding/scenarios_real/rfcs_pool/provenance.json create mode 100644 decisiongrounding/schema/run_result.schema.json create mode 100644 decisiongrounding/schema/scenario.schema.json create mode 100644 decisiongrounding/scoring/__init__.py create mode 100644 decisiongrounding/scoring/crossover.py create mode 100644 decisiongrounding/scoring/metrics.py create mode 100644 decisiongrounding/scoring/scorer.py create mode 100644 decisiongrounding/spec/scenario-taxonomy.md create mode 100644 decisiongrounding/spec/scoring-rubric.md create mode 100644 decisiongrounding/tests/test_arms_smoke.py create mode 100644 decisiongrounding/tests/test_hardening.py create mode 100644 decisiongrounding/tests/test_ingest_peps.py create mode 100644 decisiongrounding/tests/test_ingest_rfcs.py create mode 100644 decisiongrounding/tests/test_real_backends.py create mode 100644 decisiongrounding/tests/test_real_curve.py create mode 100644 decisiongrounding/tests/test_real_pilots.py create mode 100644 decisiongrounding/tests/test_real_scenarios.py create mode 100644 decisiongrounding/tests/test_schema.py create mode 100644 decisiongrounding/tests/test_scorers.py diff --git a/decisiongrounding/.github/workflows/ci.yml b/decisiongrounding/.github/workflows/ci.yml new file mode 100644 index 0000000..a9d2f67 --- /dev/null +++ b/decisiongrounding/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI + +# Test the benchmark on every push to main and every pull request. The core +# spine is stdlib-only, so the matrix installs only the dev + schema extras; +# the real-backend arms (anthropic/voyageai) and the rac arm skip cleanly when +# those tools are absent, so no API keys or external CLI are needed here. + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + name: test (py${{ matrix.python-version }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v5 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install (dev + schema extras) + run: | + python -m pip install --upgrade pip + pip install -e ".[dev,schema]" + + - name: Run tests + run: python -m pytest -q + + - name: Smoke the offline demo + run: python -m runner.cli demo diff --git a/decisiongrounding/.gitignore b/decisiongrounding/.gitignore new file mode 100644 index 0000000..4465f39 --- /dev/null +++ b/decisiongrounding/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +build/ +dist/ diff --git a/decisiongrounding/CONTRIBUTING.md b/decisiongrounding/CONTRIBUTING.md new file mode 100644 index 0000000..384e929 --- /dev/null +++ b/decisiongrounding/CONTRIBUTING.md @@ -0,0 +1,106 @@ +# Contributing to decisiongrounding + +This benchmark is only worth running if it is credible to a skeptic who *wants* +the grounding layer to lose. These rules exist to protect that credibility. They +are not optional politeness; a contribution that violates them is rejected even +if the code is perfect. + +## The credibility rules + +### 1. Blind gold-labeling + +Gold labels (the correct verdict, governing decision, prohibited/required +actions) MUST be written **before** and **independently** of running any arm, +and **without** knowing which arm produced which output. If you have already +seen an arm's answer to a task, you may not author or edit that task's gold +label. Label first, run second. + +### 2. No win-only corpora + +Scenarios MUST NOT be hand-authored to favour an arm. Production scenarios are +derived from **real or public ADR sets**, or a design partner's **real +incident** — not invented to make a chosen arm look good. The synthetic worked +scenarios in `scenarios/` exist solely to exercise the harness and are labelled +as such; they are never reported as results. + +A corpus that only contains the kinds of decisions one arm handles well is a +win-only corpus. Include the cases your preferred arm is expected to *lose* +(easy single-decision ties, and the negative control where inventing a +constraint is the failure). + +### 3. Publish losing results + +If the grounded arm ties or loses — including the falsifier in the README +(grounded ≈ `naive_rag` on superseded + prohibition at N ≥ 50) — that result is +published, not buried. Append it to `results/` like any other run. A benchmark +that can only report wins is marketing. + +### 4. Symmetric treatment of arms + +Every arm gets the **same** answering model, the **same** prompt scaffold, and +**one** symmetric opportunity to populate the context window. Do not give your +arm a richer scaffold, a retry, a better model, or a second look. Differences +must live entirely in grounding assembly. Changes that alter the answering model +or scaffold for one arm only will be rejected. + +### 5. Pre-registration is frozen + +`spec/scenario-taxonomy.md` and `spec/scoring-rubric.md` are frozen before +results exist. Changing the taxonomy or rubric is a new **spec version** with a +rationale, not a quiet edit. Never reshape the question after seeing who won. + +### 6. Append-only results + +`results/` is append-only. Never mutate or delete a prior run file. Re-running +produces a new timestamped file. Each run records the pinned model + version + +temperature + seed so it reproduces. + +## Adding a scenario + +1. Create `scenarios//scenario.json` + a `corpus/` of markdown artifacts. +2. Validate against `schema/scenario.schema.json` (`pip install -e .[schema]`). +3. Write the gold label blind (rule 1). State its provenance (rule 2) in the + `rationale`. +4. `make test`. + +### Real vs synthetic scenarios + +`scenarios/` is for **synthetic** worked scenarios that exercise the harness; +they are never reported as results (rule 2). **Real / public-derived** corpora — +the only ones eligible to be reported — live under `scenarios_real/`, kept +physically separate so the default offline demo never blurs the line. + +Real corpus material must be **reproducible**, not transcribed: derive it from a +public source pinned to an immutable revision, and commit the verbatim artifact +plus a `provenance.json` recording the source URL and a content hash. The PEP +pilot does this via `ingest/peps.py` (`build` / `verify`) against a pinned +`python/peps` commit. Excerpting or paraphrasing the source is the cherry-picking +rule 2 forbids — pin, fetch verbatim, and hash instead. + +## Adding an arm + +See "Add an arm" in the README. Then confirm it is scored by the same +deterministic scorer as every other arm — no arm gets a bespoke scorer. + +## Decisions + +Architecture decisions are recorded as RAC-style ADRs in `decisions/` +(`decisions/ADR-template.md`). The repo dogfoods the artifact format the +benchmark studies. If your change makes a non-obvious architectural choice, +record it as an ADR. + +## License and sign-off + +decisiongrounding is licensed under the [Apache License 2.0](LICENSE). By +contributing you agree your contributions are licensed under the same terms. + +Contributions must carry a [Developer Certificate of Origin](https://developercertificate.org/) +sign-off: certify that you wrote the change (or have the right to submit it) +by adding a `Signed-off-by` trailer to each commit. Git adds it for you: + +```bash +git commit -s +``` + +This produces a `Signed-off-by: Your Name ` line matching your +commit author identity. There is no CLA. diff --git a/decisiongrounding/EXPLAINER.md b/decisiongrounding/EXPLAINER.md new file mode 100644 index 0000000..a104da0 --- /dev/null +++ b/decisiongrounding/EXPLAINER.md @@ -0,0 +1,128 @@ +# DecisionGrounding — what it is, how to run it, and why it matters + +A plain-language companion to the README, for anyone — technical or not — who +wants to understand why this benchmark exists before reading the code. + +## The one-sentence version + +DecisionGrounding is a fair, reproducible test of a simple but expensive +question: **when a team uses an AI coding assistant, does giving it a structured +memory of the team's past decisions actually make it follow those decisions +better — or do today's powerful models already handle that on their own?** + +## The background + +Software teams make decisions constantly: "we don't let code talk to the +database directly," "the orders service stays in Go unless an architect signs +off," "logs must be JSON." These decisions are usually written down somewhere +(often as short documents called ADRs — Architecture Decision Records). + +AI coding assistants are now writing real code. The risk is that an assistant, +not knowing or not remembering a past decision, confidently does the wrong +thing — re-introduces a banned pattern, follows a rule the team has since +replaced, or rewrites a service in a new language nobody approved. Real +incidents like this have happened. + +A growing industry says the fix is a **decision-grounding layer**: a structured, +typed memory that feeds the right past decisions to the assistant at the right +moment. That sounds great. But there's a sharp, fair objection: + +> "Modern AI models are so capable, and can read so much text at once, that you +> can just paste all the decision documents in and they'll figure it out. A +> special memory layer adds no durable value." + +**DecisionGrounding exists to settle that argument with evidence instead of +opinion** — and it's deliberately built to be believed even when the answer is +unflattering to the memory-layer idea. + +## How it works (in plain terms) + +It pits several "contestants" against each other on the exact same task, with +the exact same AI model answering. The only thing that changes between +contestants is *how the relevant past decisions are gathered and handed to the +model*: + +- **Paste everything** — dump all the decision documents into the model. (This + is the skeptic's position.) +- **Commodity search** — a standard "find the most similar documents" approach, + no understanding of which decision replaced which. +- **The grounding layer** — structured retrieval that knows, for example, that + Decision B officially replaced Decision A, and hands over B, not A. + +Each contestant gets one fair shot at supplying context; the model then proposes +what it would do, and the benchmark checks — automatically, by inspecting the +proposal — whether it respected the team's decisions. + +**The headline result is a single chart:** how often each contestant follows the +team's decisions as the pile of decisions grows from small to large. The +interesting moment is the *crossover* — the point (if any) where the structured +approach starts to win. The benchmark even states its own kill switch up front: +if the structured approach is no better than plain search on the hardest cases, +the idea is declared dead. That honesty is the point. + +## How a non-technical person can set it up + +You don't need to be an engineer to run the built-in demonstration. You do need +to copy-paste a few commands into a terminal. + +1. **Install Python** (a free programming runtime), version 3.11 or newer, from + python.org. On Mac it's often already there. +2. **Get the code.** Download the project folder (`decisiongrounding`) — your + engineer can share it, or you can download it from the repository as a ZIP + and unzip it. +3. **Open a terminal** (the Terminal app on Mac, or "Command Prompt" / + "PowerShell" on Windows) and move into the folder: + ``` + cd decisiongrounding + ``` +4. **Run the demonstration:** + ``` + make demo + ``` + (If `make` isn't available, use: `python -m runner.cli demo`.) + +That's it. It runs entirely on your machine, needs no accounts, keys, or +internet, and finishes in seconds. It prints a small table and saves a chart +(`results/crossover.svg`) you can open in any web browser. + +**Important honesty note:** this built-in demo uses a *stand-in* for the AI +model so it can run for free, instantly, with no setup. It proves the machinery +works and illustrates the idea — it is **not** a real scientific result. A real +result requires plugging in an actual AI model and real decision documents, +which is a step your engineering team would run (it needs paid API access). The +project is explicit about this distinction everywhere, on purpose. + +## Why it's important + +- **It turns a sales argument into a measurement.** Instead of "trust us, our + memory layer helps," you get a number and a chart, on a frozen, public method + that anyone can re-run. +- **It's built to be trusted by skeptics.** The toughest baselines (just paste + everything; just do ordinary search) are mandatory, not afterthoughts. The + scoring is mostly automatic and mechanical, not a vague "does this look good?" + judgment. The method is locked down *before* any results exist, results are + append-only, and there's a public commitment to publish losing results. +- **It answers a real budget question.** "Should we buy/build a decision-memory + layer, or is our model good enough already?" — and, crucially, "at what size + of decision history does it start to matter?" A small startup and a large + enterprise may get different answers, and the chart shows where the line is. +- **It protects against a real failure mode.** AI assistants confidently doing + things a team already decided against is a genuine source of risk. Measuring + who avoids that — and avoids inventing fake rules that don't exist — is + directly useful. + +## The honest caveats (because credibility is the whole point) + +- It measures *the quality of gathering and handing over the right decisions*. + It does **not** measure whether, in day-to-day production, the assistant + actually bothers to consult its memory at the right moment — that's a separate + question about how the tool is wired into real workflows. +- The free built-in demo simulates the outcome to show the plumbing; the real + verdict requires real AI models and real decision documents. +- Decision sets used for published results must come from real teams or public + sources, with the "right answers" written down *before* seeing which + contestant produced what — so nobody can rig the test. + +If those guardrails hold, the result is believable. If the structured layer +wins, that's a real signal it adds durable value. If it doesn't, that's an +equally real signal — and the benchmark publishes it either way. diff --git a/decisiongrounding/LICENSE b/decisiongrounding/LICENSE new file mode 100644 index 0000000..302587d --- /dev/null +++ b/decisiongrounding/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 Tom Ballard + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/decisiongrounding/Makefile b/decisiongrounding/Makefile new file mode 100644 index 0000000..7b29b30 --- /dev/null +++ b/decisiongrounding/Makefile @@ -0,0 +1,36 @@ +.PHONY: demo test compare pool crossover-real + +# The one working comparison: two real arms, four worked scenarios, offline. +demo: + python -m runner.cli demo + +# Run a single named comparison, e.g. make compare ARMS=context_dump,naive_rag +compare: + python -m runner.cli compare --arms $(ARMS) + +# Build the real distractor pools from their pins (one network pass each: +# ~519 PEPs, ~371 RFCs). Pick the one matching your scenarios' domain. +pool: + python -m ingest.peps pool build + +rfc-pool: + python -m ingest.rfcs pool build + +# Rebuild + verify the real PEP and RFC scenario corpora from their pins. +corpora-verify: + python -m ingest.peps verify --out scenarios_real/peps_version_supersession + python -m ingest.rfcs verify --out scenarios_real/rfc_tls_version_supersession + python -m ingest.rfcs verify --out scenarios_real/rfc_rc4_prohibition + +# Real adherence-vs-N curve over the real PEP pool. Offline by default +# (illustration); add ANSWERING=claude EMBEDDER=voyage:voyage-4-large + keys for +# the real result. NS keeps API spend tunable, e.g. NS=10,50. +crossover-real: + python -m runner.cli demo --scenarios scenarios_real \ + --arms context_dump,naive_rag,no_grounding,rac \ + --distractors real --ns $(or $(NS),10,50,150,300) \ + --answering $(or $(ANSWERING),offline-stub) \ + --embedder $(or $(EMBEDDER),local-hash) + +test: + python -m pytest -q diff --git a/decisiongrounding/NOTICE b/decisiongrounding/NOTICE new file mode 100644 index 0000000..dd23548 --- /dev/null +++ b/decisiongrounding/NOTICE @@ -0,0 +1,4 @@ +decisiongrounding +Copyright 2026 Tom Ballard + +This product is licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/decisiongrounding/README.md b/decisiongrounding/README.md new file mode 100644 index 0000000..62fec90 --- /dev/null +++ b/decisiongrounding/README.md @@ -0,0 +1,279 @@ +# decisiongrounding + +A reproducible benchmark that answers one question: + +> Does a deterministic decision-grounding layer make a coding agent adhere to a +> team's **prior decisions** better than (a) dumping all the decision docs into +> context, (b) commodity RAG over the same docs, or (c) a general-purpose memory +> layer — and at what corpus size does any difference appear? + +It is a **standalone** project. It does not depend on, or import, any specific +grounding implementation; the layer under test is just one arm behind a uniform +adapter. + +## The objection this exists to test + +> "Frontier models plus long context just absorb the decisions, so a persistence +> layer adds no durable value." + +That objection is correct often enough that a benchmark which cannot reproduce +it is worthless marketing. So the threatening baselines are **mandatory**, not +courtesy arms: + +- **`context_dump`** — paste every artifact into the answering model's context. + This is the skeptic's position, implemented faithfully. +- **`naive_rag`** — embeddings + top-k over the same markdown. No typing, no + relationship traversal. + +A grounding layer earns its keep only by beating these — on the scenario types +where it should, at the corpus sizes where it should. + +## The falsifier (stated up front) + +**If the typed/grounded arm ≈ `naive_rag` on superseded + prohibition scenarios +at N ≥ 50, the retrieval thesis is dead.** We publish that result if we find it. +The benchmark is designed to be able to embarrass its sponsor; see +`CONTRIBUTING.md` ("publish losing results"). + +## How the comparison is kept fair + +- **Held-constant answering model.** Every arm feeds context to the *same fixed + answering model* with the *same prompt scaffold*, pinned by model + version + + temperature + seed. Arms differ **only** in how they select and assemble the + grounding context. +- **Symmetric grounding injection.** Each arm gets one equal opportunity to + populate the answering model's context: `context_dump` supplies everything, + `naive_rag` supplies its top-k, the grounded arm supplies its typed retrieval. +- **Deterministic scoring first.** Adherence is scored by structural inspection + of the agent's proposed change (did it propose the prohibited migration? did it + follow the superseded decision?). An LLM judge is a disclosed, unbuilt fallback + — see `spec/scoring-rubric.md`. + +### Symmetric-injection caveat (read this) + +This benchmark isolates **retrieval/assembly quality**: given one symmetric shot +at the context window, which assembly strategy yields better decision-adherence? +It does **not** test whether a pull-based MCP grounding layer actually *gets +consulted* in production — whether an agent invokes the tool at the right moment +is a separate deployment question, out of scope here. Reading a favourable result +as "this layer will fix adherence in production" overstates what was measured. + +## Headline metric and artifact + +- **Headline metric:** decision-adherence rate. +- **Headline artifact:** an adherence-vs-corpus-size curve over + N ∈ {10, 50, 150, 300} with rising conflict density — the story is the + crossover point. +- Also reported: stale-decision rate, false-permit / false-prohibit rate, + per-arm run-to-run variance, and **governing-decision recall** — did the arm's + grounding actually contain the binding decision? Recall is the mechanistic + explanation for why adherence moves (the analog of MemoryBench's Hit@K). There + is deliberately **no** composite score. + +## Run it (offline, no credentials) + +```bash +cd decisiongrounding +make demo # == python -m runner.cli demo +``` + +This runs the two real arms (`context_dump`, `naive_rag`) on the four worked +scenarios with a deterministic **offline** answering model, writes an +append-only report under `results/`, and emits the crossover chart +(`results/crossover.svg`, or `.png` with the `[chart]` extra). + +> The offline answering model is a deterministic stand-in so the spine runs with +> zero credentials. **Its output is a harness illustration, NOT a benchmark +> result.** Real runs swap in the pinned Claude answering model and a real +> embedding backend (`pip install -e .[real]`) on real/public-derived corpora. +> See `decisions/ADR-0001-harness-foundation.md`. + +### Run it for real (pinned model + real retrieval) + +```bash +pip install -e ".[real,schema,chart]" +export ANTHROPIC_API_KEY=... # pinned answering model: claude-opus-4-8 +export VOYAGE_API_KEY=... # real embeddings for naive_rag + +# rac arm additionally needs the `rac` CLI on PATH (or set RAC_BIN) +python -m runner.cli compare \ + --arms context_dump,naive_rag,rac \ + --answering claude \ + --embedder voyage:voyage-4-large \ + --scenarios scenarios/ --seed 0 +``` + +`naive_rag` embeds corpus sections with Voyage's `document` role and the task +with its `query` role, so the RAG baseline uses asymmetric query/document +embeddings the way Voyage intends — a fair, strong baseline, not a strawman. +The default Voyage model is `voyage-4-large` (Voyage's current flagship); +override with `--embedder voyage:`. Each report records the embedder id + +dimension and the installed `anthropic`/`voyageai` versions (`backend_versions`) +so a run says exactly what produced it. + +A real run is expensive, so the runner protects your spend two ways. It +**preflights** the configuration before doing any work — a missing +`ANTHROPIC_API_KEY`, `VOYAGE_API_KEY`, backend package, or `rac` CLI fails fast +with an actionable message instead of part-way through a paid sweep. And it +**streams every completed run** to a durable `results/run--