From ff2f5e5544a06ad798c48075e5ab03a46ba9280d Mon Sep 17 00:00:00 2001 From: ahrazzle <38987266+ahrazzle@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:49:50 -0600 Subject: [PATCH 1/2] Add explicit evidence bindings for nominal scalar experiments --- README.md | 2 +- .../2026-11-06-E-evidence-bindings/report.md | 88 +++++++++++ .../results.json | 113 ++++++++++++++ auto-research/README.md | 2 + docs/evidence_bindings.md | 56 +++++++ docs/model_contracts.md | 7 + scripts/validate_evidence_bindings.py | 73 +++++++++ tests/test_evidence_bindings.py | 145 ++++++++++++++++++ vh/graph/evidence.py | 110 +++++++++++++ 9 files changed, 595 insertions(+), 1 deletion(-) create mode 100644 auto-research/2026-11-06-E-evidence-bindings/report.md create mode 100644 auto-research/2026-11-06-E-evidence-bindings/results.json create mode 100644 docs/evidence_bindings.md create mode 100644 scripts/validate_evidence_bindings.py create mode 100644 tests/test_evidence_bindings.py create mode 100644 vh/graph/evidence.py diff --git a/README.md b/README.md index 6e41556..6f522e6 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ research tracks, and how to publish work under [`auto-research/`](auto-research/ - **Rohan Arun** ([@rohanarun](https://github.com/rohanarun)) — creator and maintainer; direction and design. - **Claude** (Anthropic) — implemented and validated the engines under the above direction, in pair with the maintainer. -- **OpenAI Codex** — graph interface contracts, clearance reference validation, and conserved reaction/intervention experiments, under the direction of [@ahrazzle](https://github.com/ahrazzle). +- **OpenAI Codex** — graph interface contracts, clearance reference validation, conserved reaction/intervention experiments, and explicit evidence bindings, under the direction of [@ahrazzle](https://github.com/ahrazzle). - **Claude** (Anthropic), Foundation-0 fork — SBML import with exact conservation and positive-mass consistency invariants. New contributors (human or agent): add yourself here in your pull request. See diff --git a/auto-research/2026-11-06-E-evidence-bindings/report.md b/auto-research/2026-11-06-E-evidence-bindings/report.md new file mode 100644 index 0000000..ea41e04 --- /dev/null +++ b/auto-research/2026-11-06-E-evidence-bindings/report.md @@ -0,0 +1,88 @@ +# Explicit evidence bindings for nominal scalar experiments + +- **Track:** E, reusable graph/model interfaces +- **Author:** gpt-forge (OpenAI Codex, Astra), under @ahrazzle's direction +- **Date:** 2026-11-06 +- **Branch:** `research/evidence-bindings-20261106` +- **Code:** `vh/graph/evidence.py`, `scripts/validate_evidence_bindings.py`, `tests/test_evidence_bindings.py` + +## Question + +Can a scalar experiment retain the evidence's complete context, source locator, +limits, and uncertainty while refusing unsupported inputs before model execution? +Can the same path perform an independently checkable positive synthetic example? + +## Method + +An opt-in immutable evidence record binds a fully specified Quantity to a +PortSpec. Recorded estimates and explicitly enabled synthetic fixtures can bind; +conjectures, unresolved conflicts and missing values cannot. Every supplied +quantity/unit/basis/context must match exactly. No conversions, semantic aliases, +parameter inference or evidence joins are implemented. The model registry checks +its own contract again before dispatch. Structured input audit records travel +through the registry's existing provenance lineage. + +Uncertainty descriptions and optional paired endpoints are preserved. They are +not sampled or propagated. Outputs are nominal values; no confidence or +prediction interval is claimed. Synthetic fixtures remain artificial. No +parameters were fitted and no biological observations were used as a referee. + +## Validation + +The independent mathematical authority is the exact rational identity relating +elimination rate, plasma concentration and clearance. Three synthetic cases use +rate/concentration pairs (3, 1/2), (7, 1/4), and (9, 3/4) in mg/h and mg/L. +The executable floating-point model produces 6, 28 and 12 L/h respectively. + +| Check | Result | Authority | Error / tolerance | +| --- | --- | --- | --- | +| Three nominal clearances | 6, 28, 12 L/h | Exact rational rate / concentration | Maximum 0 L/h; tolerance 1e-12 L/h | +| Missing kinetic inputs | Four refused | Checked-in ledger P015, P016, P019, P020 | All four lack estimates; software consistency check only | +| Interface tests | 32 new passing cases | Explicit boundary examples and exact arithmetic | No statistical accuracy claim | + +The arithmetic error has no statistical error bar: these are fixed synthetic +fixtures, not sampled observations. Source uncertainty remains attached to the +inputs. Tests also check refusal before callback execution, incompatible +contexts/units/bases, immutable context snapshots, complete audit retention, +explicit synthetic permission, and inability to weaken registered contracts. + +The CLI output in `results.json` pins the exact ledger bytes with SHA-256 and +preserves the full selected source rows. Those rows have not been re-verified +against literature in this contribution. The result is a reproducible missing +input finding, not an independent scientific validation of their content. + +## Limits + +A caller can mislabel an input or omit a relevant context field. This interface +cannot discover that mistake or certify scientific adequacy. It is opt-in and +does not prevent use of the underlying registry. CI/IQR endpoints are preserved +metadata, not hard bounds; only nominal values are checked against port bounds. +No output uncertainty or probability distribution is produced. Asma's qualitative +presence, presentation, recognition and injury assertions remain domain-owned; +they cannot be coerced to scalars through this interface. + +The four ovarian requirements are necessary examples, not a sufficient model +specification. A future non-null ledger row still needs reviewed explicit semantic +binding; the readiness script deliberately never authorizes execution. There is +no patient simulation, clinical inference, data acquisition, or treatment advice. +Independent scientific review and held-out biological outcomes remain absent. + +## Reproduce + +```bash +PYTHONPATH=. python3 scripts/validate_evidence_bindings.py +python3 -m pytest -q tests/test_evidence_bindings.py tests/test_model_contracts.py tests/test_reaction_experiments.py tests/test_ovarian_parameter_ledger.py -m 'not network' +node web/test_engines.mjs +``` + +Observed environment: Python 3.14.6, pytest 9.1.1, NumPy 2.5.4, SciPy 1.18.1. +Focused pre-serialization check: 129 passed, 1 network test deselected; final evidence-only suite: 32 passed. JavaScript engine checks +passed against checked-in fixtures, including worst plasma-profile relative error +4.8e-13. Final full local suite: 217 passed, 15 skipped, 1 network test deselected. Skips reflect unavailable OpenMM, PySCF and libRoadRunner; those independent-engine checks were not run. No dependency installation was performed. + +## References + +- Existing `vh/graph/contracts.py` interfaces and exact clearance identity tests. +- `data/clinical/ovarian_hgsoc_parameter_ledger.json`, unchanged source ledger. +- `docs/research-foundation.md` and `docs/research-question-template.md` for the + separation of evidence assertions, modeling assumptions and scientific validation. diff --git a/auto-research/2026-11-06-E-evidence-bindings/results.json b/auto-research/2026-11-06-E-evidence-bindings/results.json new file mode 100644 index 0000000..466a73d --- /dev/null +++ b/auto-research/2026-11-06-E-evidence-bindings/results.json @@ -0,0 +1,113 @@ +{ + "ledger_sha256": "37695eac4ed4c854b86c7e37a7e7324a462f69bf33c2c90fa2000db4955c0478", + "ovarian_readiness": { + "interpretation": "Readiness for these four necessary inputs only, not a complete model specification. KNOWN describes source access, not applicability; prevalence and hazard ratios are not kinetic rates.", + "ready": false, + "requirements": [ + { + "ledger_record": { + "citation": { + "doi": "10.1073/pnas.2026663118", + "first_author_surname": "Gu", + "title": "Computational modeling of ovarian cancer dynamics suggests optimal strategies for therapy and screening", + "year": 2021 + }, + "derivation": null, + "id": "P015", + "label": "MISSING", + "limits": "Gu et al. simulation parameters not copied into this ledger.", + "locator": "No single untreated HGSOC doubling time tabulated for ledger extraction", + "population": "Newly diagnosed HGSOC before systemic therapy", + "quantity": "Untreated high-grade serous ovarian cancer volumetric doubling time", + "range": null, + "units": null, + "value": null + }, + "ready": false, + "reason": "missing estimate", + "record_id": "P015", + "required_role": "tumor_growth" + }, + { + "ledger_record": { + "citation": { + "doi": "10.1016/j.jtbi.2006.02.001", + "first_author_surname": "Kohandel", + "title": "Mathematical modeling of ovarian cancer treatments: Sequencing of surgery and chemotherapy", + "year": 2006 + }, + "derivation": null, + "id": "P016", + "label": "MISSING", + "limits": "See P021 for qualitative prior art.", + "locator": "Model uses fixed-fraction surgery operator; no patient-derived numeric fraction", + "population": "Ovarian cancer patients with quantitative debulking fractions", + "quantity": "Proportional intraoperative tumor retention fraction used in sequencing models (empirical estimate)", + "range": null, + "units": null, + "value": null + }, + "ready": false, + "reason": "missing estimate", + "record_id": "P016", + "required_role": "surgical_retention" + }, + { + "ledger_record": { + "citation": { + "doi": "10.1073/pnas.2026663118", + "first_author_surname": "Gu", + "title": "Computational modeling of ovarian cancer dynamics suggests optimal strategies for therapy and screening", + "year": 2021 + }, + "derivation": null, + "id": "P019", + "label": "MISSING", + "limits": "Add when a model contract consumes PK parameters.", + "locator": "Not applicable: sequence experiments use dimensionless burden units", + "population": "Virtual-human ovarian treatment-sequence graph experiments", + "quantity": "Population carboplatin clearance for whole-body PBPK coupling in ovarian treatment-sequence models", + "range": null, + "units": null, + "value": null + }, + "ready": false, + "reason": "missing estimate", + "record_id": "P019", + "required_role": "carboplatin_clearance" + }, + { + "ledger_record": { + "citation": { + "doi": "10.1073/pnas.2026663118", + "first_author_surname": "Gu", + "title": "Computational modeling of ovarian cancer dynamics suggests optimal strategies for therapy and screening", + "year": 2021 + }, + "derivation": null, + "id": "P020", + "label": "MISSING", + "limits": "Add when a model contract consumes PK parameters.", + "locator": "Not applicable: scalar chemotherapy operators in sequence experiments", + "population": "Virtual-human ovarian treatment-sequence graph experiments", + "quantity": "Population paclitaxel central volume and clearance for sequence models", + "range": null, + "units": null, + "value": null + }, + "ready": false, + "reason": "missing estimate", + "record_id": "P020", + "required_role": "paclitaxel_volume_and_clearance" + } + ] + }, + "schema_version": 1, + "synthetic_validation": { + "cases": 3, + "interpretation": "Nominal synthetic arithmetic only; no output uncertainty or biological validation", + "maximum_absolute_error_L_per_h": 0.0, + "passed": true, + "tolerance_L_per_h": 1e-12 + } +} diff --git a/auto-research/README.md b/auto-research/README.md index 41d7577..6725884 100644 --- a/auto-research/README.md +++ b/auto-research/README.md @@ -27,3 +27,5 @@ auto-research/ | 2026-11-05 | E | [Clearance reference validation](2026-11-05-E-clearance-reference/report.md) | Exact organ mass balance and linear ODE integral | Mixed references caught; reported sum 17.104405 L/h, dose/AUC 17.103070 L/h | OpenAI Codex / @ahrazzle | | 2026-11-05 | E (partial) | [Reaction and intervention experiments](2026-11-05-E-reaction-experiments/report.md) | Exact kinetics/conservation/commutativity; independent DOP853 | Maximum trajectory error 1.479e-11 mol/L; no biological validation | OpenAI Codex / @ahrazzle | | 2026-11-05 | E | [SBML reaction networks as verified graphs](2026-11-05-E-sbml-import/report.md) | Exact rational algebra; libRoadRunner as independent SBML engine | 3 CC0 BioModels: exact conservation laws, trajectories ≤7e-10, steady states ≤2e-12; positive-mass invariant flags 8 of 49 surveyed models with exact certificates | Claude (Anthropic) | + +- [2026-11-06: Explicit evidence bindings](2026-11-06-E-evidence-bindings/report.md): nominal scalar execution with preserved context and uncertainty; exact synthetic mass balance and missing kinetic-input report. diff --git a/docs/evidence_bindings.md b/docs/evidence_bindings.md new file mode 100644 index 0000000..708eca8 --- /dev/null +++ b/docs/evidence_bindings.md @@ -0,0 +1,56 @@ +# Evidence at an executable scalar boundary + +`vh.graph.evidence` adds an optional checked path from an explicitly quantified +record to `ModelRegistry`. It does not read papers, infer model parameters, or +certify an observation's truth or applicability. The original scalar registry +remains available; this is an opt-in interface, not a security enforcement layer. + +`EvidenceInput` holds a record ID, status, an optional `Quantity`, source, exact +locator, limits, and `Uncertainty`. The quantity retains its complete context, +quantity name, unit, reference basis, and prior provenance. These fields must be +chosen and reviewed by the caller. A `recorded` status means a recorded estimate, +not a validated model, causal effect, or clinical finding. + +- `recorded` estimates may bind when every `PortSpec` field and bound matches. +- `synthetic` estimates additionally require `allow_synthetic=True`. +- `conjecture`, `missing`, and `conflicting` records cannot bind. Missing and + conflicting records carry no selected point estimate. The caller must resolve + evidence outside this interface; changing a label alone is not validation. +- Qualitative assertions have no scalar representation here. Protein presence, + antigen presentation, receptor recognition, and neural injury cannot substitute + for one another. Asma owns those assertion distinctions and evidence rules. + +`execute_evidence(registry, edge, evidence, specs)` first binds every input and +then delegates to the registry, which independently rechecks the registered model +contract. Supplying weaker binding specifications cannot override that contract. +A rejected input never invokes the registered callback. As with the registry, +callbacks should be pure: output validation cannot undo callback side effects. + +The complete input audit is serialized under an `evidence:` provenance entry and +therefore accompanies every returned output through the registry's existing +lineage union. Returned values are nominal scenario outputs. The recorded input +uncertainty is not output uncertainty. A source CI, IQR, or unquantified range is +neither a probability distribution nor an asserted hard domain; there is no +sampling, independence assumption, interval propagation, or automatic conversion. + +## Reproduce a positive example and the current missing-input result + +```bash +PYTHONPATH=. python3 scripts/validate_evidence_bindings.py +python3 -m pytest -q tests/test_evidence_bindings.py +``` + +The script executes three explicitly artificial clearance scenarios, independently +checked against exact rational rate/concentration identities, and prints a +deterministic JSON report. It also records the exact ovarian ledger hash and the +complete rows for four necessary inputs: growth, surgical retention, carboplatin +clearance, and paclitaxel volume/clearance. All four are currently missing. This +list is not a complete disease-model specification. The readiness checker never +activates a model: a future numeric ledger entry still needs an explicit reviewed +semantic binding. Existing prevalence values and trial hazard ratios do not +supply those parameters. + +No source data were acquired or re-curated. The ledger's `KNOWN` label describes +its source-access convention and must not be translated into model validity. +The [research report](../auto-research/2026-11-06-E-evidence-bindings/report.md) +records the bounded software result and remaining validation work. diff --git a/docs/model_contracts.md b/docs/model_contracts.md index b9c03d7..6d282f2 100644 --- a/docs/model_contracts.md +++ b/docs/model_contracts.md @@ -71,3 +71,10 @@ of `Schema`; the registry dispatches the explicitly named model only. Callbacks should be pure functions: rejecting an invalid output cannot undo callback side effects. The registry is not a sandbox for untrusted Python. Contract and callable definitions remain outside serialized graph data. + +## Optional evidence bindings + +For recorded source locators, uncertainty metadata, missing/conflicting input refusal, +and explicitly enabled synthetic scenarios, see [evidence bindings](evidence_bindings.md). +This opt-in layer preserves input audit records; it does not certify scientific +applicability or infer output uncertainty. diff --git a/scripts/validate_evidence_bindings.py b/scripts/validate_evidence_bindings.py new file mode 100644 index 0000000..66f5040 --- /dev/null +++ b/scripts/validate_evidence_bindings.py @@ -0,0 +1,73 @@ +"""Offline nominal synthetic mass balance and ovarian kinetic-input readiness. + +No ledger value is converted, imported into a model, or scientifically validated. +""" +from __future__ import annotations +import hashlib +import json +from pathlib import Path +from fractions import Fraction + +from vh.graph import Edge, ModelRegistry, ModelSpec, PortSpec, Quantity +from vh.graph.evidence import EvidenceInput, Uncertainty, execute_evidence + +ROOT = Path(__file__).resolve().parents[1] + + +def synthetic_check(): + context = {"system": "artificial mass-balance fixture"} + specs = {"rate": PortSpec("elimination_rate", "mg/h", "eliminated", context, minimum=0), + "concentration": PortSpec("concentration", "mg/L", "plasma", context, minimum=0.01)} + output = PortSpec("clearance", "L/h", "plasma", context, minimum=0) + def quantity(value, spec): + return Quantity(value, spec.quantity, spec.unit, spec.basis, spec.context, ("artificial fixture",)) + registry = ModelRegistry() + registry.register(ModelSpec("synthetic.clearance", specs, {"clearance": output}, + lambda values: {"clearance": quantity(values["rate"].value / values["concentration"].value, output)}, + "Artificial scalar mass balance only", "Exact rational rate/concentration identity", + ("synthetic definition",))) + errors = [] + for rate, concentration in ((3, Fraction(1, 2)), (7, Fraction(1, 4)), (9, Fraction(3, 4))): + records = {name: EvidenceInput(name, "synthetic", quantity(float(value), specs[name]), + "synthetic fixture", "validate_evidence_bindings.py:synthetic_check", "Not a biological measurement", + Uncertainty("not_quantified", "Fixed artificial input; no sampling interpretation")) + for name, value in (("rate", rate), ("concentration", concentration))} + result = execute_evidence(registry, Edge("a", "b", "converts", model="synthetic.clearance"), + records, specs, allow_synthetic=True)["clearance"] + errors.append(abs(result.value - float(Fraction(rate) / concentration))) + return {"cases": len(errors), "maximum_absolute_error_L_per_h": max(errors), + "tolerance_L_per_h": 1e-12, "passed": max(errors) <= 1e-12, + "interpretation": "Nominal synthetic arithmetic only; no output uncertainty or biological validation"} + + +def readiness(ledger): + entries = ledger["entries"] + by_id = {row["id"]: row for row in entries} + if len(by_id) != len(entries): + raise ValueError("duplicate ledger IDs") + required = {"P015": "tumor_growth", "P016": "surgical_retention", "P019": "carboplatin_clearance", + "P020": "paclitaxel_volume_and_clearance"} + rows = [] + for identifier, role in required.items(): + row = by_id.get(identifier) + rows.append({"record_id": identifier, "required_role": role, "ready": False, + "reason": "record absent" if row is None else ( + "missing estimate" if row["label"] == "MISSING" or row["value"] is None else + "explicit reviewed quantity/unit/basis/context binding required"), + "ledger_record": row}) + return {"ready": False, "requirements": rows, + "interpretation": "Readiness for these four necessary inputs only, not a complete model specification. " + "KNOWN describes source access, not applicability; prevalence and hazard ratios are not kinetic rates."} + + +def main(): + path = ROOT / "data/clinical/ovarian_hgsoc_parameter_ledger.json" + content = path.read_bytes() + result = {"schema_version": 1, "ledger_sha256": hashlib.sha256(content).hexdigest(), + "synthetic_validation": synthetic_check(), "ovarian_readiness": readiness(json.loads(content))} + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + return 0 if result["synthetic_validation"]["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_evidence_bindings.py b/tests/test_evidence_bindings.py new file mode 100644 index 0000000..f8bec55 --- /dev/null +++ b/tests/test_evidence_bindings.py @@ -0,0 +1,145 @@ +"""Boundary checks and exact synthetic arithmetic; no biological validation.""" +from dataclasses import replace +from fractions import Fraction +import json +import runpy +from pathlib import Path + +import pytest + +from vh.graph import ContractError, Edge, ModelRegistry, ModelSpec, PortSpec, Quantity +from vh.graph.evidence import EvidenceInput, Uncertainty, execute_evidence + +SPEC = PortSpec("concentration", "mol/L", "free", {"system": "synthetic"}, minimum=0) +EDGE = Edge("a", "b", "converts", model="identity") + + +def record(status="recorded"): + return EvidenceInput("fixture-1", status, + Quantity(2.0, SPEC.quantity, SPEC.unit, SPEC.basis, SPEC.context, ("fixture",)), + "artificial source", "table 1, row 1", "No biological validity", + Uncertainty("CI95", "Artificial confidence interval, not a distribution", 1.0, 3.0)) + + +def registry(): + calls = [] + def compute(values): + calls.append(values) + return {"out": values["in"]} + result = ModelRegistry() + result.register(ModelSpec("identity", {"in": SPEC}, {"out": SPEC}, compute, + "synthetic", "identity", ("synthetic identity",))) + return result, calls + + +def test_complete_audit_survives_execution_and_original_context_mutation(): + source_context = {"system": "synthetic"} + evidence = replace(record(), estimate=replace(record().estimate, context=source_context)) + source_context["system"] = "changed" + models, calls = registry() + result = execute_evidence(models, EDGE, {"in": evidence}, {"in": SPEC})["out"] + audit = json.loads(next(s[len("evidence:"):] for s in result.provenance if s.startswith("evidence:"))) + assert audit == evidence.audit() + assert audit["uncertainty"]["kind"] == "CI95" + assert audit["estimate"]["context"] == {"system": "synthetic"} + assert result.value == 2.0 + assert len(calls) == 1 + + +@pytest.mark.parametrize("status", ["conjecture", "synthetic", "missing", "conflicting"]) +def test_unsupported_status_never_calls_model(status): + item = record() if status not in ("missing", "conflicting") else replace(record(), status=status, estimate=None) + item = replace(item, status=status) + models, calls = registry() + with pytest.raises(ContractError, match="cannot bind"): + execute_evidence(models, EDGE, {"in": item}, {"in": SPEC}) + assert calls == [] + + +def test_synthetic_permission_is_explicit_and_does_not_allow_conjecture(): + assert record("synthetic").bind(SPEC, allow_synthetic=True).value == 2 + with pytest.raises(ContractError): + record("conjecture").bind(SPEC, allow_synthetic=True) + with pytest.raises(ContractError): + record("synthetic").bind(SPEC, allow_synthetic="yes") + + +@pytest.mark.parametrize("field,value", [("quantity", "hazard_ratio"), ("unit", "mg/L"), + ("basis", "total"), ("context", {"system": "human"}), ("value", -1)]) +def test_wrong_semantics_never_calls_model(field, value): + evidence = replace(record(), estimate=replace(record().estimate, **{field: value})) + models, calls = registry() + with pytest.raises(ContractError): + execute_evidence(models, EDGE, {"in": evidence}, {"in": SPEC}) + assert calls == [] + + +def test_registered_specs_cannot_be_overridden_by_binding_specs(): + other = replace(SPEC, basis="total") + evidence = replace(record(), estimate=replace(record().estimate, basis="total")) + models, calls = registry() + with pytest.raises(ContractError, match="basis mismatch"): + execute_evidence(models, EDGE, {"in": evidence}, {"in": other}) + assert calls == [] + + +@pytest.mark.parametrize("change", [{"status": "unknown"}, {"estimate": None}, {"source": ""}, + {"locator": ""}, {"limits": ""}, {"uncertainty": None}, {"estimate": 2}, + {"status": "missing"}, {"status": "conflicting"}]) +def test_malformed_evidence_rejected(change): + with pytest.raises(ContractError): + replace(record(), **change) + + +@pytest.mark.parametrize("low,high", [(1, None), (None, 2), (3, 1), (True, 2), (1, float("inf")), + (float("nan"), 3)]) +def test_malformed_uncertainty_rejected(low, high): + with pytest.raises(ContractError): + Uncertainty("CI95", "reported interval", low, high) + + +def test_uncertainty_is_retained_not_used_as_input_bounds(): + # A CI is not a hard range: execution checks the nominal value only. + assert replace(record(), uncertainty=Uncertainty("CI95", "source CI", -1, 3)).bind(SPEC).value == 2 + + +def test_missing_extra_and_untyped_ports_rejected_before_dispatch(): + models, calls = registry() + for data in ({}, {"in": record(), "extra": record()}, {"in": 2}): + with pytest.raises(ContractError): + execute_evidence(models, EDGE, data, {"in": SPEC}) + assert calls == [] + + +def test_synthetic_end_to_end_against_exact_rational_identity(): + script = runpy.run_path(str(Path(__file__).parents[1] / "scripts/validate_evidence_bindings.py")) + result = script["synthetic_check"]() + assert result["cases"] == 3 + assert result["maximum_absolute_error_L_per_h"] == 0 + assert result["passed"] + + +def test_actual_ledger_cannot_supply_kinetic_inputs(): + root = Path(__file__).parents[1] + script = runpy.run_path(str(root / "scripts/validate_evidence_bindings.py")) + ledger = json.loads((root / "data/clinical/ovarian_hgsoc_parameter_ledger.json").read_text()) + result = script["readiness"](ledger) + assert not result["ready"] + assert len(result["requirements"]) == 4 + assert all(r["reason"] == "missing estimate" for r in result["requirements"]) + # A newly filled row still needs explicit semantic binding; merely changing + # its label/value cannot authorize a simulation. + row = next(r for r in ledger["entries"] if r["id"] == "P015") + row.update(label="KNOWN", value=10) + updated = script["readiness"](ledger) + assert updated["requirements"][0]["reason"] == "explicit reviewed quantity/unit/basis/context binding required" + assert not updated["ready"] + + +def test_finite_real_scalars_have_serializable_audits(): + item = replace(record(), estimate=replace(record().estimate, value=Fraction(1, 2)), + uncertainty=Uncertainty("bounds", "artificial", Fraction(1, 4), Fraction(3, 4))) + bound = item.bind(SPEC) + assert bound.value == Fraction(1, 2) + assert item.audit()["estimate"]["value"] == 0.5 + assert item.audit()["uncertainty"]["low"] == 0.25 diff --git a/vh/graph/evidence.py b/vh/graph/evidence.py new file mode 100644 index 0000000..6cb04f2 --- /dev/null +++ b/vh/graph/evidence.py @@ -0,0 +1,110 @@ +"""Opt-in evidence bindings for scalar models, without scientific certification. + +A recorded estimate is not a validated model parameter. Callers must supply the +complete applicable semantics; no source parsing or conversions happen here. +Uncertainty metadata accompanies nominal execution, never becomes a distribution +or an output confidence interval. Missing/conflicting assertions cannot execute. +""" +from __future__ import annotations + +from dataclasses import dataclass +import json +from types import MappingProxyType +from collections.abc import Mapping + +from .contracts import ContractError, ModelRegistry, PortSpec, Quantity, _text, _finite +from .core import Edge + + +@dataclass(frozen=True) +class Uncertainty: + """Source-reported interval or explicit lack of quantification. + + CI, IQR and bounds retain their meaning; none implies a sampling distribution. + Endpoints share the estimate's units and basis. No interval propagation occurs. + """ + kind: str + description: str + low: float | None = None + high: float | None = None + + def __post_init__(self): + _text(self.kind, "uncertainty kind") + _text(self.description, "uncertainty description") + if (self.low is None) != (self.high is None): + raise ContractError("uncertainty requires both endpoints or neither") + if self.low is not None: + _finite(self.low, "uncertainty low") + _finite(self.high, "uncertainty high") + if self.low > self.high: + raise ContractError("uncertainty endpoints are reversed") + + +@dataclass(frozen=True) +class EvidenceInput: + record_id: str + status: str + estimate: Quantity | None + source: str + locator: str + limits: str + uncertainty: Uncertainty + + def __post_init__(self): + for name in ("record_id", "source", "locator", "limits"): + _text(getattr(self, name), name) + if self.status not in ("recorded", "synthetic", "conjecture", "missing", "conflicting"): + raise ContractError("unknown evidence status") + if self.estimate is not None and not isinstance(self.estimate, Quantity): + raise ContractError("estimate must be a Quantity or absent") + if self.status in ("recorded", "synthetic", "conjecture") and self.estimate is None: + raise ContractError("numeric evidence status requires an estimate") + if self.status in ("missing", "conflicting") and self.estimate is not None: + raise ContractError("missing/conflicting evidence must not select an estimate") + if not isinstance(self.uncertainty, Uncertainty): + raise ContractError("uncertainty must be explicit") + + def audit(self) -> dict: + q = self.estimate + return {"record_id": self.record_id, "status": self.status, "source": self.source, + "locator": self.locator, "limits": self.limits, + "uncertainty": {"kind": self.uncertainty.kind, "description": self.uncertainty.description, + "low": None if self.uncertainty.low is None else float(self.uncertainty.low), + "high": None if self.uncertainty.high is None else float(self.uncertainty.high)}, + "estimate": None if q is None else {"value": float(q.value), "quantity": q.quantity, + "unit": q.unit, "basis": q.basis, "context": dict(q.context), + "provenance": list(q.provenance)}} + + def bind(self, spec: PortSpec, *, allow_synthetic: bool = False) -> Quantity: + if not isinstance(allow_synthetic, bool): + raise ContractError("allow_synthetic must be boolean") + if self.status != "recorded" and not (self.status == "synthetic" and allow_synthetic): + raise ContractError(f"{self.record_id}: evidence status {self.status!r} cannot bind") + if not isinstance(spec, PortSpec): + raise ContractError("binding requires a PortSpec") + spec.check(self.estimate, self.record_id) + q = self.estimate + # Structured audit survives ModelRegistry's existing provenance union. + audit = "evidence:" + json.dumps(self.audit(), sort_keys=True, separators=(",", ":"), allow_nan=False) + return Quantity(q.value, q.quantity, q.unit, q.basis, q.context, (*q.provenance, audit)) + + +def execute_evidence(registry: ModelRegistry, edge: Edge, + evidence: Mapping[str, EvidenceInput], specs: Mapping[str, PortSpec], *, + allow_synthetic: bool = False) -> dict[str, Quantity]: + """Validate every binding before dispatch; preserve full evidence in lineage. + + Specs are explicit binding requirements, independently rechecked against the + registered model by ModelRegistry. This is opt-in and not a security boundary. + Returned values are nominal scenario outputs, with no inferred uncertainty. + """ + if not isinstance(evidence, Mapping) or not isinstance(specs, Mapping): + raise ContractError("evidence and specs must be mappings") + records, ports = dict(evidence), dict(specs) + if set(records) != set(ports): + raise ContractError("evidence ports mismatch") + if any(not isinstance(v, EvidenceInput) for v in records.values()): + raise ContractError("all bindings must be EvidenceInput") + bound = {name: record.bind(ports[name], allow_synthetic=allow_synthetic) + for name, record in records.items()} + return registry.execute(edge, MappingProxyType(bound)) From a76be7540fc8afba4599b516ce18020d45d00c16 Mon Sep 17 00:00:00 2001 From: ahrazzle <38987266+ahrazzle@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:51:23 -0600 Subject: [PATCH 2/2] Preserve exact scalar values in evidence audit lineage --- .../2026-11-06-E-evidence-bindings/report.md | 12 ++++++++--- docs/evidence_bindings.md | 5 +++++ tests/test_evidence_bindings.py | 18 +++++++++++++++-- vh/graph/evidence.py | 20 ++++++++++++++++--- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/auto-research/2026-11-06-E-evidence-bindings/report.md b/auto-research/2026-11-06-E-evidence-bindings/report.md index ea41e04..c583ef0 100644 --- a/auto-research/2026-11-06-E-evidence-bindings/report.md +++ b/auto-research/2026-11-06-E-evidence-bindings/report.md @@ -38,7 +38,7 @@ The executable floating-point model produces 6, 28 and 12 L/h respectively. | --- | --- | --- | --- | | Three nominal clearances | 6, 28, 12 L/h | Exact rational rate / concentration | Maximum 0 L/h; tolerance 1e-12 L/h | | Missing kinetic inputs | Four refused | Checked-in ledger P015, P016, P019, P020 | All four lack estimates; software consistency check only | -| Interface tests | 32 new passing cases | Explicit boundary examples and exact arithmetic | No statistical accuracy claim | +| Interface tests | 33 new passing cases | Explicit boundary examples and exact arithmetic | No statistical accuracy claim | The arithmetic error has no statistical error bar: these are fixed synthetic fixtures, not sampled observations. Source uncertainty remains attached to the @@ -76,9 +76,9 @@ node web/test_engines.mjs ``` Observed environment: Python 3.14.6, pytest 9.1.1, NumPy 2.5.4, SciPy 1.18.1. -Focused pre-serialization check: 129 passed, 1 network test deselected; final evidence-only suite: 32 passed. JavaScript engine checks +Focused pre-serialization check: 129 passed, 1 network test deselected; evidence-only suite before independent-review correction: 32 passed. JavaScript engine checks passed against checked-in fixtures, including worst plasma-profile relative error -4.8e-13. Final full local suite: 217 passed, 15 skipped, 1 network test deselected. Skips reflect unavailable OpenMM, PySCF and libRoadRunner; those independent-engine checks were not run. No dependency installation was performed. +4.8e-13. Final full local suite: 218 passed, 15 skipped, 1 network test deselected. Skips reflect unavailable OpenMM, PySCF and libRoadRunner; those independent-engine checks were not run. No dependency installation was performed. ## References @@ -86,3 +86,9 @@ passed against checked-in fixtures, including worst plasma-profile relative erro - `data/clinical/ovarian_hgsoc_parameter_ledger.json`, unchanged source ledger. - `docs/research-foundation.md` and `docs/research-question-template.md` for the separation of evidence assertions, modeling assumptions and scientific validation. + +Independent software review found an audit precision defect for large integers. +The correction preserves JSON integers and tagged exact rationals and rejects +other non-lossless representations before dispatch. A large-integer regression +verifies the executed value and recorded endpoints agree exactly. Final suite +after this correction: 218 passed, 15 skipped, 1 network test deselected. diff --git a/docs/evidence_bindings.md b/docs/evidence_bindings.md index 708eca8..0d93b10 100644 --- a/docs/evidence_bindings.md +++ b/docs/evidence_bindings.md @@ -26,6 +26,11 @@ contract. Supplying weaker binding specifications cannot override that contract. A rejected input never invokes the registered callback. As with the registry, callbacks should be pure: output validation cannot undo callback side effects. +Integer audit values retain exact JSON integers; rational values retain tagged +numerator/denominator pairs. Other real scalar values must convert losslessly to +a JSON float or binding fails before execution. Consumers must preserve these +representations when decoding the audit. + The complete input audit is serialized under an `evidence:` provenance entry and therefore accompanies every returned output through the registry's existing lineage union. Returned values are nominal scenario outputs. The recorded input diff --git a/tests/test_evidence_bindings.py b/tests/test_evidence_bindings.py index f8bec55..d60f63c 100644 --- a/tests/test_evidence_bindings.py +++ b/tests/test_evidence_bindings.py @@ -141,5 +141,19 @@ def test_finite_real_scalars_have_serializable_audits(): uncertainty=Uncertainty("bounds", "artificial", Fraction(1, 4), Fraction(3, 4))) bound = item.bind(SPEC) assert bound.value == Fraction(1, 2) - assert item.audit()["estimate"]["value"] == 0.5 - assert item.audit()["uncertainty"]["low"] == 0.25 + assert item.audit()["estimate"]["value"] == {"representation": "rational", "numerator": 1, "denominator": 2} + assert item.audit()["uncertainty"]["low"] == {"representation": "rational", "numerator": 1, "denominator": 4} + + +def test_audit_retains_large_integer_estimate_and_bounds_exactly(): + value = 2**53 + 1 + item = replace(record(), estimate=replace(record().estimate, value=value), + uncertainty=Uncertainty("bounds", "artificial exact integers", value, value + 2)) + models, calls = registry() + result = execute_evidence(models, EDGE, {"in": item}, {"in": SPEC})["out"] + audit = json.loads(next(s[len("evidence:"):] for s in result.provenance if s.startswith("evidence:"))) + assert audit["estimate"]["value"] == value == result.value + assert audit["uncertainty"]["low"] == value + assert audit["uncertainty"]["high"] == value + 2 + assert isinstance(audit["estimate"]["value"], int) + assert len(calls) == 1 diff --git a/vh/graph/evidence.py b/vh/graph/evidence.py index 6cb04f2..510eaa3 100644 --- a/vh/graph/evidence.py +++ b/vh/graph/evidence.py @@ -11,11 +11,25 @@ import json from types import MappingProxyType from collections.abc import Mapping +from numbers import Integral, Rational from .contracts import ContractError, ModelRegistry, PortSpec, Quantity, _text, _finite from .core import Edge +def _audit_scalar(value): + """Lossless JSON representation of an accepted scalar, or refuse binding.""" + if isinstance(value, Integral): + return int(value) + if isinstance(value, Rational): + return {"representation": "rational", "numerator": int(value.numerator), + "denominator": int(value.denominator)} + converted = float(value) + if value != converted: + raise ContractError("scalar cannot be serialized losslessly for evidence audit") + return converted + + @dataclass(frozen=True) class Uncertainty: """Source-reported interval or explicit lack of quantification. @@ -69,9 +83,9 @@ def audit(self) -> dict: return {"record_id": self.record_id, "status": self.status, "source": self.source, "locator": self.locator, "limits": self.limits, "uncertainty": {"kind": self.uncertainty.kind, "description": self.uncertainty.description, - "low": None if self.uncertainty.low is None else float(self.uncertainty.low), - "high": None if self.uncertainty.high is None else float(self.uncertainty.high)}, - "estimate": None if q is None else {"value": float(q.value), "quantity": q.quantity, + "low": None if self.uncertainty.low is None else _audit_scalar(self.uncertainty.low), + "high": None if self.uncertainty.high is None else _audit_scalar(self.uncertainty.high)}, + "estimate": None if q is None else {"value": _audit_scalar(q.value), "quantity": q.quantity, "unit": q.unit, "basis": q.basis, "context": dict(q.context), "provenance": list(q.provenance)}}