diff --git a/README.md b/README.md index 8d32f6e..639fd04 100644 --- a/README.md +++ b/README.md @@ -111,3 +111,10 @@ Other allocation modes remain rejected for those risk extensions. Economic comparison requires the explicit [measurement contract](docs/measurement.md). Run the [independent financial benchmark suite](docs/financial-benchmarks.md) with pinned real repository adapters; synthetic fixture agreement does not certify market validity. +# Research findings inventory + +Run `quant-lab findings --registry path/to/experiments.db` for a compact inventory, +or add `--json` for recorded scope, every attempt and verified result metrics. +The command opens the existing registry read-only and preserves failed/retried +attempts. Missing or altered result files are marked unverified. Execution success +is not a hypothesis verdict; human research decisions remain in Studio. diff --git a/src/quant_lab/cli.py b/src/quant_lab/cli.py index cd6537c..6317616 100644 --- a/src/quant_lab/cli.py +++ b/src/quant_lab/cli.py @@ -72,6 +72,16 @@ def cmd_export_html(args: argparse.Namespace) -> int: return 0 +def cmd_findings(args: argparse.Namespace) -> int: + from quant_lab.findings import findings_markdown, research_findings + + report = research_findings(Path(args.registry)) + print( + json.dumps(report, indent=2, ensure_ascii=False) if args.json else findings_markdown(report) + ) + return 0 + + def cmd_validate(args: argparse.Namespace) -> int: manifest = load_and_validate_standard_run(Path(args.run_dir)) payload = { @@ -119,6 +129,11 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--db", default=str(_default_db()), help="SQLite index path") sub = p.add_subparsers(dest="command", required=True) + findings = sub.add_parser("findings", help="Read all preregistered studies and failed attempts") + findings.add_argument("--registry", required=True, help="Existing TrialRegistry SQLite file") + findings.add_argument("--json", action="store_true", help="Include scope and verified evidence") + findings.set_defaults(func=cmd_findings) + benchmarks = sub.add_parser( "financial-benchmarks", help="Run independent real-adapter benchmarks" ) diff --git a/src/quant_lab/findings.py b/src/quant_lab/findings.py new file mode 100644 index 0000000..fbe17cd --- /dev/null +++ b/src/quant_lab/findings.py @@ -0,0 +1,115 @@ +"""Read-only research inventory; execution success is not a research verdict.""" + +from collections import Counter +from pathlib import Path + +from quant_lab.research import verify_result +from quant_lab.trials import TrialRegistry, canonical + + +def research_findings(path: Path) -> dict: + registry = TrialRegistry(path, read_only=True) + studies = [] + for study_id in registry.study_ids(): + spec = registry.definition(study_id) + definition = spec["definition"] + events = registry.history(study_id) + starts = {e["attempt_id"]: e for e in events if e["status"] == "running"} + latest = {e["attempt_id"]: e for e in events} + attempts = [] + for attempt_id, event in latest.items(): + row = {**event, "parameters": starts[attempt_id]["payload"]["parameters"]} + payload = event["payload"] + if event["status"] == "completed" and isinstance(payload.get("result"), str): + try: + root = Path(starts[attempt_id]["payload"]["context"]["artifact_root"]).resolve() + target = (root / payload["result"]).resolve() + if root not in target.parents: + raise ValueError("result escapes its registered artifact root") + result = verify_result(target, payload["sha256"]) + if result.get("study_id") != study_id or result.get("attempt_id") != attempt_id: + raise ValueError("result identity mismatch") + row["evidence"] = { + "verified": True, + "metrics": result.get("metrics"), + "comparison": result.get("comparison"), + "objective_evaluation": result.get("objective_evaluation"), + } + except (KeyError, ValueError, OSError, TypeError) as exc: + row["evidence"] = {"verified": False, "reason": str(exc)} + else: + row["evidence"] = {"verified": False, "reason": "no file-backed result"} + attempts.append(row) + attempted = {canonical(s["payload"]["parameters"]) for s in starts.values()} + recipe = definition.get("recipe", {}) + studies.append( + { + "study_id": study_id, + "hypothesis": definition["hypothesis"], + "registered_at": spec["registered_at"], + "definition_sha256": spec["sha256"], + "family_id": definition.get("family_id"), + "interval": recipe.get("interval"), + "inputs": recipe.get("inputs"), + "source": recipe.get("source"), + "selection_rule": definition["selection_rule"], + "planned_candidates": len(definition["parameters"]), + "unattempted_candidates": sum( + canonical(p) not in attempted for p in definition["parameters"] + ), + "attempt_counts": dict(Counter(e["status"] for e in latest.values())), + "attempts": attempts, + } + ) + return { + "schema": "quant.research-findings/v1", + "registry": str(Path(path).resolve()), + "policy": "All attempts retained. Execution status does not imply hypothesis support or promotion.", + "studies": studies, + } + + +def findings_markdown(report: dict) -> str: + def cell(value): + return ( + str(value if value is not None else "not recorded") + .replace("|", "\\|") + .replace("\n", " ") + ) + + lines = [ + "# Research findings", + "", + report["policy"], + "", + "| Study | Hypothesis | Family | Planned / unattempted | Attempt outcomes |", + "| --- | --- | --- | --- | --- |", + ] + for row in report["studies"]: + counts = ", ".join( + f"{key}: {value}" for key, value in sorted(row["attempt_counts"].items()) + ) + lines.append( + "| " + + " | ".join( + map( + cell, + [ + row["study_id"], + row["hypothesis"], + row["family_id"], + f"{row['planned_candidates']} / {row['unattempted_candidates']}", + counts or "not started", + ], + ) + ) + + " |" + ) + lines += [ + "", + ( + "DSR/PBO and human decisions are not inferred from run status. " + "Use verified family evidence for selection statistics and Studio Research findings for human decisions." + ), + ] + return "\n".join(lines) + "\n" diff --git a/src/quant_lab/trials.py b/src/quant_lab/trials.py index 81d43c1..5aa08cb 100644 --- a/src/quant_lab/trials.py +++ b/src/quant_lab/trials.py @@ -267,6 +267,11 @@ def history(self, study_id: str) -> list[dict]: for a, s, t, p in rows ] + def study_ids(self) -> list[str]: + """List registered studies, including those with no attempts.""" + with self.connect() as db: + return [r[0] for r in db.execute("SELECT study_id FROM studies ORDER BY study_id")] + def definition(self, study_id: str) -> dict: with self.connect() as db: row = db.execute( diff --git a/tests/test_findings.py b/tests/test_findings.py new file mode 100644 index 0000000..453a3ee --- /dev/null +++ b/tests/test_findings.py @@ -0,0 +1,58 @@ +import json + +import pytest + +from quant_lab.cli import main +from quant_lab.findings import research_findings +from quant_lab.research import file_hash +from quant_lab.trials import TrialRegistry + + +def test_findings_include_unstarted_failed_retried_and_tampered_results(tmp_path, capsys): + path = tmp_path / "trials.db" + registry = TrialRegistry(path) + definition = { + "hypothesis": "not yet proven", + "parameters": [{"w": 1}, {"w": 2}], + "code_identity": "abc", + "selection_rule": "report all", + } + registry.register("pending", definition) + registry.register("retry", definition) + failed = registry.start("retry", {"w": 1}) + registry.finish(failed, "failed", {"error": "missing data"}) + attempt = registry.start("retry", {"w": 1}, context={"artifact_root": str(tmp_path)}) + result = tmp_path / "result.json" + result.write_text( + json.dumps( + { + "study_id": "retry", + "attempt_id": attempt, + "artifacts": {}, + "metrics": {"sharpe": 0.2}, + } + ) + ) + registry.finish(attempt, "completed", {"result": "result.json", "sha256": file_hash(result)}) + before = path.read_bytes() + report = research_findings(path) + assert path.read_bytes() == before + assert report["studies"][0]["unattempted_candidates"] == 2 + retry = report["studies"][1] + assert retry["attempt_counts"] == {"failed": 1, "completed": 1} + assert retry["unattempted_candidates"] == 1 + assert retry["attempts"][-1]["evidence"]["metrics"] == {"sharpe": 0.2} + assert main(["findings", "--registry", str(path), "--json"]) == 0 + assert json.loads(capsys.readouterr().out) == report + assert main(["findings", "--registry", str(path)]) == 0 + assert "failed: 1" in capsys.readouterr().out + result.write_text("{}") + evidence = research_findings(path)["studies"][1]["attempts"][-1]["evidence"] + assert evidence["verified"] is False and "metrics" not in evidence + + +def test_reader_does_not_create_missing_registry(tmp_path): + path = tmp_path / "missing.db" + with pytest.raises(FileNotFoundError): + research_findings(path) + assert not path.exists()