From fdccab35dcfe394d193fae37bff3fb89cac56fc8 Mon Sep 17 00:00:00 2001 From: Kappaski Maintainer Date: Thu, 11 Jun 2026 10:30:24 +0800 Subject: [PATCH 1/3] feat(evidence): add l5 workspace gate --- docs/cli-reference.md | 14 + docs/html/cli-reference.html | 7 + docs/html/release-history.html | 1 + docs/html/runtime-effect-demo.html | 11 +- docs/release-history.md | 1 + docs/runtime-effect-demo.md | 16 +- src/invart/assurance/evidence_workspace.py | 373 ++++++++++++++++++++ src/invart/benchmarks/registry.py | 9 +- src/invart/benchmarks/releases_v52_v57.py | 66 +++- src/invart/commands/parser_product.py | 10 + src/invart/commands/product.py | 14 + src/invart/evaluation/benchmark_registry.py | 1 + src/invart/evaluation/release_candidate.py | 20 ++ src/invart/evaluation/roadmap.py | 15 + tests/test_integrations.py | 80 +++++ 15 files changed, 632 insertions(+), 6 deletions(-) create mode 100644 src/invart/assurance/evidence_workspace.py diff --git a/docs/cli-reference.md b/docs/cli-reference.md index eff420e..3ccdf43 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -43,6 +43,19 @@ invart replay export --ledger ledger.jsonl --out replay.html invart audit report --ledger ledger.jsonl --out-dir .invart/audit ``` +### Evidence workspace + +```bash +invart evidence export --ledger ledger.jsonl --out-dir .invart/evidence +invart evidence verify --bundle .invart/evidence/manifest.json +invart evidence inspect \ + --manifest .invart/evidence/manifest.json \ + --out-dir .invart/evidence-workspace \ + --require-layer-workflow +``` + +`evidence inspect` treats the bundle as an L5 review workspace. It verifies artifact hashes, checks required bundle contents, and reports whether the run can answer who, what, why, policy, approval, outcome, and coverage. Optional requirements such as `--require-layer-workflow` and `--require-adapter-package` turn missing links into gate failures. + ### Real agent conformance ```bash @@ -78,5 +91,6 @@ invart eval benchmark --suite v0.9.3-agent-adapter-contract invart eval benchmark --suite v0.9.4-claude-reference-adapter invart eval benchmark --suite v0.9.5-priority-agent-tracks invart eval benchmark --suite v0.9.6-layer-runtime-workflow +invart eval benchmark --suite v0.9.7-evidence-workspace-gate invart roadmap status --require-full ``` diff --git a/docs/html/cli-reference.html b/docs/html/cli-reference.html index 2637491..dc78544 100644 --- a/docs/html/cli-reference.html +++ b/docs/html/cli-reference.html @@ -16,6 +16,12 @@ invart gate verify --proof proof.json --ledger ledger.jsonl --mode ci

Replay and audit

invart replay export --ledger ledger.jsonl --out replay.html
 invart audit report --ledger ledger.jsonl --out-dir .invart/audit
+

Evidence workspace

invart evidence export --ledger ledger.jsonl --out-dir .invart/evidence
+invart evidence verify --bundle .invart/evidence/manifest.json
+invart evidence inspect \
+  --manifest .invart/evidence/manifest.json \
+  --out-dir .invart/evidence-workspace \
+  --require-layer-workflow

evidence inspect verifies bundle hashes and required artifacts, then reports whether the L5 review workspace can answer who, what, why, policy, approval, outcome, and coverage. Optional layer workflow and adapter package requirements turn missing links into gate failures.

Real agent conformance

invart adapter profile --kind claude-code
 invart adapter profiles
 invart adapter profiles --track managed_wrapper
@@ -33,6 +39,7 @@
 invart eval benchmark --suite v0.9.4-claude-reference-adapter
 invart eval benchmark --suite v0.9.5-priority-agent-tracks
 invart eval benchmark --suite v0.9.6-layer-runtime-workflow
+invart eval benchmark --suite v0.9.7-evidence-workspace-gate
 invart roadmap status --require-full
diff --git a/docs/html/release-history.html b/docs/html/release-history.html index 77af34f..bc3a4f5 100644 --- a/docs/html/release-history.html +++ b/docs/html/release-history.html @@ -34,6 +34,7 @@

0.9 Patch Track

v0.9.4ImplementedClaude Code reference full adapter: hook events and child command mediation, managed risk pause/block before launch, full L5 evidence package, and truthful degraded process-tree coverage when only portable subprocess supervision is active. v0.9.5ImplementedPriority agent tracks for Claude Code, Codex, Gemini CLI, Cursor, OpenCode, OpenClaw, Hermes, cloud agents, and frameworks, with profile-derived product matrix rows and explicit vendor/import versus Invart-mediated control positions. v0.9.6ImplementedLedger-derived L1-L5 runtime operation workflow via runtime layers, exporting a stage x layer matrix, layer timeline, operator command guide, and linked proof/replay/path graph/coverage/audit/evidence artifacts. + v0.9.7ImplementedL5 evidence workspace and gate hardening via evidence inspect, verifying bundle hashes, required artifacts, layer workflow/adapter package links, and whether proof/audit evidence answers who, what, why, policy, approval, outcome, and coverage.
diff --git a/docs/html/runtime-effect-demo.html b/docs/html/runtime-effect-demo.html index bfef672..e59b990 100644 --- a/docs/html/runtime-effect-demo.html +++ b/docs/html/runtime-effect-demo.html @@ -30,6 +30,12 @@

Run

--ledger .invart/session.jsonl \ --out-dir .invart/layer-workflow

Open .invart/layer-workflow/layer-runtime-workflow.html. The report is derived from the ledger and links proof, replay, path graph, coverage, audit, and evidence manifest artifacts.

+

To inspect the after-runtime evidence plane as a gate:

+ 
PYTHONPATH=src python -m invart.cli evidence inspect \
+  --manifest .invart/layer-workflow/evidence/manifest.json \
+  --out-dir .invart/evidence-workspace \
+  --require-layer-workflow
+

Open .invart/evidence-workspace/evidence-workspace.html. This report verifies bundle hashes and shows whether the run can answer who, what, why, policy, approval, outcome, and coverage.

Runtime Effect Matrix

@@ -39,7 +45,7 @@

Runtime Effect Matrix

L2 Runtime Fact ModelBind principal, agent identity, credentials, grants, and session.Normalize invocations with resources, taint, coverage, decisions, and outcomes.Use the ledger as the fact source. L3 Decision PlaneLoad deterministic policy and path-aware risk rules.Classify secret egress, unsafe deletion, external instruction, and suspicious skill paths.Explain why an action was allowed, paused, denied, or blocked. L4 Mediation PlaneRoute covered surfaces through managed launchers, wrappers, hooks, or brokers.Apply allow, audit, require approval, deny, enforced block, or fail-open alert.Show unresolved approvals and coverage gaps truthfully. - L5 Evidence PlaneRecord the demo claim boundary and artifact map.Generate replay, coverage, path graph, and policy artifacts.Export proof, audit, evidence bundle, and RC gate report. + L5 Evidence PlaneRecord the demo claim boundary and artifact map.Generate replay, coverage, path graph, and policy artifacts.Export proof, audit, evidence bundle, evidence workspace, and RC gate report.
@@ -64,6 +70,7 @@

Artifact Map

coverage.htmlL4/L5 truthfulnessChecking observed, mediated, enforced, and fail-open claims. audit-report.htmlL5 reviewer reportPresenting the enterprise security review view. layer-runtime-workflow.htmlL1-L5 operation workflowOperating a single ledger through before/during/after runtime questions. + evidence-workspace.htmlL5 gateable workspaceChecking bundle integrity, required links, and review-question completeness.
@@ -74,7 +81,7 @@

Layer Operation Flow

L2 Runtime Fact Modelinvart runtime record-event --ledger ledger.jsonl --event '{...}'Normalized invocation, resource, taint, identity, and outcome facts. L3 Decision Planeinvart policy check-path --ledger ledger.jsonl --out path-policy.jsonDeterministic and path-aware reasons for allow, approval, or deny. L4 Mediation Planeinvart mediation inspect --ledger ledger.jsonlPause, block, fail-open, approval, and mediation outcome state. - L5 Evidence Planeinvart runtime layers --ledger ledger.jsonl --out-dir .invart/layersA reviewable stage x layer matrix with proof/replay/graph/coverage/audit links. + L5 Evidence Planeinvart runtime layers --ledger ledger.jsonl --out-dir .invart/layers
invart evidence inspect --manifest .invart/layers/evidence/manifest.json --out-dir .invart/evidence-workspace --require-layer-workflowA reviewable stage x layer matrix plus a gateable L5 workspace for proof/replay/graph/coverage/audit links.
diff --git a/docs/release-history.md b/docs/release-history.md index b71885e..c4e3538 100644 --- a/docs/release-history.md +++ b/docs/release-history.md @@ -34,6 +34,7 @@ Invart has a long local implementation history. The public docs summarize capabi | v0.9.4 | Implemented | Claude Code reference full adapter: hook events and child command mediation, managed risk pause/block before launch, full L5 evidence package, and truthful degraded process-tree coverage when only portable subprocess supervision is active. | | v0.9.5 | Implemented | Priority agent tracks for Claude Code, Codex, Gemini CLI, Cursor, OpenCode, OpenClaw, Hermes, cloud agents, and frameworks, with profile-derived product matrix rows and explicit vendor/import versus Invart-mediated control positions. | | v0.9.6 | Implemented | Ledger-derived L1-L5 runtime operation workflow via `runtime layers`, exporting a stage x layer matrix, layer timeline, operator command guide, and linked proof/replay/path graph/coverage/audit/evidence artifacts. | +| v0.9.7 | Implemented | L5 evidence workspace and gate hardening via `evidence inspect`, verifying bundle hashes, required artifacts, layer workflow/adapter package links, and whether proof/audit evidence answers who, what, why, policy, approval, outcome, and coverage. | ## Internal History diff --git a/docs/runtime-effect-demo.md b/docs/runtime-effect-demo.md index d074f75..dfe8572 100644 --- a/docs/runtime-effect-demo.md +++ b/docs/runtime-effect-demo.md @@ -29,6 +29,17 @@ PYTHONPATH=src python -m invart.cli runtime layers \ Open `.invart/layer-workflow/layer-runtime-workflow.html`. This report is derived from the ledger and links the proof, replay, path graph, coverage, audit, and evidence manifest produced for that run. +To inspect the after-runtime evidence plane as a release gate: + +```bash +PYTHONPATH=src python -m invart.cli evidence inspect \ + --manifest .invart/layer-workflow/evidence/manifest.json \ + --out-dir .invart/evidence-workspace \ + --require-layer-workflow +``` + +Open `.invart/evidence-workspace/evidence-workspace.html`. This report verifies the bundle hashes and shows whether the run can answer who, what, why, policy, approval, outcome, and coverage. + ## What To Look For | View | What it proves | Main artifact | @@ -39,6 +50,7 @@ Open `.invart/layer-workflow/layer-runtime-workflow.html`. This report is derive | Coverage Report | Observed, mediated, and enforced are reported as different claims. | `coverage.html` | | Audit Report | A reviewer can answer who, what, why, policy, approval, outcome, and coverage. | `audit-report.html` | | Layer Runtime Workflow | A command-level way to operate L1-L5 for a single ledger. | `layer-runtime-workflow.html` | +| Evidence Workspace | A gateable L5 review workspace for bundle integrity and review questions. | `evidence-workspace.html` | ## Three Runtime Stages @@ -56,7 +68,7 @@ Open `.invart/layer-workflow/layer-runtime-workflow.html`. This report is derive | L2 Runtime Fact Model | Shows invocations, taint, identity, resources, coverage, outcomes, and ledger records. | | L3 Decision Plane | Shows deterministic rules, path-aware policy, and non-downgradable critical findings. | | L4 Mediation Plane | Shows allow, audit, require approval, deny, enforced block, and fail-open alert semantics. | -| L5 Evidence Plane | Shows proof, replay, graph, coverage, audit, evidence bundle, and release gate outputs. | +| L5 Evidence Plane | Shows proof, replay, graph, coverage, audit, evidence bundle, evidence workspace, and release gate outputs. | ## Layer Operation Flow @@ -66,7 +78,7 @@ Open `.invart/layer-workflow/layer-runtime-workflow.html`. This report is derive | L2 Runtime Fact Model | `invart runtime record-event --ledger ledger.jsonl --event '{...}'` | Normalized invocation, resource, taint, identity, and outcome facts. | | L3 Decision Plane | `invart policy check-path --ledger ledger.jsonl --out path-policy.json` | Deterministic and path-aware reasons for allow, approval, or deny. | | L4 Mediation Plane | `invart mediation inspect --ledger ledger.jsonl` | Pause, block, fail-open, approval, and mediation outcome state. | -| L5 Evidence Plane | `invart runtime layers --ledger ledger.jsonl --out-dir .invart/layers` | A reviewable stage x layer matrix with proof/replay/graph/coverage/audit links. | +| L5 Evidence Plane | `invart runtime layers --ledger ledger.jsonl --out-dir .invart/layers` then `invart evidence inspect --manifest .invart/layers/evidence/manifest.json --out-dir .invart/evidence-workspace --require-layer-workflow` | A reviewable stage x layer matrix plus a gateable L5 workspace for proof/replay/graph/coverage/audit links. | ## Boundaries diff --git a/src/invart/assurance/evidence_workspace.py b/src/invart/assurance/evidence_workspace.py new file mode 100644 index 0000000..869e622 --- /dev/null +++ b/src/invart/assurance/evidence_workspace.py @@ -0,0 +1,373 @@ +from __future__ import annotations + +import html +import json +from pathlib import Path +from typing import Any + +from invart.assurance.evidence_bundle import verify_evidence_bundle +from invart.core.artifacts import relative_href, write_html_artifact, write_json_artifact +from invart.core.models import utc_now + + +SCHEMA_VERSION = "invart.evidence_workspace.v0.9.7" +REQUIRED_ARTIFACTS = ( + "ledger", + "proof", + "replay", + "path_graph_json", + "path_graph_html", + "path_policy", + "coverage", + "audit_json", + "audit_html", +) +REVIEW_QUESTIONS = ("who", "what", "why", "policy", "approval", "outcome", "coverage") +EXPECTED_LAYERS = ("L1", "L2", "L3", "L4", "L5") +EXPECTED_STAGES = ("after-runtime", "before-runtime", "during-runtime") + + +def inspect_evidence_workspace( + manifest_path: Path, + *, + out_dir: Path | None = None, + require_questions: bool = True, + require_layer_workflow: bool = False, + require_adapter_package: bool = False, +) -> dict[str, Any]: + manifest_path = manifest_path.expanduser().resolve() + findings: list[dict[str, Any]] = [] + manifest, manifest_error = _load_json(manifest_path) + if manifest_error: + findings.append(_finding("manifest.unreadable", "fail", manifest_error, str(manifest_path))) + report = _workspace_report( + manifest_path, + manifest={}, + verification={}, + findings=findings, + answers=_empty_answers("manifest is unavailable"), + layer_workflow=_layer_workflow_status(manifest_path), + adapter_package=_adapter_package_status(manifest_path), + out_dir=out_dir, + ) + return _write_workspace(report, out_dir) + + verification = verify_evidence_bundle(manifest_path) + findings.extend(_artifact_findings(verification, manifest)) + answers = _answer_review_questions(manifest) + if require_questions: + for question, answer in answers.items(): + if not answer.get("answered"): + findings.append(_finding("question.unanswered", "fail", f"L5 review question is unanswered: {question}", question)) + + layer_workflow = _layer_workflow_status(manifest_path) + if require_layer_workflow and not layer_workflow["present"]: + findings.append(_finding("workspace.layer_workflow_missing", "fail", "Layer runtime workflow JSON/HTML is required but was not found beside the evidence bundle.", str(manifest_path))) + elif require_layer_workflow and (layer_workflow["layers"] != list(EXPECTED_LAYERS) or layer_workflow["stages"] != list(EXPECTED_STAGES)): + findings.append(_finding("workspace.layer_workflow_invalid", "fail", "Layer runtime workflow is present but does not cover all expected stages and layers.", str(manifest_path))) + + adapter_package = _adapter_package_status(manifest_path) + if require_adapter_package and not adapter_package["present"]: + findings.append(_finding("workspace.adapter_package_missing", "fail", "Adapter package descriptor is required but was not found beside the evidence bundle.", str(manifest_path))) + elif require_adapter_package and (adapter_package["status"] != "pass" or adapter_package["manifest_matches"] is not True): + findings.append(_finding("workspace.adapter_package_invalid", "fail", "Adapter package descriptor is present but does not verify against this bundle manifest.", str(manifest_path))) + + report = _workspace_report( + manifest_path, + manifest=manifest, + verification=verification, + findings=findings, + answers=answers, + layer_workflow=layer_workflow, + adapter_package=adapter_package, + out_dir=out_dir, + ) + return _write_workspace(report, out_dir) + + +def _workspace_report( + manifest_path: Path, + *, + manifest: dict[str, Any], + verification: dict[str, Any], + findings: list[dict[str, Any]], + answers: dict[str, dict[str, Any]], + layer_workflow: dict[str, Any], + adapter_package: dict[str, Any], + out_dir: Path | None, +) -> dict[str, Any]: + artifact_status = _artifact_completeness(manifest, verification) + status = "pass" if not findings and artifact_status["status"] == "pass" else "fail" + report: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "status": status, + "generated_at": utc_now(), + "manifest_path": str(manifest_path), + "ledger_is_fact_source": manifest.get("ledger_is_fact_source") is True, + "proof_is_portable_summary": manifest.get("proof_is_portable_summary") is True, + "artifact_completeness": artifact_status, + "bundle_verification": _verification_summary(verification), + "answers": answers, + "layer_workflow": layer_workflow, + "adapter_package": adapter_package, + "findings": findings, + "artifacts": {}, + "claim_boundary": "Evidence workspace is derived from the evidence bundle manifest and ledger-derived artifacts; it does not add runtime facts outside the ledger fact source.", + } + if out_dir: + out = out_dir.expanduser().resolve() + report["artifacts"] = { + "workspace_json": str(out / "evidence-workspace.json"), + "workspace_html": str(out / "evidence-workspace.html"), + } + return report + + +def _write_workspace(report: dict[str, Any], out_dir: Path | None) -> dict[str, Any]: + if not out_dir: + return report + out = out_dir.expanduser().resolve() + json_path = out / "evidence-workspace.json" + html_path = out / "evidence-workspace.html" + write_json_artifact(json_path, report) + write_html_artifact(html_path, _render_workspace_html(report)) + return report + + +def _artifact_completeness(manifest: dict[str, Any], verification: dict[str, Any]) -> dict[str, Any]: + artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), dict) else {} + missing_required = [name for name in REQUIRED_ARTIFACTS if name not in artifacts] + failed = [ + item + for item in verification.get("results", []) + if isinstance(item, dict) and item.get("status") != "pass" + ] + return { + "status": "pass" if not missing_required and not failed else "fail", + "required": list(REQUIRED_ARTIFACTS), + "missing_required": missing_required, + "verified": verification.get("summary", {}).get("artifacts", 0), + "failed": len(failed), + } + + +def _verification_summary(verification: dict[str, Any]) -> dict[str, Any]: + return { + "schema_version": verification.get("schema_version"), + "status": verification.get("status", "fail"), + "summary": verification.get("summary", {}), + } + + +def _artifact_findings(verification: dict[str, Any], manifest: dict[str, Any]) -> list[dict[str, Any]]: + findings: list[dict[str, Any]] = [] + artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), dict) else {} + for name in REQUIRED_ARTIFACTS: + if name not in artifacts: + findings.append(_finding("artifact.required_missing", "fail", f"Required evidence artifact is missing from manifest: {name}", name)) + for result in verification.get("results", []): + if not isinstance(result, dict) or result.get("status") == "pass": + continue + check_id = "artifact.missing" if result.get("actual_sha256") is None else "artifact.hash_mismatch" + findings.append( + _finding( + check_id, + "fail", + f"Evidence artifact verification failed for {result.get('artifact')}", + str(result.get("path") or result.get("artifact") or ""), + expected=result.get("expected_sha256"), + actual=result.get("actual_sha256"), + ) + ) + return findings + + +def _answer_review_questions(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]: + proof = _load_artifact(manifest, "proof") + audit = _load_artifact(manifest, "audit_json") + path_policy = _load_artifact(manifest, "path_policy") + coverage_exists = _artifact_path(manifest, "coverage").exists() if _artifact_path(manifest, "coverage") else False + + session = proof.get("session", {}) if isinstance(proof.get("session"), dict) else {} + summary = proof.get("summary", {}) if isinstance(proof.get("summary"), dict) else {} + accountability = proof.get("accountability", {}) if isinstance(proof.get("accountability"), dict) else {} + policy_decisions = proof.get("policy_decisions") if isinstance(proof.get("policy_decisions"), list) else [] + approvals = proof.get("approval_evidence") if isinstance(proof.get("approval_evidence"), list) else None + outcomes = proof.get("execution_outcomes") if isinstance(proof.get("execution_outcomes"), list) else None + coverage = proof.get("coverage") if isinstance(proof.get("coverage"), dict) else {} + audit_policy = audit.get("policy") if isinstance(audit.get("policy"), dict) else {} + + return { + "who": _answer( + bool(session.get("session_id") or session.get("agent") or accountability), + "Accountable session, agent, and principal boundary.", + { + "session_id": session.get("session_id"), + "agent": session.get("agent"), + "principal": accountability.get("principal") or accountability.get("accountable_principal"), + "identity_source": "proof.accountability" if accountability else "proof.session", + }, + ), + "what": _answer( + "total_actions" in summary or bool(proof.get("actions")), + "Runtime actions and touched resources.", + {"total_actions": summary.get("total_actions"), "action_types": summary.get("action_types", {})}, + ), + "why": _answer( + bool(path_policy or policy_decisions or proof.get("risk_statement")), + "Policy, risk, and path explanation for the execution.", + { + "policy_decisions": len(policy_decisions), + "path_policy_status": path_policy.get("status"), + "risk_statement": proof.get("risk_statement"), + }, + ), + "policy": _answer( + bool(audit_policy or path_policy), + "Policy profile and path-aware decision evidence.", + {"audit_policy_keys": sorted(audit_policy.keys()), "path_policy_schema": path_policy.get("schema_version")}, + ), + "approval": _answer( + approvals is not None, + "Approval state, including the explicit absence of approvals.", + {"approval_records": len(approvals or []), "summary": {key: summary.get(key) for key in ("missing_approvals", "approvals")}}, + ), + "outcome": _answer( + outcomes is not None or bool(session), + "Execution outcome evidence, with session status as the fallback boundary.", + {"outcome_records": len(outcomes or []), "session_status": session.get("status")}, + ), + "coverage": _answer( + bool(coverage) or coverage_exists, + "Coverage evidence preserving observed, mediated, and enforced distinctions.", + {"proof_coverage": bool(coverage), "coverage_artifact": str(_artifact_path(manifest, "coverage") or "")}, + ), + } + + +def _answer(answered: bool, statement: str, evidence: dict[str, Any]) -> dict[str, Any]: + return {"answered": bool(answered), "statement": statement, "evidence": evidence} + + +def _empty_answers(reason: str) -> dict[str, dict[str, Any]]: + return {question: _answer(False, reason, {}) for question in REVIEW_QUESTIONS} + + +def _layer_workflow_status(manifest_path: Path) -> dict[str, Any]: + json_path = _first_existing( + [ + manifest_path.parent / "layer-runtime-workflow.json", + manifest_path.parent.parent / "layer-runtime-workflow.json", + ] + ) + html_path = _first_existing( + [ + manifest_path.parent / "layer-runtime-workflow.html", + manifest_path.parent.parent / "layer-runtime-workflow.html", + ] + ) + payload = _load_json(json_path)[0] if json_path else {} + present = json_path is not None and html_path is not None + return { + "present": present, + "json": str(json_path) if json_path else None, + "html": str(html_path) if html_path else None, + "schema_version": payload.get("schema_version") if isinstance(payload, dict) else None, + "layers": sorted({item.get("layer") for item in payload.get("runtime_effect_matrix", []) if isinstance(item, dict)}) if isinstance(payload, dict) else [], + "stages": sorted({item.get("stage") for item in payload.get("runtime_effect_matrix", []) if isinstance(item, dict)}) if isinstance(payload, dict) else [], + } + + +def _adapter_package_status(manifest_path: Path) -> dict[str, Any]: + package_path = _first_existing( + [ + manifest_path.parent / "adapter-package.json", + manifest_path.parent.parent / "adapter-package.json", + ] + ) + payload, error = _load_json(package_path) if package_path else ({}, "adapter package not found") + manifest_matches = False + if isinstance(payload, dict): + raw_manifest = payload.get("manifest_path") + manifest_matches = Path(str(raw_manifest)).expanduser().resolve() == manifest_path if raw_manifest else False + return { + "present": package_path is not None, + "path": str(package_path) if package_path else None, + "status": payload.get("status") if isinstance(payload, dict) else None, + "schema_version": payload.get("schema_version") if isinstance(payload, dict) else None, + "manifest_matches": manifest_matches, + "error": None if package_path else error, + } + + +def _render_workspace_html(report: dict[str, Any]) -> str: + base = Path(report["artifacts"]["workspace_html"]).parent if report.get("artifacts", {}).get("workspace_html") else Path.cwd() + answer_rows = "".join( + "" + f"{html.escape(question)}" + f"{'yes' if answer.get('answered') else 'no'}" + f"{html.escape(str(answer.get('statement', '')))}" + f"
{html.escape(json.dumps(answer.get('evidence', {}), ensure_ascii=False, indent=2, sort_keys=True))}
" + "" + for question, answer in report.get("answers", {}).items() + ) + finding_rows = "".join( + "" + f"{html.escape(str(item.get('check_id')))}" + f"{html.escape(str(item.get('severity')))}" + f"{html.escape(str(item.get('message')))}" + f"{html.escape(str(item.get('artifact') or item.get('subject') or ''))}" + "" + for item in report.get("findings", []) + ) or "No findings." + links = [] + for section in ("layer_workflow", "adapter_package"): + item = report.get(section, {}) + for key in ("json", "html", "path"): + value = item.get(key) if isinstance(item, dict) else None + if value: + links.append(f"
  • {html.escape(section)} {html.escape(key)}: {html.escape(Path(value).name)}
    • ") + return f"""Invart L5 Evidence Workspace

    Invart L5 Evidence Workspace

    Status: {html.escape(str(report.get('status')))}

    Review Questions

    {answer_rows}
    QuestionAnsweredStatementEvidence

    Artifact Completeness

    {html.escape(json.dumps(report.get('artifact_completeness', {}), ensure_ascii=False, indent=2, sort_keys=True))}

    Layer and Adapter Links

      {''.join(links) or '
    • No linked layer workflow or adapter package.
      • '}

    Findings

    {finding_rows}
    CheckSeverityMessageSubject

    Claim Boundary

    {html.escape(str(report.get('claim_boundary')))}

    """ + + +def _load_artifact(manifest: dict[str, Any], name: str) -> dict[str, Any]: + path = _artifact_path(manifest, name) + if not path: + return {} + payload, error = _load_json(path) + return payload if not error else {} + + +def _artifact_path(manifest: dict[str, Any], name: str) -> Path | None: + artifacts = manifest.get("artifacts") if isinstance(manifest.get("artifacts"), dict) else {} + item = artifacts.get(name) if isinstance(artifacts.get(name), dict) else None + raw = item.get("path") if item else None + return Path(str(raw)).expanduser().resolve() if raw else None + + +def _load_json(path: Path | None) -> tuple[dict[str, Any], str | None]: + if path is None: + return {}, "path is missing" + if not path.exists(): + return {}, f"file not found: {path}" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return {}, f"invalid JSON in {path}: {exc}" + if not isinstance(payload, dict): + return {}, f"JSON root is not an object: {path}" + return payload, None + + +def _first_existing(paths: list[Path]) -> Path | None: + for path in paths: + if path.exists(): + return path.expanduser().resolve() + return None + + +def _finding(check_id: str, severity: str, message: str, subject: str, **extra: Any) -> dict[str, Any]: + return {"check_id": check_id, "severity": severity, "message": message, "subject": subject, **extra} + + +__all__ = ["inspect_evidence_workspace"] diff --git a/src/invart/benchmarks/registry.py b/src/invart/benchmarks/registry.py index 52e63be..6906d85 100644 --- a/src/invart/benchmarks/registry.py +++ b/src/invart/benchmarks/registry.py @@ -55,7 +55,13 @@ run_product_control_matrix_benchmark, run_reviewer_ablation_cost_benchmark, ) -from .releases_v52_v57 import run_agent_adapter_contract_benchmark, run_claude_reference_adapter_benchmark, run_layer_runtime_workflow_benchmark, run_priority_agent_tracks_benchmark +from .releases_v52_v57 import ( + run_agent_adapter_contract_benchmark, + run_claude_reference_adapter_benchmark, + run_evidence_workspace_gate_benchmark, + run_layer_runtime_workflow_benchmark, + run_priority_agent_tracks_benchmark, +) BenchmarkRunner = Callable[[], dict[str, Any]] @@ -114,6 +120,7 @@ def benchmark_runner_registry() -> dict[str, BenchmarkRunner]: "v0.9.4-claude-reference-adapter": run_claude_reference_adapter_benchmark, "v0.9.5-priority-agent-tracks": run_priority_agent_tracks_benchmark, "v0.9.6-layer-runtime-workflow": run_layer_runtime_workflow_benchmark, + "v0.9.7-evidence-workspace-gate": run_evidence_workspace_gate_benchmark, "progressive-external-validation": run_progressive_external_validation_benchmark, "real-world-agent-risk-demo": run_real_world_risk_benchmark, "containerized-risk-demo": run_container_risk_demo_benchmark, diff --git a/src/invart/benchmarks/releases_v52_v57.py b/src/invart/benchmarks/releases_v52_v57.py index ebec099..b946fb9 100644 --- a/src/invart/benchmarks/releases_v52_v57.py +++ b/src/invart/benchmarks/releases_v52_v57.py @@ -5,11 +5,13 @@ from pathlib import Path from .common import _suite_result -from invart.assurance.evidence_bundle import verify_evidence_bundle +from invart.assurance.evidence_bundle import export_evidence_bundle, verify_evidence_bundle +from invart.assurance.evidence_workspace import inspect_evidence_workspace from invart.assurance.layer_runtime import export_layer_runtime_workflow from invart.core.ledger import load_ledger_entries from invart.core.models import RuntimeEvent from invart.control.runtime import close_session, record_action, start_session +from invart.evaluation.release_candidate import verify_release_candidate from invart.evaluation.product_control_matrix import run_product_control_matrix from invart.evaluation.real_agent_conformance import run_real_agent_conformance from invart.surfaces.adapter import run_adapter_command @@ -198,9 +200,71 @@ def run_layer_runtime_workflow_benchmark() -> dict[str, object]: ) +def run_evidence_workspace_gate_benchmark() -> dict[str, object]: + with tempfile.TemporaryDirectory(prefix="invart_v097_") as tmp: + root = Path(tmp) + ledger = root / "ledger.jsonl" + session = start_session(root, ledger, agent="claude-code", goal="v0.9.7 evidence workspace benchmark", create_preflight=False) + record_action(RuntimeEvent(type="file_read", session_id=session.session_id, path=str(root / ".env"), metadata={"coverage_layer": "native_hook"}), ledger) + record_action(RuntimeEvent(type="network", session_id=session.session_id, url="https://example.com/upload", metadata={"coverage_layer": "native_hook"}), ledger) + close_session(ledger) + + workflow = export_layer_runtime_workflow(ledger, root / "layers") + workspace = inspect_evidence_workspace( + Path(workflow["artifacts"]["evidence_manifest"]), + out_dir=root / "workspace", + require_questions=True, + require_layer_workflow=True, + ) + + tamper_bundle = export_evidence_bundle(ledger, root / "tamper-bundle", profile={"name": "tamper", "mode": "managed"}) + proof_path = Path(tamper_bundle["artifacts"]["proof"]) + proof_path.write_text(proof_path.read_text(encoding="utf-8") + "\n{\"tampered\": true}\n", encoding="utf-8") + tampered = inspect_evidence_workspace(Path(tamper_bundle["manifest_path"]), out_dir=root / "tampered") + + claude = run_claude_code_adapter( + target=root, + command=[sys.executable, "-c", "pass"], + out_dir=root / "claude", + session_id="ks_v097_benchmark_claude", + policy_mode="advisory", + ) + adapter_workspace = inspect_evidence_workspace( + Path(claude["adapter_package"]["manifest_path"]), + out_dir=root / "adapter-workspace", + require_adapter_package=True, + ) + rc = verify_release_candidate( + root / "rc", + run_pytest=False, + benchmark_suites=["v0.9.6-layer-runtime-workflow"], + evidence_workspace_manifest=Path(workflow["artifacts"]["evidence_manifest"]), + require_evidence_layer_workflow=True, + ) + checks = { + "workspace_answers_l5_questions": workspace.get("status") == "pass" and all(answer.get("answered") for answer in workspace.get("answers", {}).values()), + "workspace_requires_layer_workflow": workspace.get("layer_workflow", {}).get("present") is True, + "tamper_fails_workspace": tampered.get("status") == "fail" and any(item.get("check_id") == "artifact.hash_mismatch" for item in tampered.get("findings", [])), + "adapter_package_requirement_passes": adapter_workspace.get("status") == "pass" and adapter_workspace.get("adapter_package", {}).get("present") is True, + "rc_consumes_workspace_gate": rc.get("status") == "pass" and rc.get("checks", {}).get("evidence_workspace", {}).get("status") == "pass", + } + return _suite_result( + "v0.9.7-evidence-workspace-gate", + checks, + artifacts={ + "workspace_json": workspace.get("artifacts", {}).get("workspace_json"), + "workspace_html": workspace.get("artifacts", {}).get("workspace_html"), + "layer_workflow": workflow["artifacts"]["workflow_json"], + "adapter_workspace": adapter_workspace.get("artifacts", {}).get("workspace_json"), + "rc_report": rc.get("artifacts", {}).get("report_json"), + }, + ) + + __all__ = [ "run_agent_adapter_contract_benchmark", "run_claude_reference_adapter_benchmark", + "run_evidence_workspace_gate_benchmark", "run_priority_agent_tracks_benchmark", "run_layer_runtime_workflow_benchmark", ] diff --git a/src/invart/commands/parser_product.py b/src/invart/commands/parser_product.py index c42c6a2..9bea104 100644 --- a/src/invart/commands/parser_product.py +++ b/src/invart/commands/parser_product.py @@ -53,6 +53,13 @@ def register_product_commands(subparsers: argparse._SubParsersAction[argparse.Ar evidence_export.add_argument("--profile", default=None) evidence_verify = evidence_sub.add_parser("verify", help="Verify an evidence bundle manifest.") evidence_verify.add_argument("--bundle", required=True) + evidence_inspect = evidence_sub.add_parser("inspect", help="Inspect a bundle as an L5 evidence workspace.") + evidence_inspect.add_argument("--manifest", "--bundle", dest="manifest", required=True) + evidence_inspect.add_argument("--out-dir", default=None) + evidence_inspect.add_argument("--require-questions", dest="require_questions", action="store_true", default=True) + evidence_inspect.add_argument("--no-require-questions", dest="require_questions", action="store_false") + evidence_inspect.add_argument("--require-layer-workflow", action="store_true") + evidence_inspect.add_argument("--require-adapter-package", action="store_true") external_evidence = subparsers.add_parser("external-evidence", help="Import, attach, and verify external benchmark evidence manifests.") external_evidence.set_defaults(handler=handle_external_evidence) @@ -135,6 +142,9 @@ def register_product_commands(subparsers: argparse._SubParsersAction[argparse.Ar rc_verify.add_argument("--final", action="store_true") rc_verify.add_argument("--require-external-validation", action="store_true") rc_verify.add_argument("--external-evidence", default=None) + rc_verify.add_argument("--evidence-workspace-manifest", default=None) + rc_verify.add_argument("--require-evidence-layer-workflow", action="store_true") + rc_verify.add_argument("--require-evidence-adapter-package", action="store_true") rc_verify.add_argument("--paper", action="store_true", help="Run the separate v0.51 research-readiness gate after product RC checks.") rc_verify.add_argument("--paper-tables", default=None) rc_verify.add_argument("--coverage", default=None) diff --git a/src/invart/commands/product.py b/src/invart/commands/product.py index d2d6658..4cfb2f0 100644 --- a/src/invart/commands/product.py +++ b/src/invart/commands/product.py @@ -12,6 +12,7 @@ from invart.assurance.audit_demo import record_audit_signoff, run_enterprise_audit_demo, run_enterprise_audit_live_adapter_demo from invart.evaluation.pre_v1 import run_pre_v1_control_plane_demo from invart.assurance.evidence_bundle import export_evidence_bundle, verify_evidence_bundle +from invart.assurance.evidence_workspace import inspect_evidence_workspace from invart.evaluation.release_candidate import verify_release_candidate from invart.evaluation.experiment_cases import export_experiment_report, list_experiment_suites, run_experiment_suite, run_paper_suite from invart.evaluation.audit_reconstruction import run_audit_reconstruction_study @@ -147,6 +148,16 @@ def handle_evidence(args: argparse.Namespace) -> int: result = verify_evidence_bundle(Path(args.bundle)) print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)) return 0 if result.get("status") == "pass" else 1 + if args.evidence_command == "inspect": + result = inspect_evidence_workspace( + Path(args.manifest), + out_dir=Path(args.out_dir) if args.out_dir else None, + require_questions=args.require_questions, + require_layer_workflow=args.require_layer_workflow, + require_adapter_package=args.require_adapter_package, + ) + print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if result.get("status") == "pass" else 1 return 2 @@ -205,6 +216,9 @@ def handle_release_candidate(args: argparse.Namespace) -> int: final=args.final, require_external_validation=args.require_external_validation, external_evidence_manifest=Path(args.external_evidence) if args.external_evidence else None, + evidence_workspace_manifest=Path(args.evidence_workspace_manifest) if args.evidence_workspace_manifest else None, + require_evidence_layer_workflow=args.require_evidence_layer_workflow, + require_evidence_adapter_package=args.require_evidence_adapter_package, ) if args.paper: research = verify_research_readiness( diff --git a/src/invart/evaluation/benchmark_registry.py b/src/invart/evaluation/benchmark_registry.py index 588db90..d8aeaee 100644 --- a/src/invart/evaluation/benchmark_registry.py +++ b/src/invart/evaluation/benchmark_registry.py @@ -35,6 +35,7 @@ {"suite": "v0.9.4-claude-reference-adapter", "version": "v0.9.4", "category": "agent-adapter", "optional_heavy": False, "claim_scope": "local_claude_reference_adapter", "evidence_level": "local_runtime_fixture"}, {"suite": "v0.9.5-priority-agent-tracks", "version": "v0.9.5", "category": "agent-adapter", "optional_heavy": False, "claim_scope": "local_agent_track_matrix", "evidence_level": "fixture_backed_profile_matrix"}, {"suite": "v0.9.6-layer-runtime-workflow", "version": "v0.9.6", "category": "runtime", "optional_heavy": False, "claim_scope": "local_layer_runtime_workflow", "evidence_level": "ledger_derived_runtime_fixture"}, + {"suite": "v0.9.7-evidence-workspace-gate", "version": "v0.9.7", "category": "evidence", "optional_heavy": False, "claim_scope": "local_l5_evidence_workspace", "evidence_level": "ledger_derived_runtime_fixture"}, {"suite": "progressive-external-validation", "version": "pre-release", "category": "external-validation", "optional_heavy": False, "claim_scope": "progressive_external_validation", "evidence_level": "external_progressive_sample"}, {"suite": "real-world-agent-risk-demo", "version": "pre-release", "category": "demo", "optional_heavy": False, "claim_scope": "public_source_mapping", "evidence_level": "public_source_seed_plus_local_demo"}, {"suite": "containerized-risk-demo", "version": "pre-release", "category": "demo", "optional_heavy": False, "claim_scope": "containerized_local_demo", "evidence_level": "per_case_container_artifact_bundle"}, diff --git a/src/invart/evaluation/release_candidate.py b/src/invart/evaluation/release_candidate.py index 1c23113..c528893 100644 --- a/src/invart/evaluation/release_candidate.py +++ b/src/invart/evaluation/release_candidate.py @@ -12,6 +12,7 @@ from invart.core.artifacts import write_html_artifact, write_json_artifact from invart.assurance.evidence_bundle import export_evidence_bundle, verify_evidence_bundle +from invart.assurance.evidence_workspace import inspect_evidence_workspace from invart.evaluation.external_evidence import verify_external_evidence from invart.evaluation.pre_1_0 import run_pre_1_0_final_demo from invart.evaluation.pre_v1 import run_pre_v1_control_plane_demo @@ -51,6 +52,7 @@ "v0.9.4-claude-reference-adapter", "v0.9.5-priority-agent-tracks", "v0.9.6-layer-runtime-workflow", + "v0.9.7-evidence-workspace-gate", "progressive-external-validation", "pre-v1-control-plane", ) @@ -107,6 +109,9 @@ def verify_release_candidate( final: bool = False, require_external_validation: bool = False, external_evidence_manifest: Path | None = None, + evidence_workspace_manifest: Path | None = None, + require_evidence_layer_workflow: bool = False, + require_evidence_adapter_package: bool = False, ) -> dict[str, Any]: out_dir.mkdir(parents=True, exist_ok=True) checks: dict[str, Any] = {} @@ -143,12 +148,27 @@ def verify_release_candidate( demo = run_pre_v1_control_plane_demo(out_dir / "demo") evidence = export_evidence_bundle(Path(demo["artifacts"]["ledger"]), out_dir / "evidence", profile={"name": "rc", "mode": "managed"}) evidence_verify = verify_evidence_bundle(Path(evidence["manifest_path"])) + workspace_manifest = evidence_workspace_manifest or Path(evidence["manifest_path"]) + evidence_workspace = inspect_evidence_workspace( + workspace_manifest, + out_dir=out_dir / "evidence-workspace", + require_questions=True, + require_layer_workflow=require_evidence_layer_workflow, + require_adapter_package=require_evidence_adapter_package, + ) checks["artifact_completeness"] = { "status": "pass" if demo.get("status") == "pass" and evidence_verify.get("status") == "pass" else "fail", "demo": demo, "evidence": evidence, "evidence_verify": evidence_verify, } + checks["evidence_workspace"] = { + "status": "pass" if evidence_workspace.get("status") == "pass" else "fail", + "manifest_path": str(workspace_manifest), + "requires_layer_workflow": require_evidence_layer_workflow, + "requires_adapter_package": require_evidence_adapter_package, + "report": evidence_workspace, + } final_readiness = _final_readiness( out_dir, final=final, diff --git a/src/invart/evaluation/roadmap.py b/src/invart/evaluation/roadmap.py index 451f94e..c3a2766 100644 --- a/src/invart/evaluation/roadmap.py +++ b/src/invart/evaluation/roadmap.py @@ -827,6 +827,21 @@ def internal_docs_for_capability(docs: list[str]) -> list[str]: external_validation="not_run_optional", next_step="Harden L5 evidence workspace and release gate checks so missing or inconsistent artifacts fail clearly.", ), + RoadmapCapability( + version="v0.9.7", + capability_id="l5_evidence_workspace_gate", + title="L5 evidence workspace and gate hardening", + target="Inspect evidence bundles as review workspaces that answer who, what, why, policy, approval, outcome, and coverage while failing clearly on tamper, missing layer workflow, or missing adapter package requirements.", + status="implemented", + implementation=["src/invart/assurance/evidence_workspace.py", "src/invart/evaluation/release_candidate.py", "src/invart/commands/parser_product.py", "src/invart/commands/product.py", "src/invart/benchmarks/releases_v52_v57.py"], + tests=["test_v097_evidence_workspace_answers_l5_review_questions", "test_v097_evidence_workspace_fails_on_tamper_and_requires_layer_workflow", "test_v097_evidence_workspace_cli_rc_and_benchmark_are_registered", "v0.9.7-evidence-workspace-gate"], + docs=["docs/cli-reference.md", "docs/html/cli-reference.html", "docs/runtime-effect-demo.md", "docs/html/runtime-effect-demo.html", "docs/release-history.md", "docs/html/release-history.html"], + product_boundaries=["Local ledger-derived runtime fixture validates evidence workspace checks and RC gate integration; it is not a hosted evidence review UI.", "The workspace verifies artifact hashes and required links but does not create runtime facts outside the ledger-derived bundle.", "Layer workflow and adapter package requirements are explicit gate inputs; weak or missing coverage is reported as a finding rather than silently upgraded."], + claim_scope="local_l5_evidence_workspace", + evidence_level="ledger_derived_runtime_fixture", + external_validation="not_run_optional", + next_step="Use the workspace gate as the pre-release evidence review surface while continuing live-agent conformance sampling.", + ), ) diff --git a/tests/test_integrations.py b/tests/test_integrations.py index 16cffd3..b30d883 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -758,6 +758,86 @@ def test_v096_runtime_layers_cli_and_benchmark_are_registered(tmp_path: Path) -> assert main(["eval", "benchmark", "--suite", "v0.9.6-layer-runtime-workflow"]) == 0 +def test_v097_evidence_workspace_answers_l5_review_questions(tmp_path: Path) -> None: + from invart.assurance.evidence_bundle import export_evidence_bundle + from invart.assurance.evidence_workspace import inspect_evidence_workspace + + ledger = tmp_path / "ledger.jsonl" + session = start_session(tmp_path, ledger, agent="claude-code", goal="v0.9.7 evidence workspace", create_preflight=False) + action, decision, _taint = record_action(RuntimeEvent(type="file_read", session_id=session.session_id, path=str(tmp_path / ".env"), metadata={"coverage_layer": "native_hook"}), ledger) + record_approval(ledger, decision.decision_id, "approved", approver="security-reviewer", reason="fixture approval for L5 workspace") + record_outcome(ledger, "executed", decision_id=decision.decision_id, invocation_id=action.invocation_id, actor="claude-code-adapter", reason="fixture completed") + record_action(RuntimeEvent(type="network", session_id=session.session_id, url="https://example.com/upload", metadata={"coverage_layer": "native_hook"}), ledger) + close_session(ledger) + + bundle = export_evidence_bundle(ledger, tmp_path / "bundle", profile={"name": "workspace", "mode": "managed"}) + workspace = inspect_evidence_workspace(Path(bundle["manifest_path"]), out_dir=tmp_path / "workspace", require_questions=True) + + assert workspace["schema_version"] == "invart.evidence_workspace.v0.9.7" + assert workspace["status"] == "pass" + assert workspace["artifact_completeness"]["status"] == "pass" + assert workspace["ledger_is_fact_source"] is True + assert workspace["proof_is_portable_summary"] is True + assert set(workspace["answers"]) == {"who", "what", "why", "policy", "approval", "outcome", "coverage"} + assert all(answer["answered"] is True for answer in workspace["answers"].values()) + assert Path(workspace["artifacts"]["workspace_json"]).exists() + html = Path(workspace["artifacts"]["workspace_html"]).read_text(encoding="utf-8") + assert "L5 Evidence Workspace" in html + assert "who" in html + assert "coverage" in html + + +def test_v097_evidence_workspace_fails_on_tamper_and_requires_layer_workflow(tmp_path: Path) -> None: + from invart.assurance.evidence_bundle import export_evidence_bundle + from invart.assurance.evidence_workspace import inspect_evidence_workspace + from invart.assurance.layer_runtime import export_layer_runtime_workflow + + ledger = tmp_path / "ledger.jsonl" + session = start_session(tmp_path, ledger, agent="codex", goal="v0.9.7 evidence tamper", create_preflight=False) + record_action(RuntimeEvent(type="shell", session_id=session.session_id, command="cat .env", metadata={"coverage_layer": "shell_wrapper"}), ledger) + close_session(ledger) + + bundle = export_evidence_bundle(ledger, tmp_path / "bundle", profile={"name": "workspace", "mode": "managed"}) + missing_layer = inspect_evidence_workspace(Path(bundle["manifest_path"]), out_dir=tmp_path / "missing-layer", require_layer_workflow=True) + assert missing_layer["status"] == "fail" + assert any(finding["check_id"] == "workspace.layer_workflow_missing" for finding in missing_layer["findings"]) + + proof_path = Path(bundle["artifacts"]["proof"]) + proof_path.write_text(proof_path.read_text(encoding="utf-8") + "\n{\"tampered\": true}\n", encoding="utf-8") + tampered = inspect_evidence_workspace(Path(bundle["manifest_path"]), out_dir=tmp_path / "tampered") + assert tampered["status"] == "fail" + assert any(finding["check_id"] == "artifact.hash_mismatch" for finding in tampered["findings"]) + + workflow = export_layer_runtime_workflow(ledger, tmp_path / "layers") + layered = inspect_evidence_workspace(Path(workflow["artifacts"]["evidence_manifest"]), out_dir=tmp_path / "layered", require_layer_workflow=True) + assert layered["status"] == "pass" + assert layered["layer_workflow"]["present"] is True + + +def test_v097_evidence_workspace_cli_rc_and_benchmark_are_registered(tmp_path: Path) -> None: + from invart.assurance.layer_runtime import export_layer_runtime_workflow + from invart.evaluation.release_candidate import verify_release_candidate + + ledger = tmp_path / "ledger.jsonl" + session = start_session(tmp_path, ledger, agent="claude-code", goal="v0.9.7 cli rc", create_preflight=False) + record_action(RuntimeEvent(type="file_read", session_id=session.session_id, path=str(tmp_path / "README.md"), metadata={"coverage_layer": "native_hook"}), ledger) + close_session(ledger) + workflow = export_layer_runtime_workflow(ledger, tmp_path / "layers") + manifest = workflow["artifacts"]["evidence_manifest"] + + assert main(["evidence", "inspect", "--manifest", manifest, "--out-dir", str(tmp_path / "cli-workspace"), "--require-questions", "--require-layer-workflow"]) == 0 + rc = verify_release_candidate( + tmp_path / "rc", + run_pytest=False, + benchmark_suites=["v0.9.7-evidence-workspace-gate"], + evidence_workspace_manifest=Path(manifest), + require_evidence_layer_workflow=True, + ) + assert rc["status"] == "pass" + assert rc["checks"]["evidence_workspace"]["status"] == "pass" + assert main(["eval", "benchmark", "--suite", "v0.9.7-evidence-workspace-gate"]) == 0 + + def test_v09_swe_bench_lite_runner_skips_cleanly_without_dependencies(tmp_path: Path) -> None: out = tmp_path / "swebench-report.json" assert main([ From 05625ed73b90c2f06b56dc0b6bad5c83fa45987f Mon Sep 17 00:00:00 2001 From: Kappaski Maintainer Date: Thu, 11 Jun 2026 11:29:48 +0800 Subject: [PATCH 2/3] docs: reorganize user documentation journeys --- README.md | 2 + docs/README.md | 1 + docs/api-sdk.md | 4 +- docs/architecture.md | 10 +- docs/cli-reference.md | 15 ++- docs/concepts.md | 25 ++-- docs/evaluation.md | 13 ++ docs/examples.md | 31 +++-- docs/five-layer-operator-guide.md | 144 +++++++++++++++++++++ docs/html/api-sdk.html | 2 + docs/html/architecture.html | 10 +- docs/html/cli-reference.html | 16 ++- docs/html/concepts.html | 27 ++-- docs/html/evaluation.html | 14 +- docs/html/examples.html | 17 ++- docs/html/five-layer-operator-guide.html | 105 +++++++++++++++ docs/html/index.html | 38 ++++-- docs/html/product.html | 18 ++- docs/html/quickstart.html | 15 ++- docs/html/release-history.html | 5 +- docs/html/runtime-effect-demo.html | 4 +- docs/index.md | 53 ++++---- docs/product.md | 16 ++- docs/quickstart.md | 19 ++- docs/release-history.md | 2 + docs/runtime-effect-demo.md | 8 +- src/invart/evaluation/release_candidate.py | 2 + tests/test_policy_evidence_rc.py | 58 +++++++++ uv.lock | 2 +- 29 files changed, 573 insertions(+), 103 deletions(-) create mode 100644 docs/five-layer-operator-guide.md create mode 100644 docs/html/five-layer-operator-guide.html diff --git a/README.md b/README.md index 1ccc102..fb5da13 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ For a first pass, the most useful public-facing surfaces are: | --- | --- | | [`docs/product.md`](docs/product.md) | One-page product overview for the before / during / after control loop. | | [`docs/quickstart.md`](docs/quickstart.md) | Minimal local session that writes a ledger and verifies proof. | +| [`docs/five-layer-operator-guide.md`](docs/five-layer-operator-guide.md) | Practical guide for operating L1-L5 on a real ledger. | | [`docs/runtime-effect-demo.md`](docs/runtime-effect-demo.md) | Shows the demo's before / during / after stages and L1-L5 control effects. | | [`docs/evaluation.md`](docs/evaluation.md) | Explains what the built-in benchmarks measure and what they do not claim. | | [`examples/`](examples/) | Small runnable examples before running the larger demos. | @@ -148,6 +149,7 @@ Open the local docs: - [HTML docs home](docs/html/index.html) - [Product page](docs/product.md) - [Quickstart](docs/quickstart.md) +- [Five-layer operator guide](docs/five-layer-operator-guide.md) - [Concepts](docs/concepts.md) - [CLI reference](docs/cli-reference.md) - [API and SDK](docs/api-sdk.md) diff --git a/docs/README.md b/docs/README.md index a0bae05..109bd1a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,6 +16,7 @@ Public pages: | --- | --- | --- | | Product overview | [`product.md`](product.md) | [`html/product.html`](html/product.html) | | Quickstart | [`quickstart.md`](quickstart.md) | [`html/quickstart.html`](html/quickstart.html) | +| Five-layer operator guide | [`five-layer-operator-guide.md`](five-layer-operator-guide.md) | [`html/five-layer-operator-guide.html`](html/five-layer-operator-guide.html) | | Concepts | [`concepts.md`](concepts.md) | [`html/concepts.html`](html/concepts.html) | | CLI reference | [`cli-reference.md`](cli-reference.md) | [`html/cli-reference.html`](html/cli-reference.html) | | API and SDK | [`api-sdk.md`](api-sdk.md) | [`html/api-sdk.html`](html/api-sdk.html) | diff --git a/docs/api-sdk.md b/docs/api-sdk.md index 7d76d60..5378e5a 100644 --- a/docs/api-sdk.md +++ b/docs/api-sdk.md @@ -5,7 +5,7 @@ Invart 0.9 is CLI-first. The stable integration surface is the CLI plus verifiable artifacts. Python helpers are available for local tooling, but only a small set should be treated as provisional SDK entry points before 1.0. -[Docs Home](index.md) · [Quickstart](quickstart.md) · [CLI](cli-reference.md) · [Architecture](architecture.md) · [Examples](examples.md) +[Docs Home](index.md) · [Quickstart](quickstart.md) · [Operate L1-L5](five-layer-operator-guide.md) · [CLI](cli-reference.md) · [Architecture](architecture.md) · [Examples](examples.md) ## Public Surface @@ -77,6 +77,8 @@ These helpers are suitable for local adapters, experiments, and internal automat | Paper tables JSON/CSV/HTML | invart experiment paper-tables | Research reports, appendix tables, product validation review | Derived summaries. Each row should link back to ledger/proof/replay/path graph/evidence artifacts. | | Research readiness report | invart release-candidate verify --paper | Pre-release research review | Separate from product RC. It checks evidence completeness without claiming external benchmark completion. | +For the operational meaning of proof, replay, path graph, coverage, audit, and evidence workspace, use the [five-layer operator guide](five-layer-operator-guide.md). It explains which artifact answers each L1-L5 review question. + ## Compatibility Names New integrations should import invart and use the canonical subpackages such as invart.core, invart.control, and invart.assurance. The older flat invart.ledger style imports and the former kappaski import path remain compatibility aliases during the rename period, but new documentation and examples use the organized Invart layout. diff --git a/docs/architecture.md b/docs/architecture.md index 454098a..c14ff2e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -3,10 +3,16 @@ [HTML version](html/architecture.html) -Invart is not a plugin-only wrapper. Plugin and hook integration can improve coverage, but the durable product boundary is the runtime control plane. +Invart is not a plugin-only wrapper. Plugin and hook integration can improve coverage, but the durable product boundary is the runtime control plane. For operational usage of the same model, read the [five-layer operator guide](five-layer-operator-guide.md). ## Lifecycle +| Stage | Architecture role | +| --- | --- | +| Before runtime | Inventory surfaces, bind identity, resolve profile and grants, and establish the claim boundary. | +| During runtime | Normalize actions into ledger facts, evaluate policy, mediate covered surfaces, and record outcomes. | +| After runtime | Derive proof, replay, path graph, coverage, audit, evidence bundles, and gates from the ledger. | + ## Layer Model | Layer | Responsibility | Example modules | @@ -19,7 +25,7 @@ Invart is not a plugin-only wrapper. Plugin and hook integration can improve cov ## Demo Verification -The [runtime effect demo](runtime-effect-demo.md) is the concrete verification surface for this model. It renders a stage × layer matrix in the final demo entrypoint and an action timeline in each container risk case audit page. The timeline connects agent intent/action, Invart observation, policy or mediation decision, outcome, and the artifact that proves the step. +The [runtime effect demo](runtime-effect-demo.md) is the concrete verification surface for this model. It renders a stage × layer matrix in the final demo entrypoint and an action timeline in each container risk case audit page. The [five-layer operator guide](five-layer-operator-guide.md) is the command-level usage surface for the same layers. Coverage terms remain strict: observed, mediated, and enforced are not interchangeable. Unmanaged direct execution is shown as a coverage gap unless it enters a managed launcher, wrapper, hook, broker, or shim. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 3ccdf43..a879010 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -3,7 +3,20 @@ [HTML version](html/cli-reference.html) -Run invart --help for the complete parser. These are the entry points most people should start with. +Run `invart --help` for the complete parser. These are the entry points most people should start with. For a guided L1-L5 workflow, use the [five-layer operator guide](five-layer-operator-guide.md). + +## User intent to command + +| I want to... | Start with | +| --- | --- | +| Scan before an agent runs | `invart pre-runtime --target . --save` | +| Run one managed session | `invart run --target . --agent codex --goal "..." -- ` | +| Inspect L1-L5 for a ledger | `invart runtime layers --ledger ledger.jsonl --out-dir .invart/layers` | +| Explain a risky path | `invart policy check-path --ledger ledger.jsonl --out path-policy.json` | +| Inspect mediation state | `invart mediation inspect --ledger ledger.jsonl` | +| Export reviewable evidence | `invart evidence export --ledger ledger.jsonl --out-dir .invart/evidence` | +| Validate real-agent integration | `invart real-agent check --agent claude-code --out-dir .invart/real-agent` | +| Run product benchmarks | `invart eval benchmark --suite full-product-readiness` | ### Pre-runtime diff --git a/docs/concepts.md b/docs/concepts.md index 4235157..f63b380 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -1,17 +1,18 @@ -# The nouns that matter. +# Core concepts reference [HTML version](html/concepts.html) +This page is a glossary-style compatibility reference. If you are learning Invart for the first time, start with [Product Overview](product.md), then use the [five-layer operator guide](five-layer-operator-guide.md) for command-level usage. -Invart is easier to understand when you separate facts, decisions, mediation, and portable evidence. +| Concept | Meaning | Where to use it | +| --- | --- | --- | +| Session | A bounded agent run with target, goal, agent identity, principal, and ledger. | [Quickstart](quickstart.md) | +| Invocation | A normalized runtime action such as shell, file, network, MCP, skill, or content activity. | [Five-layer operator guide](five-layer-operator-guide.md) | +| Ledger | The append-only fact source. Proof, replay, graph, audit, and gates derive from it. | [Product Overview](product.md) | +| Proof | A portable summary of the run. It should answer who, what, why, policy, approval, outcome, and coverage. | [API & SDK](api-sdk.md) | +| Mediation | The decision contract for allow, audit, require approval, deny, enforced block, and fail-open alert. | [Five-layer operator guide](five-layer-operator-guide.md) | +| Coverage | The strength of observation or control: declared, observed, mediated, enforced, or fail-open. | [Evaluation](evaluation.md) | +| Path graph | A ledger-derived graph connecting identity, grant, invocation, resource, taint, decision, approval, outcome, and artifact. | [Runtime effect demo](runtime-effect-demo.md) | +| Evidence bundle | A verifiable package containing manifest hashes, proof, replay, graph, coverage, audit JSON/HTML, and policy summary. | [API & SDK](api-sdk.md) | -| Concept | Meaning | -| --- | --- | -| Session | A bounded agent run with target, goal, agent identity, principal, and ledger. | -| Invocation | A normalized runtime action such as shell, file, network, MCP, skill, or content activity. | -| Ledger | The append-only fact source. Proof, replay, graph, audit, and gates derive from it. | -| Proof | A portable summary of the run. It should answer who, what, why, policy, approval, outcome, and coverage. | -| Mediation | The decision contract for allow, audit, require approval, deny, enforced block, and fail-open alert. | -| Coverage | The strength of observation or control: declared, observed, mediated, enforced, or fail-open. | -| Path graph | A ledger-derived graph connecting identity, grant, invocation, resource, taint, decision, approval, outcome, and artifact. | -| Evidence bundle | A verifiable package containing manifest hashes, proof, replay, graph, coverage, audit JSON/HTML, and policy summary. | +For internal module boundaries, use [Architecture](architecture.md). diff --git a/docs/evaluation.md b/docs/evaluation.md index e0b9260..019f450 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -5,6 +5,8 @@ Invart separates local product checks from optional heavy external benchmark evidence. +[Operate L1-L5](five-layer-operator-guide.md) · [Runtime Effect Demo](runtime-effect-demo.md) · [CLI](cli-reference.md) + ## Default Local Checks ```bash @@ -24,6 +26,17 @@ PYTHONPATH=src python -m invart.cli roadmap status --require-full | Coverage truthfulness | Observed, mediated, enforced, and fail-open are not the same claim. | | Audit reconstruction | A reviewer should reconstruct the run from durable evidence. | +## Product claim to benchmark + +| Product claim | Local check | Evidence to inspect | +| --- | --- | --- | +| Invart can preserve a closed before / during / after runtime loop. | `invart eval benchmark --suite pre-v1-control-plane` | ledger, proof, replay, path graph, coverage, audit report | +| High-risk agent-like actions are blocked, paused, or escalated without silent pass-through. | `invart eval benchmark --suite real-world-agent-risk-demo` | risk demo HTML, case audit report, mediation decisions | +| Coverage labels remain truthful across observed, mediated, enforced, fail-open, and bypassed paths. | `invart eval benchmark --suite v0.47-coverage-mediation-pilot` | coverage matrix JSON/HTML | +| Auditors can reconstruct who, what, why, policy, approval, outcome, and coverage. | `invart eval benchmark --suite v0.48-audit-reconstruction-study` | audit reconstruction study and evidence bundle | +| Plugin-only or vendor-native visibility is not reported as full runtime mediation. | `invart eval benchmark --suite v0.50-product-control-matrix` | product control matrix | +| Pre-release evidence can be checked separately from external benchmark completion. | `invart eval benchmark --suite v0.51-pre-1.0-research-ready-gate` | research-ready gate report | + ## Research-Ready Validation Loop The pre-1.0 research track validates an LLM/agent workflow end to end: diff --git a/docs/examples.md b/docs/examples.md index df1fd46..7324cb7 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,24 +1,27 @@ -# Small examples before full demos. +# Examples and safe fixtures [HTML version](html/examples.html) +This page is a compatibility index for small examples. For a guided first run, use [Quickstart](quickstart.md). For layer-by-layer operation, use the [five-layer operator guide](five-layer-operator-guide.md). -The examples/ directory contains short runnable or copyable examples for local evaluation. +## Runnable examples -### Basic managed session +| Example | Use it for | +| --- | --- | +| [`../examples/basic-managed-session.sh`](../examples/basic-managed-session.sh) | Create one managed session, ledger, and proof for a low-risk command. | +| [`../examples/policy-profile.toml`](../examples/policy-profile.toml) | Inspect a compact policy-as-code profile with path-aware rules. | +| [`../examples/unsafe-action-event.json`](../examples/unsafe-action-event.json) | Analyze an unsafe shell event without executing it. | -examples/basic-managed-session.sh creates a ledger and proof for one low-risk command. - -### Policy profile - -examples/policy-profile.toml shows a small TOML profile with path-aware risk rules. - -### Unsafe event - -examples/unsafe-action-event.json is a shell event you can analyze without executing it. - -## Analyze The Unsafe Event +## Analyze the unsafe event ```bash invart runtime analyze-event --event "$(cat examples/unsafe-action-event.json)" ``` + +This command analyzes a JSON event payload. It does not execute the unsafe command inside the fixture. + +## Next steps + +- Use [Quickstart](quickstart.md) to create a real local ledger. +- Use [Five-layer Operator Guide](five-layer-operator-guide.md) to inspect L1-L5 on that ledger. +- Use [Runtime Effect Demo](runtime-effect-demo.md) for generated demo artifacts and action timelines. diff --git a/docs/five-layer-operator-guide.md b/docs/five-layer-operator-guide.md new file mode 100644 index 0000000..abe7b1f --- /dev/null +++ b/docs/five-layer-operator-guide.md @@ -0,0 +1,144 @@ +# Operate the five runtime layers. + +[HTML version](html/five-layer-operator-guide.html) + +This guide is the practical path for using Invart's five-layer model. It assumes you have either a managed Invart ledger from `invart run` / `invart runtime shell`, or a demo ledger generated by `invart demo pre-1.0-final`. + +The short version: start with the question you need to answer, run the layer command, open the artifact, then check whether the evidence proves the claim without inflating coverage. + +## Ten-minute path + +```bash +mkdir -p .invart/operator-guide + +invart pre-runtime --target . --save \ + --preflight .invart/operator-guide/preflight.json + +invart session start \ + --target . \ + --agent demo-agent \ + --goal "Inspect the repository without risky changes" \ + --session-id invart_operator_guide \ + --ledger .invart/operator-guide/ledger.jsonl \ + --preflight .invart/operator-guide/preflight.json + +invart runtime shell \ + --session invart_operator_guide \ + --ledger .invart/operator-guide/ledger.jsonl \ + -- python -c "from pathlib import Path; print(len(list(Path('.').iterdir())))" + +invart session close --ledger .invart/operator-guide/ledger.jsonl + +invart runtime layers \ + --ledger .invart/operator-guide/ledger.jsonl \ + --out-dir .invart/operator-guide/layers + +invart evidence inspect \ + --manifest .invart/operator-guide/layers/evidence/manifest.json \ + --out-dir .invart/operator-guide/evidence-workspace \ + --require-layer-workflow +``` + +Open these two files first: + +- `.invart/operator-guide/layers/layer-runtime-workflow.html` +- `.invart/operator-guide/evidence-workspace/evidence-workspace.html` + +## Which layer should I use? + +| User question | Layer | Command | Artifact to open | +| --- | --- | --- | --- | +| What can this agent touch before it runs? | L1 Execution Surface | `invart pre-runtime --target . --save` | preflight JSON or layer workflow L1 rows | +| What actually happened? | L2 Runtime Fact Model | `invart runtime layers --ledger ledger.jsonl --out-dir .invart/layers` | `ledger.jsonl`, `proof.json`, `layer-runtime-workflow.html` | +| Why was an action allowed, paused, denied, or blocked? | L3 Decision Plane | `invart policy check-path --ledger ledger.jsonl --out path-policy.json` | `path-policy.json`, `path-graph.html` | +| Did Invart only observe, mediate, or enforce? | L4 Mediation Plane | `invart mediation inspect --ledger ledger.jsonl` | mediation JSON, `replay.html`, `coverage.html` | +| Can a reviewer verify the run after the fact? | L5 Evidence Plane | `invart evidence inspect --manifest .invart/layers/evidence/manifest.json --out-dir .invart/evidence-workspace --require-layer-workflow` | `evidence-workspace.html`, `audit-report.html` | + +## Layer-by-layer operation + +### L1 Execution Surface + +Use L1 before-runtime to inventory where agent actions can enter the control plane: shell commands, file writes, network intent, skills, MCP servers, native hooks, launchers, wrappers, and unmanaged surfaces. + +```bash +invart pre-runtime --target . --save --preflight .invart/preflight.json +``` + +- Healthy signal: expected repo, skill, MCP, and agent configuration surfaces are visible, and unmanaged surfaces are named instead of hidden. +- Failure signal: an agent, skill, MCP server, launcher, or credential-adjacent surface is missing from inventory. +- Next action: route the agent through a managed launcher, wrapper, hook, or adapter before claiming mediation. + +### L2 Runtime Fact Model + +Use L2 during-runtime and after-runtime to reconstruct facts from the ledger. The ledger is the fact source; proof, replay, graph, audit, and evidence bundles derive from it. + +```bash +invart runtime layers --ledger ledger.jsonl --out-dir .invart/layers +``` + +- Healthy signal: the workflow reports session boundaries, invocations, resources, taint, decisions, approvals, outcomes, and artifact links. +- Failure signal: the ledger only has session start/close entries, or actions lack resource, decision, outcome, or coverage facts. +- Next action: record actions through `invart runtime shell`, an adapter, native bridge, MCP broker, or another managed surface. + +### L3 Decision Plane + +Use L3 to answer why a risky path was allowed, audited, paused, denied, or blocked. Deterministic critical policy cannot be downgraded by LLM review. + +```bash +invart policy check-path --ledger ledger.jsonl --out path-policy.json +``` + +- Healthy signal: secret egress, unsafe deletion, external instruction hijack, and deploy/CI mutation paths have explainable policy outcomes. +- Failure signal: a high-risk path has no policy reason, no taint chain, or only an LLM allow without deterministic policy support. +- Next action: tighten the policy profile, add missing path predicates, or move the surface into managed mediation. + +### L4 Mediation Plane + +Use L4 to inspect whether Invart observed the action, mediated it, enforced a block, or recorded a fail-open alert. These coverage labels are not interchangeable: observed, mediated, enforced, fail-open, and unmanaged mean different things. + +```bash +invart mediation inspect --ledger ledger.jsonl +``` + +- Healthy signal: allow, audit, require approval, deny, enforced block, and fail-open outcomes are visible with approval and outcome state. +- Failure signal: proof or docs claim enforcement while the coverage report only shows observed or fail-open behavior. +- Next action: resolve pending approvals, route through stronger managed surfaces, or keep the report honest about weak coverage. + +### L5 Evidence Plane + +Use L5 after-runtime to give a reviewer a portable, verifiable workspace. L5 should answer who, what, why, policy, approval, outcome, and coverage. + +```bash +invart evidence inspect \ + --manifest .invart/layers/evidence/manifest.json \ + --out-dir .invart/evidence-workspace \ + --require-layer-workflow +``` + +- Healthy signal: the workspace verifies hashes, links proof/replay/path graph/coverage/audit artifacts, and marks review questions complete. +- Failure signal: artifact hashes fail, layer workflow is missing, adapter package evidence is required but absent, or review questions are incomplete. +- Next action: regenerate the evidence bundle from the ledger, attach missing artifacts, or fail the release gate. + +## Before-runtime, during-runtime, after-runtime + +| Layer | before-runtime | during-runtime | after-runtime | +| --- | --- | --- | --- | +| L1 Execution Surface | Inventory agent, skill, MCP, launcher, wrapper, hook, and unmanaged surfaces. | Attribute each action to a runtime surface. | Review coverage gaps and unmanaged execution paths. | +| L2 Runtime Fact Model | Bind session, principal, agent, credential boundary, and grants. | Record invocations, resources, taint, decisions, approvals, and outcomes. | Keep the ledger as the fact source for reconstruction. | +| L3 Decision Plane | Load deterministic policy and path-aware rules. | Explain allow, audit, approval, deny, and critical path decisions. | Reconstruct causal chains with path policy and graph artifacts. | +| L4 Mediation Plane | Decide which surfaces are observed, mediated, enforced, or weakly covered. | Apply mediation outcomes and approval state. | Report unresolved approvals and fail-open conditions truthfully. | +| L5 Evidence Plane | Record the claim boundary and expected artifacts. | Generate replay, coverage, proof, and graph artifacts. | Export audit, evidence bundle, evidence workspace, and release gate inputs. | + +## Reading the result + +Start with `layer-runtime-workflow.html` when you need to understand a single run. It shows the before-runtime, during-runtime, and after-runtime matrix across L1-L5. + +Then open `evidence-workspace.html` when you need a gateable review. It verifies bundle integrity and checks whether evidence can answer the core audit questions. + +Use [Runtime effect demo](runtime-effect-demo.md) when you want a richer demo entrypoint with risk-shaped cases. Use [CLI reference](cli-reference.md) for every command group. Use [Evaluation](evaluation.md) when you need to map product claims to benchmark evidence. + +## Boundaries + +The operator guide does not claim that every vendor plugin or extension is equivalent to Invart mediation. A plugin-only or imported trace can be useful evidence, but it is not the same as a managed, mediated, or enforced runtime surface. + +Local demos use safe equivalent trajectories. They are designed to show how Invart records, mediates, and audits agent-like behavior without installing hostile packages or replaying private incidents. diff --git a/docs/html/api-sdk.html b/docs/html/api-sdk.html index 715219d..76a4629 100644 --- a/docs/html/api-sdk.html +++ b/docs/html/api-sdk.html @@ -14,6 +14,7 @@

    Integrate with Invart without guessing the boundary.

    diff --git a/docs/html/architecture.html b/docs/html/architecture.html index 3390362..5b00975 100644 --- a/docs/html/architecture.html +++ b/docs/html/architecture.html @@ -2,11 +2,17 @@ Invart Architecture -
    Architecture

    Three lifecycle stages, five control layers.

    Invart is not a plugin-only wrapper. Plugin and hook integration can improve coverage, but the durable product boundary is the runtime control plane.

    +
    Architecture

    Three lifecycle stages, five control layers.

    Invart is not a plugin-only wrapper. Plugin and hook integration can improve coverage, but the durable product boundary is the runtime control plane.

    Lifecycle

    Pre-runtime discovery->Runtime mediation->Post-runtime assurance
    + + + + + +
    StageArchitecture role
    Before runtimeInventory surfaces, bind identity, resolve profile and grants, and establish the claim boundary.
    During runtimeNormalize actions into ledger facts, evaluate policy, mediate covered surfaces, and record outcomes.
    After runtimeDerive proof, replay, path graph, coverage, audit, evidence bundles, and gates from the ledger.

    Layer Model

    @@ -21,7 +27,7 @@

    Layer Model

    Demo Verification

    -

    Runtime Effect Demo is the concrete verification surface for this architecture. It renders a stage × layer matrix in the final demo entrypoint and an action timeline in each container risk case audit page.

    +

    Runtime Effect Demo is the concrete verification surface for this architecture. It renders a stage × layer matrix in the final demo entrypoint and an action timeline in each container risk case audit page. The five-layer operator guide is the command-level usage surface for the same layers.

    Coverage terms remain strict: observed, mediated, and enforced are not interchangeable. Unmanaged direct execution is shown as a coverage gap unless it enters a managed launcher, wrapper, hook, broker, or shim.

    diff --git a/docs/html/cli-reference.html b/docs/html/cli-reference.html index dc78544..d4e6b30 100644 --- a/docs/html/cli-reference.html +++ b/docs/html/cli-reference.html @@ -2,8 +2,22 @@ Invart CLI Reference -
    CLI

    The command groups users need first.

    Run invart --help for the complete parser. These are the entry points most people should start with.

    +
    CLI

    The command groups users need first.

    Run invart --help for the complete parser. These are the entry points most people should start with.

    +
    +

    User intent to command

    + + + + + + + + + + +
    I want to...Start with
    Scan before an agent runsinvart pre-runtime --target . --save
    Run one managed sessioninvart run --target . --agent codex --goal "..." -- <command>
    Inspect L1-L5 for a ledgerinvart runtime layers --ledger ledger.jsonl --out-dir .invart/layers
    Explain a risky pathinvart policy check-path --ledger ledger.jsonl --out path-policy.json
    Inspect mediation stateinvart mediation inspect --ledger ledger.jsonl
    Export reviewable evidenceinvart evidence export --ledger ledger.jsonl --out-dir .invart/evidence
    Validate real-agent integrationinvart real-agent check --agent claude-code --out-dir .invart/real-agent
    Run product benchmarksinvart eval benchmark --suite full-product-readiness
    +

    Pre-runtime

    invart pre-runtime --target . --save

    Managed session

    invart session start --target . --agent codex --goal "..."
diff --git a/docs/html/concepts.html b/docs/html/concepts.html
index 42265b0..4261d51 100644
--- a/docs/html/concepts.html
+++ b/docs/html/concepts.html
@@ -2,19 +2,22 @@
 
 Invart Concepts
 
-  
    Concepts
    The nouns that matter.
    Invart is easier to understand when you separate facts, decisions, mediation, and portable evidence.
    
+  
    Reference
    Core concepts reference
    A glossary for readers who need precise names after seeing the product and operator flows.
    
   
    
-    
-      
-      
-      
-      
-      
-      
-      
-      
-      
-    
    ConceptMeaning
    SessionA bounded agent run with target, goal, agent identity, principal, and ledger.
    InvocationA normalized runtime action such as shell, file, network, MCP, skill, or content activity.
    LedgerThe append-only fact source. Proof, replay, graph, audit, and gates derive from it.
    ProofA portable summary of the run. It should answer who, what, why, policy, approval, outcome, and coverage.
    MediationThe decision contract for allow, audit, require approval, deny, enforced block, and fail-open alert.
    CoverageThe strength of observation or control: declared, observed, mediated, enforced, or fail-open.
    Path graphA ledger-derived graph connecting identity, grant, invocation, resource, taint, decision, approval, outcome, and artifact.
    Evidence bundleA verifiable package containing manifest hashes, proof, replay, graph, coverage, audit JSON/HTML, and policy summary.
    
+    
    
+      
    This page is a compatibility reference. If you are learning Invart for the first time, start with Product Overview, then use the five-layer operator guide for command-level usage.
    
+      
+        
+        
+        
+        
+        
+        
+        
+        
+        
+      
    ConceptMeaningWhere to use it
    SessionA bounded agent run with target, goal, agent identity, principal, and ledger.Quickstart
    InvocationA normalized runtime action such as shell, file, network, MCP, skill, or content activity.Five-layer operator guide
    LedgerThe append-only fact source. Proof, replay, graph, audit, and gates derive from it.Product Overview
    ProofA portable summary that should answer who, what, why, policy, approval, outcome, and coverage.API & SDK
    MediationThe decision contract for allow, audit, require approval, deny, enforced block, and fail-open alert.Five-layer operator guide
    CoverageThe strength of observation or control: declared, observed, mediated, enforced, or fail-open.Evaluation
    Path graphA ledger-derived graph connecting identity, grant, invocation, resource, taint, decision, approval, outcome, and artifact.Runtime effect demo
    Evidence bundleA verifiable package containing manifest hashes, proof, replay, graph, coverage, audit JSON/HTML, and policy summary.API & SDK
    
+    
    
   
    
 
 
diff --git a/docs/html/evaluation.html b/docs/html/evaluation.html
index a7120fe..f60e307 100644
--- a/docs/html/evaluation.html
+++ b/docs/html/evaluation.html
@@ -2,7 +2,7 @@
 
 Invart Evaluation
 
-  
    Evaluation
    Measure control-plane behavior, not just code paths.
    Invart separates local product checks from optional heavy external benchmark evidence.
    
+  
    Evaluation
    Measure control-plane behavior, not just code paths.
    Invart separates local product checks from optional heavy external benchmark evidence.
    
   
    
     
    
       
    Default Local Checks
    
@@ -22,6 +22,18 @@ 
    Metrics
    
         Audit reconstructionA reviewer should reconstruct the run from durable evidence.
       
     
    
+    
    
+      
    Product claim to benchmark
    
+      
+        
+        
+        
+        
+        
+        
+        
+      
    Product claimLocal checkEvidence to inspect
    Closed before / during / after runtime loop.invart eval benchmark --suite pre-v1-control-planeledger, proof, replay, path graph, coverage, audit report
    High-risk agent-like actions are blocked, paused, or escalated.invart eval benchmark --suite real-world-agent-risk-demorisk demo HTML, case audit report, mediation decisions
    Coverage labels remain truthful across observed, mediated, enforced, fail-open, and bypassed paths.invart eval benchmark --suite v0.47-coverage-mediation-pilotcoverage matrix JSON/HTML
    Auditors can reconstruct who, what, why, policy, approval, outcome, and coverage.invart eval benchmark --suite v0.48-audit-reconstruction-studyaudit reconstruction study and evidence bundle
    Plugin-only or vendor-native visibility is not reported as full runtime mediation.invart eval benchmark --suite v0.50-product-control-matrixproduct control matrix
    Pre-release evidence is separate from external benchmark completion.invart eval benchmark --suite v0.51-pre-1.0-research-ready-gateresearch-ready gate report
    
+    
    
     
    
       
    Research-Ready Validation Loop
    
       
    The pre-1.0 research track validates an LLM/agent workflow end to end:
    
diff --git a/docs/html/examples.html b/docs/html/examples.html
index 7c75160..240c55e 100644
--- a/docs/html/examples.html
+++ b/docs/html/examples.html
@@ -2,16 +2,21 @@
 
 Invart Examples
 
-  
    Examples
    Small examples before full demos.
    The examples/ directory contains short runnable or copyable examples for local evaluation.
    
+  
    Examples
    Examples and safe fixtures
    Small local inputs for learning Invart before running larger demos.
    
   
    
-    
    
-      
    Basic managed session
    examples/basic-managed-session.sh creates a ledger and proof for one low-risk command.
    
-      
    Policy profile
    examples/policy-profile.toml shows a small TOML profile with path-aware risk rules.
    
-      
    Unsafe event
    examples/unsafe-action-event.json is a shell event you can analyze without executing it.
    
+    
    
+      
    This page is a compatibility index for small examples. For a guided first run, use Quickstart. For layer-by-layer operation, use the five-layer operator guide.
    
+      
+        
+        
+        
+        
+      
    ExampleUse it for
    basic-managed-session.shCreate one managed session, ledger, and proof for a low-risk command.
    policy-profile.tomlInspect a compact policy-as-code profile with path-aware rules.
    unsafe-action-event.jsonAnalyze an unsafe shell event without executing it.
    
     
    
     
    
-      
    Analyze The Unsafe Event
    
+      
    Analyze the unsafe event
    
       
    invart runtime analyze-event --event "$(cat examples/unsafe-action-event.json)"
    
+      
    This command analyzes a JSON event payload. It does not execute the unsafe command inside the fixture.
    
     
    
   
    
 
diff --git a/docs/html/five-layer-operator-guide.html b/docs/html/five-layer-operator-guide.html
new file mode 100644
index 0000000..8b47eca
--- /dev/null
+++ b/docs/html/five-layer-operator-guide.html
@@ -0,0 +1,105 @@
+
+
+
+  
+  
+  Invart Five-Layer Operator Guide
+  
+  
+
+
+  
    
+    Operate
+    
    Operate the five runtime layers.
    
+    
    Turn Invart's L1-L5 model into concrete commands, artifacts, interpretation rules, and failure signals for a single agent run.
    
+    
+  
    
+  
    
+    
    
+      
    Ten-minute path
    
+      
    Start with one managed session, then derive the layer workflow and L5 evidence workspace.
    
+      
    mkdir -p .invart/operator-guide
+
+invart pre-runtime --target . --save \
+  --preflight .invart/operator-guide/preflight.json
+
+invart session start \
+  --target . \
+  --agent demo-agent \
+  --goal "Inspect the repository without risky changes" \
+  --session-id invart_operator_guide \
+  --ledger .invart/operator-guide/ledger.jsonl \
+  --preflight .invart/operator-guide/preflight.json
+
+invart runtime shell \
+  --session invart_operator_guide \
+  --ledger .invart/operator-guide/ledger.jsonl \
+  -- python -c "from pathlib import Path; print(len(list(Path('.').iterdir())))"
+
+invart session close --ledger .invart/operator-guide/ledger.jsonl
+
+invart runtime layers \
+  --ledger .invart/operator-guide/ledger.jsonl \
+  --out-dir .invart/operator-guide/layers
+
+invart evidence inspect \
+  --manifest .invart/operator-guide/layers/evidence/manifest.json \
+  --out-dir .invart/operator-guide/evidence-workspace \
+  --require-layer-workflow
    
+    
    
+
+    
    
+      
    Which layer should I use?
    
+      
+        
+        
+        
+        
+        
+        
+      
    User questionLayerCommandArtifact
    What can this agent touch before it runs?L1 Execution Surfaceinvart pre-runtime --target . --savepreflight JSON, L1 workflow rows
    What actually happened?L2 Runtime Fact Modelinvart runtime layers --ledger ledger.jsonl --out-dir .invart/layersledger.jsonl, proof.json, workflow HTML
    Why was an action allowed, paused, denied, or blocked?L3 Decision Planeinvart policy check-path --ledger ledger.jsonl --out path-policy.jsonpath-policy.json, path graph
    Did Invart only observe, mediate, or enforce?L4 Mediation Planeinvart mediation inspect --ledger ledger.jsonlmediation JSON, replay, coverage
    Can a reviewer verify the run after the fact?L5 Evidence Planeinvart evidence inspect --manifest .invart/layers/evidence/manifest.json --out-dir .invart/evidence-workspace --require-layer-workflowevidence workspace, audit report
    
+    
    
+
+    
    
+      
    Layer operation table
    
+      
+        
+        
+        
+        
+        
+        
+      
    LayerHealthy signalFailure signalNext action
    L1 Execution SurfaceRepo, skills, MCP, hooks, wrappers, launchers, and unmanaged surfaces are visible.A surface is missing from inventory.Route the agent through a managed launcher, wrapper, hook, or adapter.
    L2 Runtime Fact ModelSession, invocation, resource, taint, decision, approval, outcome, and coverage facts exist.The ledger has only session boundaries or incomplete action facts.Record actions through managed runtime surfaces.
    L3 Decision PlanePolicy and path graph explain allow, audit, approval, deny, or critical outcomes.A high-risk path has no reason or only an LLM allow.Tighten profile rules or add missing path predicates.
    L4 Mediation Planeallow, audit, require approval, deny, enforced block, and fail-open are visible without coverage inflation.Proof claims enforcement while coverage only shows observed or fail-open.Resolve approvals or strengthen the execution surface.
    L5 Evidence PlaneHashes verify and evidence answers who, what, why, policy, approval, outcome, and coverage.Bundle tamper, missing layer workflow, or incomplete review questions.Regenerate or attach artifacts, then fail the gate if still incomplete.
    
+    
    
+
+    
    
+      
    Runtime stage matrix
    
+      
+        
+        
+        
+        
+        
+        
+      
    Layerbefore-runtimeduring-runtimeafter-runtime
    L1 Execution SurfaceInventory agent, skill, MCP, launcher, wrapper, hook, and unmanaged surfaces.Attribute each action to a runtime surface.Review coverage gaps and unmanaged execution paths.
    L2 Runtime Fact ModelBind session, principal, agent, credential boundary, and grants.Record invocations, resources, taint, decisions, approvals, and outcomes.Keep the ledger as the fact source.
    L3 Decision PlaneLoad deterministic policy and path-aware rules.Explain allow, audit, approval, deny, and critical path decisions.Reconstruct causal chains with policy and graph artifacts.
    L4 Mediation PlaneDecide which surfaces are observed, mediated, enforced, fail-open, or unmanaged.Apply mediation outcomes and approval state.Report unresolved approvals and fail-open conditions truthfully.
    L5 Evidence PlaneRecord claim boundary and expected artifacts.Generate replay, coverage, proof, and graph artifacts.Export audit, evidence bundle, evidence workspace, and release gate inputs.
    
+    
    
+
+    
    
+      
    Read the result
    
+      
    Open layer-runtime-workflow.html for a single-run L1-L5 matrix. Open evidence-workspace.html when a reviewer needs a gateable L5 workspace.
    
+      
    Use Runtime effect demo for risk-shaped demo artifacts, CLI reference for every command group, and Evaluation to connect claims to benchmark evidence.
    
+    
    
+
+    
    
+      
    Boundary
    
+      
    Plugin-only or imported traces can provide useful evidence, but they are not equivalent to Invart mediation or enforcement. Local demos use safe equivalent trajectories and do not claim live external benchmark completion.
    
+    
    
+  
    
+
+
diff --git a/docs/html/index.html b/docs/html/index.html
index 8885f74..caea40e 100644
--- a/docs/html/index.html
+++ b/docs/html/index.html
@@ -15,44 +15,54 @@ 
    Agent Runtime Control Plane
    
     
   
   
    
     
+
     
    
       
    Start
    
       
    
-        
    Product Overview
    What Invart is, who it helps, and how the three-stage control loop works.
    
-        
    Quickstart
    Install locally, create a managed session, record one action, export proof, and verify it.
    
-        
    Examples
    Runnable examples for local sessions, policy profiles, risk demos, and containerized flows.
    
-        
    Runtime Effect Demo
    See before / during / after runtime mapped to L1-L5 with a matrix and action timeline.
    
+        
    Product Overview
    What Invart is, why plugin-only control is not enough, and how the before / during / after runtime loop works.
    
+        
    Quickstart
    Create one managed session, record one action, export proof, and continue into layer inspection.
    
+        
    Runtime Effect Demo
    Read a generated matrix and action timeline from a local demo package.
    
       
    
     
    
+
     
    
-      
    Understand
    
+      
    Operate
    
       
    
-        
    Core Concepts
    Ledger, proof, policy, mediation, path graph, coverage, approval, and evidence bundle.
    
-        
    Architecture
    Three stages, five layers, and how adapters, daemon state, policy, ledger, and proof fit together.
    
-        
    Release History
    A compact public summary of the implementation tracks behind the 0.9 pre-release.
    
+        
    Five-layer Operator Guide
    Run the L1-L5 workflow for a ledger and inspect coverage, mediation, policy, and evidence.
    
+        
    CLI Reference
    Find the command group for scan, run, inspect layers, integrate an agent, export evidence, or evaluate.
    
       
    
     
    
+
     
    
       
    Integrate
    
       
    
-        
    CLI Reference
    The practical command groups users should automate first.
    
-        
    API & SDK
    The stable CLI and artifact contracts, plus provisional Python helper entry points.
    
+        
    API & SDK
    Stable CLI and artifact contracts, plus provisional Python helper entry points.
    
+        
    Architecture
    Five layers, source layout, and integration boundaries for implementers.
    
       
    
     
    
+
     
    
       
    Evaluate
    
       
    
-        
    Evaluation
    Built-in benchmarks, product-effectiveness metrics, and external validation boundaries.
    
-        
    Open-source Boundary
    What is intended for GitHub and what stays in local internal planning.
    
+        
    Evaluation
    Map product claims to benchmark suites, metrics, evidence artifacts, and external-validation boundaries.
    
+        
    Examples
    Small safe fixtures and runnable examples before larger demos.
    
+      
    
+    
    
+
+    
    
+      
    Reference
    
+      
    
+        
    Core Concepts
    Glossary for ledger, proof, policy, mediation, path graph, coverage, approval, and evidence bundle.
    
+        
    Release History
    Compact public summary of the 0.9 pre-release capability tracks.
    
+        
    Open-source Boundary
    What belongs in the public repository and what remains local-only planning material.
    
       
    
     
    
   
    
diff --git a/docs/html/product.html b/docs/html/product.html
index 91fce84..4b52dce 100644
--- a/docs/html/product.html
+++ b/docs/html/product.html
@@ -13,6 +13,7 @@ 
    The local-first control plane for AI agent runtime risk.
    
     
    Invart wraps agent execution with identity, policy, mediation, ledger-backed evidence, and post-run audit. It is built for teams that want agent autonomy without blind trust.
    @@ -57,6 +58,19 @@

    Five Control Layers

    L5 Evidence Plane

    Proof, replay, audit, path graph, evidence bundle, benchmark metrics, and release gates.

    +
    +
    +

    Core Evidence Terms

    + + + + + + + +
    TermWhy it matters
    LedgerThe append-only fact source. Proof, replay, graph, audit, and gates derive from it.
    ProofA portable summary that should answer who, what, why, policy, approval, outcome, and coverage.
    MediationThe decision contract for allow, audit, require approval, deny, enforced block, and fail-open alert.
    CoverageThe strength of observation or control. observed, mediated, enforced, and fail-open are not interchangeable.
    Evidence bundleA verifiable package containing manifest hashes and post-run artifacts.
    +
    +

    What You Can Show A Team

    @@ -69,7 +83,7 @@

    What You Can Show A Team

    How To Verify The Model In A Demo

    -

    Run invart demo pre-1.0-final --out-dir .invart/pre-1-demo-check, then open pre-1.0-final-demo.html. The Runtime Effect Matrix maps before-runtime, during-runtime, and after-runtime to L1-L5, while each container case audit page shows agent intent, Invart observation, policy or mediation decision, outcome, and artifact.

    +

    Run invart demo pre-1.0-final --out-dir .invart/pre-1-demo-check, then open pre-1.0-final-demo.html. The Runtime Effect Matrix maps before-runtime, during-runtime, and after-runtime to L1-L5, while each container case audit page shows agent intent, Invart observation, policy or mediation decision, outcome, and artifact. Use Quickstart for the smallest local run and Operate L1-L5 for command-level inspection.

    diff --git a/docs/html/quickstart.html b/docs/html/quickstart.html index ec92488..e11acda 100644 --- a/docs/html/quickstart.html +++ b/docs/html/quickstart.html @@ -2,7 +2,7 @@ Invart Quickstart -
    Quickstart

    Run one managed Invart session.

    This path creates a ledger, records a runtime command, exports proof, and verifies the proof against the ledger.

    +
    Quickstart

    Run one managed Invart session.

    This path creates a ledger, records a runtime command, exports proof, verifies it, and points you to L1-L5 inspection.

    Install

    @@ -40,6 +40,19 @@

    Verify

    --ledger .invart/quickstart/ledger.jsonl

    The ledger is the fact source. The proof is a portable summary that can be shared with a gate, reviewer, or audit process.

    +
    +

    Inspect L1-L5

    + 
    invart runtime layers \
+  --ledger .invart/quickstart/ledger.jsonl \
+  --out-dir .invart/quickstart/layers
+
+invart evidence inspect \
+  --manifest .invart/quickstart/layers/evidence/manifest.json \
+  --out-dir .invart/quickstart/evidence-workspace \
+  --require-layer-workflow
    +

    Open .invart/quickstart/layers/layer-runtime-workflow.html for the before-runtime, during-runtime, and after-runtime L1-L5 matrix. Open .invart/quickstart/evidence-workspace/evidence-workspace.html to verify who, what, why, policy, approval, outcome, and coverage.

    +

    For interpretation rules, read the five-layer operator guide.

    +
    diff --git a/docs/html/release-history.html b/docs/html/release-history.html index bc3a4f5..b79683f 100644 --- a/docs/html/release-history.html +++ b/docs/html/release-history.html @@ -2,8 +2,11 @@ Invart Release History -
    Release History

    0.9 pre-release boundary.

    Invart has a long local implementation history. The public docs summarize capability areas instead of publishing every internal version-design page.

    +
    Reference

    0.9 pre-release boundary.

    Invart has a long local implementation history. The public docs summarize capability areas instead of publishing every internal version-design page.

    +
    +

    This page is a reference, not the first-run learning path. Start with Product Overview, Quickstart, or the five-layer operator guide.

    +
    diff --git a/docs/html/runtime-effect-demo.html b/docs/html/runtime-effect-demo.html index e59b990..c942697 100644 --- a/docs/html/runtime-effect-demo.html +++ b/docs/html/runtime-effect-demo.html @@ -10,10 +10,11 @@
    Runtime Effect Demo

    See the three stages and five layers in one run.

    -

    Invart's demo artifacts show what was discovered before runtime, mediated during runtime, and reconstructed after runtime.

    +

    Invart's demo artifacts show what was discovered before runtime, mediated during runtime, and reconstructed after runtime. For command-by-command operation, use the five-layer operator guide.

    @@ -47,6 +48,7 @@

    Runtime Effect Matrix

    TrackCapability Area
    FoundationManaged sessions, runtime event normalization, deterministic rules, ledger, proof, proof verification.
    L4 Mediation PlaneRoute covered surfaces through managed launchers, wrappers, hooks, or brokers.Apply allow, audit, require approval, deny, enforced block, or fail-open alert.Show unresolved approvals and coverage gaps truthfully.
    L5 Evidence PlaneRecord the demo claim boundary and artifact map.Generate replay, coverage, path graph, and policy artifacts.Export proof, audit, evidence bundle, evidence workspace, and RC gate report.
    +

    For healthy signals, failure signals, and next actions for each row, open the five-layer operator guide.

    Action Timeline

    diff --git a/docs/index.md b/docs/index.md index dc266b0..c73478d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,11 +1,10 @@ -# Agent Runtime Control Plane +# Invart Docs [HTML version](html/index.html) - Use Invart to discover agent runtime risk before execution, mediate behavior during execution, and produce verifiable evidence after execution. -[Product](product.md) · [Quickstart](quickstart.md) · [CLI](cli-reference.md) · [API & SDK](api-sdk.md) · [Architecture](architecture.md) · [Evaluation](evaluation.md) +[Product](product.md) · [Quickstart](quickstart.md) · [Five-layer operator guide](five-layer-operator-guide.md) · [CLI](cli-reference.md) · [API & SDK](api-sdk.md) · [Evaluation](evaluation.md) ![Invart logo](../assets/brand/png-from-original/invart-logo-docs-header-1200x400.png) @@ -13,50 +12,56 @@ Use Invart to discover agent runtime risk before execution, mediate behavior dur ### [Product Overview](product.md) -What Invart is, who it helps, and how the three-stage control loop works. +Understand what Invart is, why plugin-only control is not enough, and how the before / during / after runtime loop works. ### [Quickstart](quickstart.md) -Install locally, create a managed session, record one action, export proof, and verify it. +Install locally, create one managed session, record one action, export proof, and continue into layer inspection. -### [Examples](examples.md) +### [Runtime Effect Demo](runtime-effect-demo.md) -Runnable examples for local sessions, policy profiles, risk demos, and containerized flows. +Open a generated demo matrix and action timeline when you want to see Invart's product story in one local artifact package. -### [Runtime Effect Demo](runtime-effect-demo.md) +## Operate -Read the demo matrix and action timeline that map before / during / after runtime to L1-L5. +### [Five-layer Operator Guide](five-layer-operator-guide.md) -## Understand +Run the L1-L5 workflow for a ledger, inspect evidence, and decide whether the run was observed, mediated, enforced, fail-open, or unmanaged. -### [Core Concepts](concepts.md) +### [CLI Reference](cli-reference.md) + +Find the command group for a concrete task: scan, run, inspect layers, integrate an agent, export evidence, or evaluate. + +## Integrate + +### [API & SDK](api-sdk.md) -Ledger, proof, policy, mediation, path graph, coverage, approval, and evidence bundle. +Use stable CLI and artifact contracts first, then provisional Python helpers where direct imports are useful. ### [Architecture](architecture.md) -Three stages, five layers, and how adapters, daemon state, policy, ledger, and proof fit together. +Map the five control layers to the Invart source layout and integration boundaries. -### [Release History](release-history.md) +## Evaluate -A compact public summary of the implementation tracks behind the 0.9 pre-release. +### [Evaluation](evaluation.md) -## Integrate +Connect product claims to benchmark suites, evidence artifacts, and external-validation boundaries. -### [CLI Reference](cli-reference.md) +### [Examples](examples.md) -The practical command groups users should automate first. +Find small safe fixtures and runnable examples before using larger demos. -### [API & SDK](api-sdk.md) +## Reference -The stable CLI and artifact contracts, plus provisional Python helper entry points. +### [Core Concepts](concepts.md) -## Evaluate +Glossary-style reference for ledger, proof, policy, mediation, path graph, coverage, approval, and evidence bundle. -### [Evaluation](evaluation.md) +### [Release History](release-history.md) -Built-in benchmarks, product-effectiveness metrics, and external validation boundaries. +Compact public summary of the 0.9 pre-release capability tracks. ### [Open-source Boundary](open-source-boundary.md) -What is intended for GitHub and what stays in local internal planning. +What belongs in the public repository and what remains local-only planning material. diff --git a/docs/product.md b/docs/product.md index e090399..597907d 100644 --- a/docs/product.md +++ b/docs/product.md @@ -5,7 +5,7 @@ Invart wraps agent execution with identity, policy, mediation, ledger-backed evidence, and post-run audit. It is built for teams that want agent autonomy without blind trust. -[Start](quickstart.md) · [Architecture](architecture.md) · [Evaluate](evaluation.md) · [GitHub README](../README.md) +[Start](quickstart.md) · [Operate L1-L5](five-layer-operator-guide.md) · [Architecture](architecture.md) · [Evaluate](evaluation.md) · [GitHub README](../README.md) ## Why Invart Exists @@ -20,7 +20,7 @@ invart demo real-world-risk-cases \ Outputs a safe risk demo with ledger, proof, replay, audit, and evidence artifacts. -For the clearest model walkthrough, open the [runtime effect demo](runtime-effect-demo.md). It maps each demo run across before-runtime, during-runtime, after-runtime and the L1-L5 control layers. +For the clearest model walkthrough, open the [runtime effect demo](runtime-effect-demo.md). To operate the same model on your own ledger, use the [five-layer operator guide](five-layer-operator-guide.md). ## Three Stages @@ -52,6 +52,16 @@ Allow, audit, require approval, deny, enforced block, and fail-open alert with c Proof, replay, audit, path graph, evidence bundle, benchmark metrics, and release gates. +## Core Evidence Terms + +| Term | Why it matters | +| --- | --- | +| Ledger | The append-only fact source. Proof, replay, graph, audit, and gates derive from it. | +| Proof | A portable summary that should answer who, what, why, policy, approval, outcome, and coverage. | +| Mediation | The decision contract for allow, audit, require approval, deny, enforced block, and fail-open alert. | +| Coverage | The strength of observation or control. `observed`, `mediated`, `enforced`, and `fail-open` are not interchangeable. | +| Evidence bundle | A verifiable package containing manifest hashes and post-run artifacts. | + ## What You Can Show A Team ### Security @@ -68,4 +78,4 @@ Low-risk commands continue automatically, while high-risk steps receive precise ## How To Verify The Model In A Demo -Run `invart demo pre-1.0-final --out-dir .invart/pre-1-demo-check`, then open `pre-1.0-final-demo.html`. The Runtime Effect Matrix shows the three lifecycle stages across L1-L5, and each container risk case audit page shows an action timeline from agent intent through Invart observation, policy or mediation decision, outcome, and artifact. +Run `invart demo pre-1.0-final --out-dir .invart/pre-1-demo-check`, then open `pre-1.0-final-demo.html`. The Runtime Effect Matrix shows the three lifecycle stages across L1-L5, and each container risk case audit page shows an action timeline from agent intent through Invart observation, policy or mediation decision, outcome, and artifact. Use [Quickstart](quickstart.md) for the smallest local run and [Operate L1-L5](five-layer-operator-guide.md) for command-level inspection. diff --git a/docs/quickstart.md b/docs/quickstart.md index 89e348b..ba770f4 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -3,7 +3,7 @@ [HTML version](html/quickstart.html) -This path creates a ledger, records a runtime command, exports proof, and verifies the proof against the ledger. +This path creates a ledger, records a runtime command, exports proof, verifies the proof against the ledger, and points you to the five-layer inspection workflow. ## Install @@ -47,3 +47,20 @@ invart proof verify \ ``` The ledger is the fact source. The proof is a portable summary that can be shared with a gate, reviewer, or audit process. + +## Inspect L1-L5 + +```bash +invart runtime layers \ + --ledger .invart/quickstart/ledger.jsonl \ + --out-dir .invart/quickstart/layers + +invart evidence inspect \ + --manifest .invart/quickstart/layers/evidence/manifest.json \ + --out-dir .invart/quickstart/evidence-workspace \ + --require-layer-workflow +``` + +Open `.invart/quickstart/layers/layer-runtime-workflow.html` to see before-runtime, during-runtime, and after-runtime across L1-L5. Open `.invart/quickstart/evidence-workspace/evidence-workspace.html` to verify whether the run can answer who, what, why, policy, approval, outcome, and coverage. + +For layer-by-layer interpretation rules, read the [five-layer operator guide](five-layer-operator-guide.md). For small copyable fixtures, see [Examples](examples.md). diff --git a/docs/release-history.md b/docs/release-history.md index c4e3538..faadb94 100644 --- a/docs/release-history.md +++ b/docs/release-history.md @@ -5,6 +5,8 @@ Invart has a long local implementation history. The public docs summarize capability areas instead of publishing every internal version-design page. +This page is a reference, not the first-run learning path. Start with [Product Overview](product.md), [Quickstart](quickstart.md), or the [five-layer operator guide](five-layer-operator-guide.md). + | Track | Capability Area | | --- | --- | | Foundation | Managed sessions, runtime event normalization, deterministic rules, ledger, proof, proof verification. | diff --git a/docs/runtime-effect-demo.md b/docs/runtime-effect-demo.md index dfe8572..84bc9ce 100644 --- a/docs/runtime-effect-demo.md +++ b/docs/runtime-effect-demo.md @@ -2,9 +2,9 @@ [HTML version](html/runtime-effect-demo.html) -This page explains how to see Invart's three runtime stages and five control layers in the demo artifacts. +This page explains how to read Invart's three runtime stages and five control layers in generated demo artifacts. For the command-by-command operating path, use the [five-layer operator guide](five-layer-operator-guide.md). -The short version: run a demo, open the HTML entrypoint, then read the matrix first and the timeline second. +The short version: run a demo, open the HTML entrypoint, then read the matrix first and the timeline second. The demo shows the effect; the operator guide shows how to run the same layer workflow on your own ledger. ## Run The Demo @@ -19,7 +19,7 @@ For isolated per-risk-case runs: scripts/container-demo.sh all .invart/container-risk-demo ``` -For any existing Invart ledger: +For any existing Invart ledger, the operator path starts here: ```bash PYTHONPATH=src python -m invart.cli runtime layers \ @@ -80,6 +80,8 @@ Open `.invart/evidence-workspace/evidence-workspace.html`. This report verifies | L4 Mediation Plane | `invart mediation inspect --ledger ledger.jsonl` | Pause, block, fail-open, approval, and mediation outcome state. | | L5 Evidence Plane | `invart runtime layers --ledger ledger.jsonl --out-dir .invart/layers` then `invart evidence inspect --manifest .invart/layers/evidence/manifest.json --out-dir .invart/evidence-workspace --require-layer-workflow` | A reviewable stage x layer matrix plus a gateable L5 workspace for proof/replay/graph/coverage/audit links. | +For interpretation rules, healthy signals, failure signals, and next actions for each row, read the [five-layer operator guide](five-layer-operator-guide.md). + ## Boundaries The container demo uses safe equivalent trajectories. It does not install hostile packages or replay private incidents. diff --git a/src/invart/evaluation/release_candidate.py b/src/invart/evaluation/release_candidate.py index c528893..87bd71f 100644 --- a/src/invart/evaluation/release_candidate.py +++ b/src/invart/evaluation/release_candidate.py @@ -66,6 +66,7 @@ "docs/index.md", "docs/product.md", "docs/quickstart.md", + "docs/five-layer-operator-guide.md", "docs/concepts.md", "docs/cli-reference.md", "docs/api-sdk.md", @@ -78,6 +79,7 @@ "docs/html/index.html", "docs/html/product.html", "docs/html/quickstart.html", + "docs/html/five-layer-operator-guide.html", "docs/html/concepts.html", "docs/html/cli-reference.html", "docs/html/api-sdk.html", diff --git a/tests/test_policy_evidence_rc.py b/tests/test_policy_evidence_rc.py index dbaad42..9149def 100644 --- a/tests/test_policy_evidence_rc.py +++ b/tests/test_policy_evidence_rc.py @@ -579,20 +579,28 @@ def test_public_docs_include_api_sdk_page_and_valid_local_links() -> None: api_markdown = docs_dir / "api-sdk.md" runtime_effect_doc = html_dir / "runtime-effect-demo.html" runtime_effect_markdown = docs_dir / "runtime-effect-demo.md" + operator_doc = html_dir / "five-layer-operator-guide.html" + operator_markdown = docs_dir / "five-layer-operator-guide.md" assert api_doc.exists() assert api_markdown.exists() assert runtime_effect_doc.exists() assert runtime_effect_markdown.exists() + assert operator_doc.exists() + assert operator_markdown.exists() assert 'href="api-sdk.html"' in (html_dir / "index.html").read_text(encoding="utf-8") assert 'href="runtime-effect-demo.html"' in (html_dir / "index.html").read_text(encoding="utf-8") + assert 'href="five-layer-operator-guide.html"' in (html_dir / "index.html").read_text(encoding="utf-8") docs_readme = (docs_dir / "README.md").read_text(encoding="utf-8") assert "[`api-sdk.md`](api-sdk.md)" in docs_readme assert "[`html/api-sdk.html`](html/api-sdk.html)" in docs_readme assert "[`runtime-effect-demo.md`](runtime-effect-demo.md)" in docs_readme assert "[`html/runtime-effect-demo.html`](html/runtime-effect-demo.html)" in docs_readme + assert "[`five-layer-operator-guide.md`](five-layer-operator-guide.md)" in docs_readme + assert "[`html/five-layer-operator-guide.html`](html/five-layer-operator-guide.html)" in docs_readme assert "[API and SDK](docs/api-sdk.md)" in (root / "README.md").read_text(encoding="utf-8") assert "[Runtime effect demo](docs/runtime-effect-demo.md)" in (root / "README.md").read_text(encoding="utf-8") + assert "[Five-layer operator guide](docs/five-layer-operator-guide.md)" in (root / "README.md").read_text(encoding="utf-8") assert "[HTML docs home](docs/html/index.html)" in (root / "README.md").read_text(encoding="utf-8") assert sorted(path.name for path in docs_dir.glob("*.html")) == [] @@ -613,6 +621,54 @@ def test_public_docs_include_api_sdk_page_and_valid_local_links() -> None: assert (html_path.parent / local).resolve().exists(), f"{html_path} links to missing {href}" +def test_public_docs_are_organized_by_user_journey() -> None: + root = Path(__file__).resolve().parents[1] + markdown_index = (root / "docs" / "index.md").read_text(encoding="utf-8") + html_index = (root / "docs" / "html" / "index.html").read_text(encoding="utf-8") + + for section in ("## Start", "## Operate", "## Integrate", "## Evaluate", "## Reference"): + assert section in markdown_index + for section in ("Start", "Operate", "Integrate", "Evaluate", "Reference"): + assert f"

    {section}

    " in html_index + assert markdown_index.index("## Operate") < markdown_index.index("## Reference") + assert "five-layer-operator-guide.md" in markdown_index + assert "five-layer-operator-guide.html" in html_index + assert "release-history.md" in markdown_index[markdown_index.index("## Reference") :] + + +def test_five_layer_operator_guide_documents_operational_path() -> None: + root = Path(__file__).resolve().parents[1] + markdown_page = (root / "docs" / "five-layer-operator-guide.md").read_text(encoding="utf-8") + html_page = (root / "docs" / "html" / "five-layer-operator-guide.html").read_text(encoding="utf-8") + + for phrase in ( + "L1 Execution Surface", + "L2 Runtime Fact Model", + "L3 Decision Plane", + "L4 Mediation Plane", + "L5 Evidence Plane", + "before-runtime", + "during-runtime", + "after-runtime", + "observed", + "mediated", + "enforced", + "fail-open", + "invart pre-runtime", + "invart runtime layers", + "invart policy check-path", + "invart mediation inspect", + "invart evidence inspect", + ): + assert phrase in markdown_page + + for href in ("runtime-effect-demo.html", "cli-reference.html", "evaluation.html"): + assert f'href="{href}"' in html_page + assert "Which layer should I use?" in html_page + assert "Healthy signal" in markdown_page + assert "Failure signal" in markdown_page + + def test_api_sdk_page_documents_real_python_helpers() -> None: from invart.core.ledger import load_ledger_entries as documented_load_ledger_entries from invart.core.ledger import verify_ledger as documented_verify_ledger @@ -634,6 +690,8 @@ def test_api_sdk_page_documents_real_python_helpers() -> None: assert "docs/html/api-sdk.html" in DEFAULT_REQUIRED_DOCS assert "docs/runtime-effect-demo.md" in DEFAULT_REQUIRED_DOCS assert "docs/html/runtime-effect-demo.html" in DEFAULT_REQUIRED_DOCS + assert "docs/five-layer-operator-guide.md" in DEFAULT_REQUIRED_DOCS + assert "docs/html/five-layer-operator-guide.html" in DEFAULT_REQUIRED_DOCS assert callable(documented_load_ledger_entries) assert callable(documented_verify_ledger) diff --git a/uv.lock b/uv.lock index 7fe47ab..a458b20 100644 --- a/uv.lock +++ b/uv.lock @@ -4,5 +4,5 @@ requires-python = ">=3.9" [[package]] name = "invart" -version = "0.9.0" +version = "0.9.2" source = { editable = "." } From 033d47c1ecb00975fdb8eefab739fda26bc537c5 Mon Sep 17 00:00:00 2001 From: Kappaski Maintainer Date: Thu, 11 Jun 2026 14:51:43 +0800 Subject: [PATCH 3/3] docs: tighten readme documentation entry --- README.md | 35 +++++++++----------- tests/test_policy_evidence_rc.py | 57 +++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index fb5da13..ac04f9c 100644 --- a/README.md +++ b/README.md @@ -36,20 +36,21 @@ Invart is currently a CLI-first **0.9 pre-release**. It is suitable for local ev - Run built-in benchmarks and demos, including real-world-risk-shaped and containerized risk flows. - Keep old `kappaski` imports and CLI as compatibility aliases while the public project name moves to `Invart`. -## What To Inspect First +## Start Here -For a first pass, the most useful public-facing surfaces are: +README is the landing page. The docs are organized as a user journey: -| Surface | Why it matters | +| If you want to... | Start with | | --- | --- | -| [`docs/product.md`](docs/product.md) | One-page product overview for the before / during / after control loop. | -| [`docs/quickstart.md`](docs/quickstart.md) | Minimal local session that writes a ledger and verifies proof. | -| [`docs/five-layer-operator-guide.md`](docs/five-layer-operator-guide.md) | Practical guide for operating L1-L5 on a real ledger. | -| [`docs/runtime-effect-demo.md`](docs/runtime-effect-demo.md) | Shows the demo's before / during / after stages and L1-L5 control effects. | -| [`docs/evaluation.md`](docs/evaluation.md) | Explains what the built-in benchmarks measure and what they do not claim. | -| [`examples/`](examples/) | Small runnable examples before running the larger demos. | +| Understand what Invart is and why it exists | [`docs/product.md`](docs/product.md) | +| Run one local managed session | [`docs/quickstart.md`](docs/quickstart.md) | +| Operate the five runtime layers on a ledger | [`docs/five-layer-operator-guide.md`](docs/five-layer-operator-guide.md) | +| See the before / during / after demo artifacts | [`docs/runtime-effect-demo.md`](docs/runtime-effect-demo.md) | +| Integrate with CLI, artifacts, or Python helpers | [`docs/cli-reference.md`](docs/cli-reference.md) and [`docs/api-sdk.md`](docs/api-sdk.md) | +| Evaluate claims and benchmarks | [`docs/evaluation.md`](docs/evaluation.md) | +| Browse the full documentation map | [`docs/index.md`](docs/index.md) or [`docs/html/index.html`](docs/html/index.html) | -The main demo command below produces a local HTML entrypoint plus audit, replay, path graph, coverage, proof, and ledger artifacts. Those artifacts are the best way to see Invart intervening in and explaining an agent-like run. +The recommended first path is: **Quickstart -> Five-layer operator guide -> Runtime effect demo -> Evaluation**. That path shows the whole loop from a local ledger to proof, replay, path graph, coverage, audit, and evidence review. ## Install For Local Development @@ -142,21 +143,17 @@ Heavy external validation such as full SWE-Bench runs is intentionally separate ## Documentation -Open the local docs: +Open the docs home for the structured journey: -- [Docs README](docs/README.md) - [Markdown docs home](docs/index.md) - [HTML docs home](docs/html/index.html) -- [Product page](docs/product.md) -- [Quickstart](docs/quickstart.md) -- [Five-layer operator guide](docs/five-layer-operator-guide.md) + +Reference pages: + +- [Docs README](docs/README.md) - [Concepts](docs/concepts.md) -- [CLI reference](docs/cli-reference.md) -- [API and SDK](docs/api-sdk.md) - [Architecture](docs/architecture.md) - [Examples](docs/examples.md) -- [Runtime effect demo](docs/runtime-effect-demo.md) -- [Evaluation](docs/evaluation.md) - [Open-source boundary](docs/open-source-boundary.md) - [Brand assets](assets/brand/README.md) diff --git a/tests/test_policy_evidence_rc.py b/tests/test_policy_evidence_rc.py index 9149def..6c723cb 100644 --- a/tests/test_policy_evidence_rc.py +++ b/tests/test_policy_evidence_rc.py @@ -1,4 +1,5 @@ import json +import re import sys from html.parser import HTMLParser from pathlib import Path @@ -74,6 +75,10 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None self.hrefs.append(value) +def _markdown_links(text: str) -> list[str]: + return [match.group(1).split("#", 1)[0] for match in re.finditer(r"\[[^\]]+\]\(([^)]+)\)", text)] + + def _write_full_swebench_evidence_fixture(root: Path, *, run_id: str = "invart_full", total: int = 3) -> dict[str, Path]: results = root / "results" run_dir = results / run_id @@ -598,10 +603,14 @@ def test_public_docs_include_api_sdk_page_and_valid_local_links() -> None: assert "[`html/runtime-effect-demo.html`](html/runtime-effect-demo.html)" in docs_readme assert "[`five-layer-operator-guide.md`](five-layer-operator-guide.md)" in docs_readme assert "[`html/five-layer-operator-guide.html`](html/five-layer-operator-guide.html)" in docs_readme - assert "[API and SDK](docs/api-sdk.md)" in (root / "README.md").read_text(encoding="utf-8") - assert "[Runtime effect demo](docs/runtime-effect-demo.md)" in (root / "README.md").read_text(encoding="utf-8") - assert "[Five-layer operator guide](docs/five-layer-operator-guide.md)" in (root / "README.md").read_text(encoding="utf-8") - assert "[HTML docs home](docs/html/index.html)" in (root / "README.md").read_text(encoding="utf-8") + readme = (root / "README.md").read_text(encoding="utf-8") + for href in ( + "docs/api-sdk.md", + "docs/runtime-effect-demo.md", + "docs/five-layer-operator-guide.md", + "docs/html/index.html", + ): + assert f"({href})" in readme assert sorted(path.name for path in docs_dir.glob("*.html")) == [] markdown_pages = [path for path in docs_dir.glob("*.md") if path.name != "README.md"] @@ -621,6 +630,46 @@ def test_public_docs_include_api_sdk_page_and_valid_local_links() -> None: assert (html_path.parent / local).resolve().exists(), f"{html_path} links to missing {href}" +def test_markdown_docs_have_valid_local_links() -> None: + root = Path(__file__).resolve().parents[1] + markdown_pages = [root / "README.md", *sorted((root / "docs").glob("*.md"))] + + missing: list[str] = [] + for markdown_path in markdown_pages: + for href in _markdown_links(markdown_path.read_text(encoding="utf-8")): + if not href or href.startswith(("http://", "https://", "mailto:")): + continue + target = (markdown_path.parent / href).resolve() + if not target.exists(): + missing.append(f"{markdown_path.relative_to(root)} -> {href}") + + assert missing == [] + + +def test_readme_routes_users_into_documentation_journey() -> None: + root = Path(__file__).resolve().parents[1] + readme = (root / "README.md").read_text(encoding="utf-8") + + assert "## Start Here" in readme + assert "README is the landing page" in readme + assert "Quickstart -> Five-layer operator guide -> Runtime effect demo -> Evaluation" in readme + assert readme.index("## Start Here") < readme.index("## Install For Local Development") + + for href in ( + "docs/product.md", + "docs/quickstart.md", + "docs/five-layer-operator-guide.md", + "docs/runtime-effect-demo.md", + "docs/cli-reference.md", + "docs/api-sdk.md", + "docs/evaluation.md", + "docs/index.md", + "docs/html/index.html", + ): + assert f"({href})" in readme + assert (root / href).exists() + + def test_public_docs_are_organized_by_user_journey() -> None: root = Path(__file__).resolve().parents[1] markdown_index = (root / "docs" / "index.md").read_text(encoding="utf-8")