From 98d883f44101d0cdd56e8e7ddb9e68bf1870a602 Mon Sep 17 00:00:00 2001 From: Kappaski Maintainer Date: Mon, 8 Jun 2026 16:14:07 +0800 Subject: [PATCH] feat(demo): show runtime effects across control layers Add runtime effect matrices and per-case action timelines so demos show how Invart observes, mediates, and preserves evidence across before, during, and after runtime. --- README.md | 2 + docs/README.md | 1 + docs/architecture.md | 6 + docs/html/architecture.html | 5 + docs/html/index.html | 2 + docs/html/product.html | 6 +- docs/html/runtime-effect-demo.html | 70 ++++++++++++ docs/index.md | 4 + docs/product.md | 6 + docs/runtime-effect-demo.md | 54 +++++++++ src/invart/evaluation/container_demo.py | 127 ++++++++++++++++++--- src/invart/evaluation/pre_1_0.py | 74 ++++++++++++ src/invart/evaluation/release_candidate.py | 2 + tests/test_demos_roadmap.py | 23 ++++ tests/test_policy_evidence_rc.py | 10 ++ 15 files changed, 378 insertions(+), 14 deletions(-) create mode 100644 docs/html/runtime-effect-demo.html create mode 100644 docs/runtime-effect-demo.md diff --git a/README.md b/README.md index e2afec9..1ccc102 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ For a first pass, the most useful public-facing surfaces are: | --- | --- | | [`docs/product.md`](docs/product.md) | One-page product overview for the before / during / after control loop. | | [`docs/quickstart.md`](docs/quickstart.md) | Minimal local session that writes a ledger and verifies proof. | +| [`docs/runtime-effect-demo.md`](docs/runtime-effect-demo.md) | Shows the demo's before / during / after stages and L1-L5 control effects. | | [`docs/evaluation.md`](docs/evaluation.md) | Explains what the built-in benchmarks measure and what they do not claim. | | [`examples/`](examples/) | Small runnable examples before running the larger demos. | @@ -152,6 +153,7 @@ Open the local docs: - [API and SDK](docs/api-sdk.md) - [Architecture](docs/architecture.md) - [Examples](docs/examples.md) +- [Runtime effect demo](docs/runtime-effect-demo.md) - [Evaluation](docs/evaluation.md) - [Open-source boundary](docs/open-source-boundary.md) - [Brand assets](assets/brand/README.md) diff --git a/docs/README.md b/docs/README.md index 5c1fc78..a0bae05 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,6 +20,7 @@ Public pages: | CLI reference | [`cli-reference.md`](cli-reference.md) | [`html/cli-reference.html`](html/cli-reference.html) | | API and SDK | [`api-sdk.md`](api-sdk.md) | [`html/api-sdk.html`](html/api-sdk.html) | | Examples | [`examples.md`](examples.md) | [`html/examples.html`](html/examples.html) | +| Runtime effect demo | [`runtime-effect-demo.md`](runtime-effect-demo.md) | [`html/runtime-effect-demo.html`](html/runtime-effect-demo.html) | | Architecture | [`architecture.md`](architecture.md) | [`html/architecture.html`](html/architecture.html) | | Evaluation | [`evaluation.md`](evaluation.md) | [`html/evaluation.html`](html/evaluation.html) | | Open-source boundary | [`open-source-boundary.md`](open-source-boundary.md) | [`html/open-source-boundary.html`](html/open-source-boundary.html) | diff --git a/docs/architecture.md b/docs/architecture.md index e6e5ef6..454098a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,6 +17,12 @@ Invart is not a plugin-only wrapper. Plugin and hook integration can improve cov | L4 Mediation Plane | Unified decision and outcome contract across surfaces. | control/mediation.py, control/approval.py, control/gate.py | | L5 Evidence Plane | Proof, replay, graph, audit, coverage, evidence bundle, release gate. | assurance/postruntime.py, assurance/replay.py, assurance/path_graph.py, assurance/evidence_bundle.py | +## Demo Verification + +The [runtime effect demo](runtime-effect-demo.md) is the concrete verification surface for this model. It renders a stage × layer matrix in the final demo entrypoint and an action timeline in each container risk case audit page. The timeline connects agent intent/action, Invart observation, policy or mediation decision, outcome, and the artifact that proves the step. + +Coverage terms remain strict: observed, mediated, and enforced are not interchangeable. Unmanaged direct execution is shown as a coverage gap unless it enters a managed launcher, wrapper, hook, broker, or shim. + ## Source Layout ```bash diff --git a/docs/html/architecture.html b/docs/html/architecture.html index 86a76fc..3390362 100644 --- a/docs/html/architecture.html +++ b/docs/html/architecture.html @@ -19,6 +19,11 @@

Layer Model

L5 Evidence PlaneProof, replay, graph, audit, coverage, evidence bundle, release gate.assurance/postruntime.py, assurance/replay.py, assurance/path_graph.py, assurance/evidence_bundle.py +
+

Demo Verification

+

Runtime Effect Demo is the concrete verification surface for this architecture. It renders a stage × layer matrix in the final demo entrypoint and an action timeline in each container risk case audit page.

+

Coverage terms remain strict: observed, mediated, and enforced are not interchangeable. Unmanaged direct execution is shown as a coverage gap unless it enters a managed launcher, wrapper, hook, broker, or shim.

+

Source Layout

src/invart/core/       fact model, ledger, stable artifact helpers
diff --git a/docs/html/index.html b/docs/html/index.html
index 31278b9..8885f74 100644
--- a/docs/html/index.html
+++ b/docs/html/index.html
@@ -17,6 +17,7 @@ 
Agent Runtime Control Plane

       Quickstart
       CLI
       API & SDK
+      Runtime Effects
       Architecture
       Evaluation
     
@@ -29,6 +30,7 @@ 
Start

         
Product Overview
What Invart is, who it helps, and how the three-stage control loop works.

         
Quickstart
Install locally, create a managed session, record one action, export proof, and verify it.

         
Examples
Runnable examples for local sessions, policy profiles, risk demos, and containerized flows.

+        
Runtime Effect Demo
See before / during / after runtime mapped to L1-L5 with a matrix and action timeline.
diff --git a/docs/html/product.html b/docs/html/product.html index 709a32a..91fce84 100644 --- a/docs/html/product.html +++ b/docs/html/product.html @@ -33,7 +33,7 @@

Why Invart Exists

Quick Demo

invart demo real-world-risk-cases \
   --out-dir .invart/real-world-risk-demo
-

Outputs a safe risk demo with ledger, proof, replay, audit, and evidence artifacts.

+

Outputs a safe risk demo with ledger, proof, replay, audit, and evidence artifacts. For the clearest walkthrough, open Runtime Effect Demo.

@@ -67,6 +67,10 @@

What You Can Show A Team

+
+

How To Verify The Model In A Demo

+

Run invart demo pre-1.0-final --out-dir .invart/pre-1-demo-check, then open pre-1.0-final-demo.html. The Runtime Effect Matrix maps before-runtime, during-runtime, and after-runtime to L1-L5, while each container case audit page shows agent intent, Invart observation, policy or mediation decision, outcome, and artifact.

+
diff --git a/docs/html/runtime-effect-demo.html b/docs/html/runtime-effect-demo.html new file mode 100644 index 0000000..69d624a --- /dev/null +++ b/docs/html/runtime-effect-demo.html @@ -0,0 +1,70 @@ + + + + + + Invart Runtime Effect Demo + + + +
+ Runtime Effect Demo +

See the three stages and five layers in one run.

+

Invart's demo artifacts show what was discovered before runtime, mediated during runtime, and reconstructed after runtime.

+ +
+
+
+

Run

+ 
PYTHONPATH=src python -m invart.cli demo pre-1.0-final \
+  --out-dir .invart/pre-1-demo-check
+
+scripts/container-demo.sh all .invart/container-risk-demo
+
+
+

Runtime Effect Matrix

+ + + + + + + +
LayerBefore runtimeDuring runtimeAfter runtime
L1 Execution SurfaceInventory agent config, skills, MCP, wrappers, launchers, and unmanaged surfaces.Observe file, network, shell, content, skill, and tool activity.Review where coverage was strong, weak, or missing.
L2 Runtime Fact ModelBind principal, agent identity, credentials, grants, and session.Normalize invocations with resources, taint, coverage, decisions, and outcomes.Use the ledger as the fact source.
L3 Decision PlaneLoad deterministic policy and path-aware risk rules.Classify secret egress, unsafe deletion, external instruction, and suspicious skill paths.Explain why an action was allowed, paused, denied, or blocked.
L4 Mediation PlaneRoute covered surfaces through managed launchers, wrappers, hooks, or brokers.Apply allow, audit, require approval, deny, enforced block, or fail-open alert.Show unresolved approvals and coverage gaps truthfully.
L5 Evidence PlaneRecord the demo claim boundary and artifact map.Generate replay, coverage, path graph, and policy artifacts.Export proof, audit, evidence bundle, and RC gate report.
+
+
+

Action Timeline

+
+ Agent intent/action-> + Invart observation-> + Policy / mediation decision-> + Outcome-> + Artifact +
+

Each container risk case audit page renders this sequence as a table so a reviewer can trace the run without reading raw JSON first.

+
+
+

Artifact Map

+ + + + + + + + +
ArtifactLayer signalUse it for
ledger.jsonlL2 fact sourceVerifying the append-only event chain.
proof.jsonL5 portable summaryAnswering who, what, why, policy, approval, outcome, and coverage.
replay.htmlL4 mediation traceReading approvals, denials, raw evidence folds, and outcomes.
path-graph.htmlL3 causal chainTracing risky paths such as secret read to network sink.
coverage.htmlL4/L5 truthfulnessChecking observed, mediated, enforced, and fail-open claims.
audit-report.htmlL5 reviewer reportPresenting the enterprise security review view.
+
+
+

Boundary

+

The container demo uses safe equivalent trajectories. It does not install hostile packages or replay private incidents.

+

observed, mediated, and enforced are separate claims. Unmanaged direct agent execution is shown as a coverage gap unless it enters a managed Invart surface.

+
+
+ + diff --git a/docs/index.md b/docs/index.md index c3d6442..dc266b0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -23,6 +23,10 @@ Install locally, create a managed session, record one action, export proof, and Runnable examples for local sessions, policy profiles, risk demos, and containerized flows. +### [Runtime Effect Demo](runtime-effect-demo.md) + +Read the demo matrix and action timeline that map before / during / after runtime to L1-L5. + ## Understand ### [Core Concepts](concepts.md) diff --git a/docs/product.md b/docs/product.md index 52b08e1..e090399 100644 --- a/docs/product.md +++ b/docs/product.md @@ -20,6 +20,8 @@ invart demo real-world-risk-cases \ Outputs a safe risk demo with ledger, proof, replay, audit, and evidence artifacts. +For the clearest model walkthrough, open the [runtime effect demo](runtime-effect-demo.md). It maps each demo run across before-runtime, during-runtime, after-runtime and the L1-L5 control layers. + ## Three Stages | Stage | Question | Invart Output | @@ -63,3 +65,7 @@ Whether managed launchers, wrappers, hooks, and benchmark harnesses can run with ### Developers Low-risk commands continue automatically, while high-risk steps receive precise context instead of vague warnings. + +## How To Verify The Model In A Demo + +Run `invart demo pre-1.0-final --out-dir .invart/pre-1-demo-check`, then open `pre-1.0-final-demo.html`. The Runtime Effect Matrix shows the three lifecycle stages across L1-L5, and each container risk case audit page shows an action timeline from agent intent through Invart observation, policy or mediation decision, outcome, and artifact. diff --git a/docs/runtime-effect-demo.md b/docs/runtime-effect-demo.md new file mode 100644 index 0000000..904ef65 --- /dev/null +++ b/docs/runtime-effect-demo.md @@ -0,0 +1,54 @@ +# Runtime effect demo + +[HTML version](html/runtime-effect-demo.html) + +This page explains how to see Invart's three runtime stages and five control layers in the demo artifacts. + +The short version: run a demo, open the HTML entrypoint, then read the matrix first and the timeline second. + +## Run The Demo + +```bash +PYTHONPATH=src python -m invart.cli demo pre-1.0-final \ + --out-dir .invart/pre-1-demo-check +``` + +For isolated per-risk-case runs: + +```bash +scripts/container-demo.sh all .invart/container-risk-demo +``` + +## What To Look For + +| View | What it proves | Main artifact | +| --- | --- | --- | +| Runtime Effect Matrix | Before, during, and after runtime are mapped to L1-L5. | `pre-1.0-final-demo.html` | +| Action Timeline | One case shows agent action, Invart observation, policy decision, outcome, and artifact. | `container-risk-audit.html` | +| Path Graph | Risk chains such as secret read to external network sink can be traced. | `path-graph.html` | +| Coverage Report | Observed, mediated, and enforced are reported as different claims. | `coverage.html` | +| Audit Report | A reviewer can answer who, what, why, policy, approval, outcome, and coverage. | `audit-report.html` | + +## Three Runtime Stages + +| Stage | Demo signal | +| --- | --- | +| Before runtime | Surface inventory, unmanaged agent findings, identity binding, credential boundary, and grant setup. | +| During runtime | File, network, shell, skill, content, and mediation events become ledger-backed facts. | +| After runtime | Proof, replay, path graph, coverage, audit, and evidence bundles reconstruct what happened. | + +## Five Control Layers + +| Layer | Demo effect | +| --- | --- | +| L1 Execution Surface | Shows where commands, files, network, skills, MCP, launchers, and hooks enter the control plane. | +| L2 Runtime Fact Model | Shows invocations, taint, identity, resources, coverage, outcomes, and ledger records. | +| L3 Decision Plane | Shows deterministic rules, path-aware policy, and non-downgradable critical findings. | +| L4 Mediation Plane | Shows allow, audit, require approval, deny, enforced block, and fail-open alert semantics. | +| L5 Evidence Plane | Shows proof, replay, graph, coverage, audit, evidence bundle, and release gate outputs. | + +## Boundaries + +The container demo uses safe equivalent trajectories. It does not install hostile packages or replay private incidents. + +Coverage labels are intentionally strict. `observed`, `mediated`, and `enforced` are not interchangeable, and unmanaged direct agent execution is shown as a coverage gap unless it enters a managed Invart surface. diff --git a/src/invart/evaluation/container_demo.py b/src/invart/evaluation/container_demo.py index 3ba6b58..dcf93d5 100644 --- a/src/invart/evaluation/container_demo.py +++ b/src/invart/evaluation/container_demo.py @@ -145,6 +145,18 @@ def run_container_risk_case(case_id: str, out_dir: Path) -> dict[str, Any]: gate = verify_gate(ledger_path=ledger, proof_path=proof_path, mode="managed") mediation = replay_mediation(ledger) audit_path = out_dir / "container-risk-audit.html" + artifacts = { + "ledger": str(ledger), + "proof": str(proof_path), + "replay": replay["replay"], + "path_graph": graph_html["output"], + "path_graph_json": graph_json["output"], + "path_policy": str(out_dir / "path-policy.json"), + "coverage_report": coverage["output"], + "audit_report": str(audit_path), + "case_json": str(out_dir / CASE_RESULT_NAME), + } + layer_timeline = _layer_timeline(case, artifacts, gate, path_policy, mediation) result = { "schema_version": SCHEMA_VERSION, @@ -153,17 +165,9 @@ def run_container_risk_case(case_id: str, out_dir: Path) -> dict[str, Any]: "public_sources": sources, "container": _container_context(), "summary": _case_summary(proof, gate, path_policy, mediation), - "artifacts": { - "ledger": str(ledger), - "proof": str(proof_path), - "replay": replay["replay"], - "path_graph": graph_html["output"], - "path_graph_json": graph_json["output"], - "path_policy": str(out_dir / "path-policy.json"), - "coverage_report": coverage["output"], - "audit_report": str(audit_path), - "case_json": str(out_dir / CASE_RESULT_NAME), - }, + "artifacts": artifacts, + "layer_timeline": layer_timeline, + "runtime_effect_matrix": _runtime_effect_matrix_for_case(layer_timeline), "gate": gate, "path_policy": path_policy, "mediation": mediation, @@ -188,6 +192,7 @@ def run_container_risk_suite(out_dir: Path, *, collect_existing: bool = False) - "container_isolated_cases": sum(1 for case in cases if case.get("container", {}).get("case")), "blocked_or_paused_cases": sum(1 for case in cases if case.get("summary", {}).get("blocked_or_paused")), "source_mapped_cases": sum(1 for case in cases if case.get("public_sources")), + "runtime_effect_cases": sum(1 for case in cases if case.get("runtime_effect_matrix") and case.get("layer_timeline")), }, "cases": cases, "artifacts": {"suite_json": str(out_dir / SUITE_RESULT_NAME), "suite_html": str(html_path)}, @@ -207,6 +212,7 @@ def run_container_risk_demo_benchmark() -> dict[str, Any]: "each_case_has_public_sources": all(case.get("public_sources") for case in suite["cases"]), "each_case_has_core_artifacts": all(_case_has_core_artifacts(case) for case in suite["cases"]), "each_case_has_block_or_pause_signal": all(case.get("summary", {}).get("blocked_or_paused") for case in suite["cases"]), + "each_case_has_runtime_effect_model": all(case.get("runtime_effect_matrix") and case.get("layer_timeline") for case in suite["cases"]), "suite_html_exists": Path(suite["artifacts"]["suite_html"]).exists(), "case_output_dirs_are_distinct": len({Path(case["artifacts"]["case_json"]).parent for case in suite["cases"]}) == 3, } @@ -354,6 +360,86 @@ def _case_has_core_artifacts(case: dict[str, Any]) -> bool: return all(Path(case["artifacts"][key]).exists() for key in required) +def _layer_timeline( + case: ContainerRiskCase, + artifacts: dict[str, str], + gate: dict[str, Any], + path_policy: dict[str, Any], + mediation: dict[str, Any], +) -> list[dict[str, Any]]: + gate_status = str(gate.get("status", "unknown")) + path_status = str(path_policy.get("status", "unknown")) + mediation_summary = mediation.get("summary", {}) if isinstance(mediation.get("summary"), dict) else {} + paused = int(mediation_summary.get("paused", 0) or 0) + blocked = int(mediation_summary.get("blocked", 0) or 0) + mediation_effect = "pause/deny" if paused or blocked or gate_status == "fail" or path_status == "fail" else "allow/audit" + return [ + { + "stage": "before-runtime", + "layer": "L1", + "layer_name": "Execution Surface", + "agent_intent": case.objective, + "invart_observation": "Session launch, declared agent, skill/file/network/shell surfaces, and safe equivalent risk source are inventoried.", + "decision": "surface discovered", + "outcome": "case enters a managed Invart run instead of an unobserved agent path", + "artifact": artifacts["ledger"], + }, + { + "stage": "during-runtime", + "layer": "L2", + "layer_name": "Runtime Fact Model", + "agent_intent": "Agent-like action touches command, file, network, skill, or content surface.", + "invart_observation": "Runtime events are normalized into invocations with identity, taint, coverage, resource, and outcome facts.", + "decision": "facts recorded", + "outcome": "ledger preserves the accountable trajectory", + "artifact": artifacts["ledger"], + }, + { + "stage": "during-runtime", + "layer": "L3", + "layer_name": "Decision Plane", + "agent_intent": "Risky path is evaluated against deterministic and path-aware policy.", + "invart_observation": f"Path policy status is {path_status}; deterministic critical findings cannot be downgraded.", + "decision": path_status, + "outcome": case.expected_invart_action, + "artifact": artifacts["path_policy"], + }, + { + "stage": "during-runtime", + "layer": "L4", + "layer_name": "Mediation Plane", + "agent_intent": "The action attempts to continue toward a side effect.", + "invart_observation": f"Mediation summary is {json.dumps(mediation_summary, ensure_ascii=False, sort_keys=True)}.", + "decision": mediation_effect, + "outcome": "high-risk action is paused, denied, or blocked while low-risk work can continue", + "artifact": artifacts["replay"], + }, + { + "stage": "after-runtime", + "layer": "L5", + "layer_name": "Evidence Plane", + "agent_intent": "Reviewer needs to reconstruct what happened and why.", + "invart_observation": "Proof, replay, path graph, coverage, audit, and manifest-style artifacts explain the run.", + "decision": gate_status, + "outcome": "security review can inspect who, what, why, policy, approval, outcome, and coverage", + "artifact": artifacts["proof"], + }, + ] + + +def _runtime_effect_matrix_for_case(layer_timeline: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + { + "stage": item["stage"], + "layer": item["layer"], + "layer_name": item["layer_name"], + "effect": item["outcome"], + "artifact": item["artifact"], + } + for item in layer_timeline + ] + + def _load_existing_case_results(out_dir: Path) -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] for path in sorted(out_dir.glob(f"*/{CASE_RESULT_NAME}")): @@ -379,8 +465,12 @@ def _render_case_html(result: dict[str, Any]) -> str: ("ledger", artifacts["ledger"]), ] ) + timeline_rows = "".join( + f"{esc(item['stage'])}{esc(item['layer'])} {esc(item['layer_name'])}{esc(item['agent_intent'])}{esc(item['invart_observation'])}{esc(item['decision'])}{esc(item['outcome'])}{esc(Path(item['artifact']).name)}" + for item in result["layer_timeline"] + ) raw = esc(json.dumps(result["summary"], ensure_ascii=False, indent=2, sort_keys=True)) - return f"""{esc(result['case']['title'])}
container risk case

{esc(result['case']['title'])}

{esc(result['case']['objective'])}

Invart Action

{esc(result['case']['expected_invart_action'])}

{raw}

Artifacts

{artifact_links}

Public Source Seeds

{source_rows}
SourceTypeURLEvidence Anchor
""" + return f"""{esc(result['case']['title'])}
container risk case

{esc(result['case']['title'])}

{esc(result['case']['objective'])}

Invart Action

{esc(result['case']['expected_invart_action'])}

{raw}

Action Timeline

{timeline_rows}
StageLayerAgent intent/actionInvart observationPolicy / mediation decisionOutcomeArtifact

Artifacts

{artifact_links}

Public Source Seeds

{source_rows}
SourceTypeURLEvidence Anchor
""" def _render_suite_html(result: dict[str, Any]) -> str: @@ -390,7 +480,18 @@ def _render_suite_html(result: dict[str, Any]) -> str: f"

{esc(case['case']['title'])}

{esc(case['case']['risk_class'])}

Status: {esc(case['status'])}; gate: {esc(str(case['summary']['gate_status']))}; blocked/paused: {esc(str(case['summary']['blocked_or_paused']))}

audit · replay · path graph

" for case in result["cases"] ) - return f"""Invart Containerized Risk Demo

Invart Containerized Risk Demo

Each card is produced by an isolated container run for one safe equivalent risk trajectory.

Run

scripts/container-demo.sh all .invart/container-risk-demo

{esc(result['claim_boundary'])}

Cases

{cards}
""" + layer_rows = "".join(_suite_layer_rows(case) for case in result["cases"]) + return f"""Invart Containerized Risk Demo

Invart Containerized Risk Demo

Each card is produced by an isolated container run for one safe equivalent risk trajectory.

Run

scripts/container-demo.sh all .invart/container-risk-demo

{esc(result['claim_boundary'])}

Coverage labels are truthful: observed / mediated / enforced are separate claims.

Cases

{cards}

Layer Effect Summary

{layer_rows}
CaseL1 Execution SurfaceL2 Runtime FactsL3 Decision PlaneL4 Mediation PlaneL5 Evidence Plane
""" + + +def _suite_layer_rows(case: dict[str, Any]) -> str: + esc = html.escape + by_layer = {item["layer"]: item for item in case.get("layer_timeline", [])} + cells = [] + for layer in ["L1", "L2", "L3", "L4", "L5"]: + item = by_layer.get(layer, {}) + cells.append(f"{esc(str(item.get('outcome', 'missing')))}") + return f"{esc(case['case']['title'])}{''.join(cells)}" __all__ = [ diff --git a/src/invart/evaluation/pre_1_0.py b/src/invart/evaluation/pre_1_0.py index 3f603be..eb32fe9 100644 --- a/src/invart/evaluation/pre_1_0.py +++ b/src/invart/evaluation/pre_1_0.py @@ -29,6 +29,14 @@ def run_pre_1_0_final_demo(out_dir: Path, *, external_evidence_manifest: Path | pre_v1_demo = run_pre_v1_control_plane_demo(out_dir / "pre-v1-demo") external = _external_status(external_evidence_manifest) entrypoint = out_dir / "pre-1.0-final-demo.html" + runtime_effect_matrix = _runtime_effect_matrix( + entrypoint=entrypoint, + vendor_matrix=matrix_path, + unmanaged_inventory=unmanaged_path, + risk_demo=risk_demo, + pre_v1_demo=pre_v1_demo, + external=external, + ) report = { "schema_version": SCHEMA_VERSION, "status": "pass", @@ -40,6 +48,7 @@ def run_pre_1_0_final_demo(out_dir: Path, *, external_evidence_manifest: Path | "pre_v1_demo": pre_v1_demo, "external_evidence": external, }, + "runtime_effect_matrix": runtime_effect_matrix, "summary": { "vendor_agents": matrix["summary"]["agents"], "unmanaged_findings": unmanaged["summary"]["findings"], @@ -51,6 +60,46 @@ def run_pre_1_0_final_demo(out_dir: Path, *, external_evidence_manifest: Path | return report +def _runtime_effect_matrix( + *, + entrypoint: Path, + vendor_matrix: Path, + unmanaged_inventory: Path, + risk_demo: dict[str, Any], + pre_v1_demo: dict[str, Any], + external: dict[str, Any], +) -> list[dict[str, Any]]: + pre_v1_artifacts = pre_v1_demo["artifacts"] + risk_entry = Path(risk_demo["artifacts"]["html"]) + rows = [ + ("before-runtime", "L1", "Execution Surface", "Agent config, skills, MCP, wrappers, launchers, and unmanaged surfaces are discovered.", vendor_matrix), + ("before-runtime", "L2", "Runtime Fact Model", "Session identity, principal, credential boundary, and grants are prepared as ledger facts.", pre_v1_artifacts["ledger"]), + ("before-runtime", "L3", "Decision Plane", "Policy profile and deterministic critical boundaries are visible before action.", pre_v1_artifacts["path_policy"]), + ("before-runtime", "L4", "Mediation Plane", "Managed launch path distinguishes covered and weakly covered surfaces before execution.", unmanaged_inventory), + ("before-runtime", "L5", "Evidence Plane", "The demo entrypoint records the claim boundary and artifact map.", entrypoint), + ("during-runtime", "L1", "Execution Surface", "Agent-like file, network, shell, content, and skill/tool actions enter observed surfaces.", risk_entry), + ("during-runtime", "L2", "Runtime Fact Model", "Invocations record resource, taint, identity, coverage, and outcome facts.", pre_v1_artifacts["proof"]), + ("during-runtime", "L3", "Decision Plane", "Path-aware policy reconstructs risky chains such as secret read to network sink.", pre_v1_artifacts["path_policy"]), + ("during-runtime", "L4", "Mediation Plane", "Actions are allowed, audited, paused, denied, or enforced without conflating coverage levels.", pre_v1_artifacts["replay"]), + ("during-runtime", "L5", "Evidence Plane", "Coverage and replay show observed / mediated / enforced as separate claims.", pre_v1_artifacts["coverage_report"]), + ("after-runtime", "L1", "Execution Surface", "Surface inventory remains available for coverage gap review.", vendor_matrix), + ("after-runtime", "L2", "Runtime Fact Model", "The ledger remains the fact source for all reconstructed actions.", pre_v1_artifacts["ledger"]), + ("after-runtime", "L3", "Decision Plane", "Path graph explains why a decision was allowed, paused, or blocked.", pre_v1_artifacts["path_graph"]), + ("after-runtime", "L4", "Mediation Plane", "Gate output shows approval and enforcement gaps without overstating control.", pre_v1_artifacts["proof"]), + ("after-runtime", "L5", "Evidence Plane", f"Audit and optional external evidence status are reviewable; external status is {external.get('status')}.", pre_v1_artifacts["audit_report"]), + ] + return [ + { + "stage": stage, + "layer": layer, + "layer_name": layer_name, + "effect": effect, + "artifact": str(artifact), + } + for stage, layer, layer_name, effect, artifact in rows + ] + + def _write_demo_surfaces(root: Path) -> None: (root / ".claude").mkdir(parents=True, exist_ok=True) (root / ".claude" / "settings.json").write_text(json.dumps({"hooks": {"PreToolUse": []}, "mcpServers": {"fs": {}}}), encoding="utf-8") @@ -89,6 +138,7 @@ def _render_final_demo_html(report: dict[str, Any]) -> str: pre_v1_replay = relative_href(base, Path(pre_v1_artifacts["replay"])) pre_v1_graph = relative_href(base, Path(pre_v1_artifacts["path_graph"])) actions = _action_rows(pre_v1_artifacts) + effect_matrix = _effect_matrix_rows(report["runtime_effect_matrix"], base) external = report["artifacts"]["external_evidence"] return f""" @@ -113,6 +163,7 @@ def _render_final_demo_html(report: dict[str, Any]) -> str:
  • real-world risk demo
  • audit report · replay · path graph
    • +

    Runtime Effect Matrix

    Before runtime, during runtime, and after runtime are shown against the five control layers. Coverage labels are truthful: observed / mediated / enforced are separate claims.

    {effect_matrix}
    LayerBefore runtimeDuring runtimeAfter runtime

    Invart actions

    {actions}
    ActionEvidenceMeaning

    External validation

    Status: {html.escape(str(external.get("status")))}

    {html.escape(str(external.get("claim_boundary") or external.get("evidence_level") or ""))}

    Claim boundary

    {html.escape(report["claim_boundary"])}

    @@ -121,6 +172,29 @@ def _render_final_demo_html(report: dict[str, Any]) -> str: """ +def _effect_matrix_rows(matrix: list[dict[str, Any]], base: Path) -> str: + by_layer: dict[str, dict[str, dict[str, Any]]] = {} + for item in matrix: + by_layer.setdefault(item["layer"], {})[item["stage"]] = item + labels = { + "L1": "L1 Execution Surface", + "L2": "L2 Runtime Fact Model", + "L3": "L3 Decision Plane", + "L4": "L4 Mediation Plane", + "L5": "L5 Evidence Plane", + } + stages = ["before-runtime", "during-runtime", "after-runtime"] + rows = [] + for layer in ["L1", "L2", "L3", "L4", "L5"]: + cells = [] + for stage in stages: + item = by_layer[layer][stage] + href = relative_href(base, Path(item["artifact"])) + cells.append(f"{html.escape(item['effect'])}
    {html.escape(Path(item['artifact']).name)}") + rows.append(f"{labels[layer]}{''.join(cells)}") + return "".join(rows) + + def _action_rows(pre_v1_artifacts: dict[str, Any]) -> str: rows = [ ("identity binding", "proof.json", "principal, agent, credential boundary, and grants are visible"), diff --git a/src/invart/evaluation/release_candidate.py b/src/invart/evaluation/release_candidate.py index e50362a..0d65106 100644 --- a/src/invart/evaluation/release_candidate.py +++ b/src/invart/evaluation/release_candidate.py @@ -58,6 +58,7 @@ "docs/cli-reference.md", "docs/api-sdk.md", "docs/examples.md", + "docs/runtime-effect-demo.md", "docs/architecture.md", "docs/evaluation.md", "docs/open-source-boundary.md", @@ -69,6 +70,7 @@ "docs/html/cli-reference.html", "docs/html/api-sdk.html", "docs/html/examples.html", + "docs/html/runtime-effect-demo.html", "docs/html/architecture.html", "docs/html/evaluation.html", "docs/html/open-source-boundary.html", diff --git a/tests/test_demos_roadmap.py b/tests/test_demos_roadmap.py index 7e18e6e..ac1c479 100644 --- a/tests/test_demos_roadmap.py +++ b/tests/test_demos_roadmap.py @@ -204,15 +204,29 @@ def test_containerized_risk_demo_generates_per_case_artifact_bundles(tmp_path: P assert secret_case["summary"]["blocked_or_paused"] is True for key in ["ledger", "proof", "replay", "path_graph", "coverage_report", "audit_report", "case_json"]: assert Path(secret_case["artifacts"][key]).exists() + assert {item["layer"] for item in secret_case["layer_timeline"]} == {"L1", "L2", "L3", "L4", "L5"} + assert {item["stage"] for item in secret_case["layer_timeline"]} == {"before-runtime", "during-runtime", "after-runtime"} + assert all(item["artifact"] for item in secret_case["layer_timeline"]) + case_html = Path(secret_case["artifacts"]["audit_report"]).read_text(encoding="utf-8") + assert "Action Timeline" in case_html + assert "Agent intent/action" in case_html + assert "Invart observation" in case_html + assert "Policy / mediation decision" in case_html + assert "ledger.jsonl" in case_html + assert "proof.json" in case_html suite = run_container_risk_suite(tmp_path / "suite") assert suite["status"] == "pass" assert suite["summary"]["cases"] == 3 assert suite["summary"]["blocked_or_paused_cases"] == 3 assert suite["summary"]["source_mapped_cases"] == 3 + assert suite["summary"]["runtime_effect_cases"] == 3 html = Path(suite["artifacts"]["suite_html"]).read_text(encoding="utf-8") assert "Invart Containerized Risk Demo" in html assert "isolated container" in html + assert "Layer Effect Summary" in html + assert "L1 Execution Surface" in html + assert "L5 Evidence Plane" in html benchmark = run_benchmark("containerized-risk-demo") assert benchmark["passed"] is True @@ -239,9 +253,18 @@ def test_v045_pre_1_0_final_demo_links_control_plane_evidence(tmp_path: Path) -> assert Path(demo["artifacts"]["vendor_matrix"]).exists() assert Path(demo["artifacts"]["unmanaged_inventory"]).exists() assert Path(demo["artifacts"]["pre_v1_demo"]["artifacts"]["proof"]).exists() + assert {item["stage"] for item in demo["runtime_effect_matrix"]} == {"before-runtime", "during-runtime", "after-runtime"} + assert {item["layer"] for item in demo["runtime_effect_matrix"]} == {"L1", "L2", "L3", "L4", "L5"} + assert all(item["artifact"] for item in demo["runtime_effect_matrix"]) html = Path(demo["artifacts"]["entrypoint"]).read_text(encoding="utf-8") assert "Invart Pre-1.0 Final Demo" in html assert "Invart actions" in html + assert "Runtime Effect Matrix" in html + assert "Before runtime" in html + assert "During runtime" in html + assert "After runtime" in html + assert "L1 Execution Surface" in html + assert "observed / mediated / enforced are separate claims" in html assert "vendor-matrix.json" in html assert "external validation" in html.lower() assert main(["demo", "pre-1.0-final", "--out-dir", str(tmp_path / "cli-final-demo")]) == 0 diff --git a/tests/test_policy_evidence_rc.py b/tests/test_policy_evidence_rc.py index fdb0e10..6d19124 100644 --- a/tests/test_policy_evidence_rc.py +++ b/tests/test_policy_evidence_rc.py @@ -577,14 +577,22 @@ def test_public_docs_include_api_sdk_page_and_valid_local_links() -> None: html_dir = docs_dir / "html" api_doc = html_dir / "api-sdk.html" api_markdown = docs_dir / "api-sdk.md" + runtime_effect_doc = html_dir / "runtime-effect-demo.html" + runtime_effect_markdown = docs_dir / "runtime-effect-demo.md" assert api_doc.exists() assert api_markdown.exists() + assert runtime_effect_doc.exists() + assert runtime_effect_markdown.exists() assert 'href="api-sdk.html"' in (html_dir / "index.html").read_text(encoding="utf-8") + assert 'href="runtime-effect-demo.html"' in (html_dir / "index.html").read_text(encoding="utf-8") docs_readme = (docs_dir / "README.md").read_text(encoding="utf-8") assert "[`api-sdk.md`](api-sdk.md)" in docs_readme assert "[`html/api-sdk.html`](html/api-sdk.html)" in docs_readme + assert "[`runtime-effect-demo.md`](runtime-effect-demo.md)" in docs_readme + assert "[`html/runtime-effect-demo.html`](html/runtime-effect-demo.html)" in docs_readme assert "[API and SDK](docs/api-sdk.md)" in (root / "README.md").read_text(encoding="utf-8") + assert "[Runtime effect demo](docs/runtime-effect-demo.md)" in (root / "README.md").read_text(encoding="utf-8") assert "[HTML docs home](docs/html/index.html)" in (root / "README.md").read_text(encoding="utf-8") assert sorted(path.name for path in docs_dir.glob("*.html")) == [] @@ -624,6 +632,8 @@ def test_api_sdk_page_documents_real_python_helpers() -> None: assert "Not available in 0.9" in page assert "docs/api-sdk.md" in DEFAULT_REQUIRED_DOCS assert "docs/html/api-sdk.html" in DEFAULT_REQUIRED_DOCS + assert "docs/runtime-effect-demo.md" in DEFAULT_REQUIRED_DOCS + assert "docs/html/runtime-effect-demo.html" in DEFAULT_REQUIRED_DOCS assert callable(documented_load_ledger_entries) assert callable(documented_verify_ledger)