diff --git a/docs/cli-reference.md b/docs/cli-reference.md index d826c81..4a74ef2 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -80,6 +80,7 @@ invart real-agent run --agent claude-code --require-live --out-dir .invart/live- invart real-agent run --agent opencode --require-live --out-dir .invart/live-opencode -- invart real-agent run --agent gemini-cli --require-live --out-dir .invart/live-gemini -- invart real-agent run --agent aider --require-live --out-dir .invart/live-aider -- +invart real-agent run --agent codex --require-live --out-dir .invart/live-codex -- invart real-agent report --run-dir .invart/real-agent --out .invart/real-agent/report.html ``` @@ -117,5 +118,6 @@ invart eval benchmark --suite v0.9.8-claude-full-live-adapter invart eval benchmark --suite v0.9.9-conformance-contract-v2 invart eval benchmark --suite v0.9.10-opencode-real-adapter invart eval benchmark --suite v0.9.11-terminal-agent-managed-wrappers +invart eval benchmark --suite v0.9.12-codex-boundary invart roadmap status --require-full ``` diff --git a/docs/html/cli-reference.html b/docs/html/cli-reference.html index 2a0f3d5..9d6960e 100644 --- a/docs/html/cli-reference.html +++ b/docs/html/cli-reference.html @@ -44,6 +44,7 @@

User intent to command

invart real-agent run --agent opencode --require-live --out-dir .invart/live-opencode -- <opencode-or-fixture-command> invart real-agent run --agent gemini-cli --require-live --out-dir .invart/live-gemini -- <gemini-or-fixture-command> invart real-agent run --agent aider --require-live --out-dir .invart/live-aider -- <aider-or-fixture-command> +invart real-agent run --agent codex --require-live --out-dir .invart/live-codex -- <codex-or-fixture-command> invart real-agent report --run-dir .invart/real-agent --out .invart/real-agent/report.html

Use --require-live when missing local binaries should fail instead of being recorded as blocked evidence. adapter profiles lists priority tracks and keeps vendor/cloud import separate from Invart-mediated control. real-agent check emits a conformance contract row so imported, discovered, fixture-backed, and live evidence cannot be mixed into stronger claims.

Claude Code reference adapter

invart adapter claude-code \
   --target . \
@@ -64,6 +65,7 @@ 
User intent to command

 invart eval benchmark --suite v0.9.9-conformance-contract-v2
 invart eval benchmark --suite v0.9.10-opencode-real-adapter
 invart eval benchmark --suite v0.9.11-terminal-agent-managed-wrappers
+invart eval benchmark --suite v0.9.12-codex-boundary
 invart roadmap status --require-full
diff --git a/docs/html/release-history.html b/docs/html/release-history.html index 958c7eb..13a1043 100644 --- a/docs/html/release-history.html +++ b/docs/html/release-history.html @@ -42,6 +42,7 @@

0.9 Patch Track

v0.9.9ImplementedLive adapter conformance contract v2: each real-agent row now records evidence level, control position, side-effect timing, required artifacts, claimable coverage, and a claim gate that rejects vendor/import/discovery evidence inflated into Invart-mediated or enforced coverage. v0.9.10ImplementedOpenCode real adapter track: binary-backed managed wrapper run, plugin/MCP config inventory, L5 artifact export, benign autonomy preservation, and managed risk blocking before side effects without treating plugin-only inventory as mediation. v0.9.11ImplementedTerminal-agent managed wrappers for Gemini CLI and Aider: binary-backed wrapper runs, Gemini MCP/config inventory, Aider config/repo-context inventory, artifact parity, and low approval-noise checks for benign workflows. + v0.9.12ImplementedCodex deep adapter boundary: Codex managed-wrapper runs remain Invart-mediated, while Codex-native sandbox, approval, network, and credential-boundary facts are imported as vendor-owned evidence and rejected if inflated into Invart enforcement.
diff --git a/docs/release-history.md b/docs/release-history.md index d1aa345..a1318df 100644 --- a/docs/release-history.md +++ b/docs/release-history.md @@ -41,6 +41,7 @@ This page is a reference, not the first-run learning path. Start with [Product O | v0.9.9 | Implemented | Live adapter conformance contract v2: each real-agent row now records evidence level, control position, side-effect timing, required artifacts, claimable coverage, and a claim gate that rejects vendor/import/discovery evidence inflated into Invart-mediated or enforced coverage. | | v0.9.10 | Implemented | OpenCode real adapter track: binary-backed managed wrapper run, plugin/MCP config inventory, L5 artifact export, benign autonomy preservation, and managed risk blocking before side effects without treating plugin-only inventory as mediation. | | v0.9.11 | Implemented | Terminal-agent managed wrappers for Gemini CLI and Aider: binary-backed wrapper runs, Gemini MCP/config inventory, Aider config/repo-context inventory, artifact parity, and low approval-noise checks for benign workflows. | +| v0.9.12 | Implemented | Codex deep adapter boundary: Codex managed-wrapper runs remain Invart-mediated, while Codex-native sandbox, approval, network, and credential-boundary facts are imported as vendor-owned evidence and rejected if inflated into Invart enforcement. | ## Internal History diff --git a/src/invart/benchmarks/registry.py b/src/invart/benchmarks/registry.py index 55b2494..e0d9be3 100644 --- a/src/invart/benchmarks/registry.py +++ b/src/invart/benchmarks/registry.py @@ -59,6 +59,7 @@ run_agent_adapter_contract_benchmark, run_claude_full_live_adapter_benchmark, run_claude_reference_adapter_benchmark, + run_codex_boundary_benchmark, run_conformance_contract_v2_benchmark, run_evidence_workspace_gate_benchmark, run_layer_runtime_workflow_benchmark, @@ -129,6 +130,7 @@ def benchmark_runner_registry() -> dict[str, BenchmarkRunner]: "v0.9.9-conformance-contract-v2": run_conformance_contract_v2_benchmark, "v0.9.10-opencode-real-adapter": run_opencode_real_adapter_benchmark, "v0.9.11-terminal-agent-managed-wrappers": run_terminal_agent_managed_wrappers_benchmark, + "v0.9.12-codex-boundary": run_codex_boundary_benchmark, "progressive-external-validation": run_progressive_external_validation_benchmark, "real-world-agent-risk-demo": run_real_world_risk_benchmark, "containerized-risk-demo": run_container_risk_demo_benchmark, diff --git a/src/invart/benchmarks/releases_v52_v57.py b/src/invart/benchmarks/releases_v52_v57.py index 6f9bf05..905ee22 100644 --- a/src/invart/benchmarks/releases_v52_v57.py +++ b/src/invart/benchmarks/releases_v52_v57.py @@ -18,6 +18,7 @@ from invart.surfaces.claude_adapter import run_claude_code_adapter from invart.surfaces.live_adapter import run_live_agent_adapter from invart.surfaces.adapter_profiles import adapter_track_matrix, list_adapter_profiles, validate_adapter_profile_truthfulness +from invart.surfaces.vendor_evidence import import_vendor_native_evidence, validate_vendor_claim_boundary def run_agent_adapter_contract_benchmark() -> dict[str, object]: @@ -466,11 +467,60 @@ def run_terminal_agent_managed_wrappers_benchmark() -> dict[str, object]: ) +def run_codex_boundary_benchmark() -> dict[str, object]: + with tempfile.TemporaryDirectory(prefix="invart_v0912_") as tmp: + root = Path(tmp) + source = root / "codex-native.json" + source.write_text('{"sandbox":"workspace-write","approval":"on-request","network_policy":"restricted","credential_boundary":"redacted-env"}\n', encoding="utf-8") + vendor = import_vendor_native_evidence(agent="codex", source_path=source, out_dir=root / "vendor") + inflated = {**vendor, "coverage": {**vendor["coverage"], "invart_enforced": True}} + inflated_check = validate_vendor_claim_boundary(inflated) + fake = root / "fake-codex" + marker = root / "codex-marker.txt" + fake.write_text( + "#!/usr/bin/env python3\n" + "import pathlib, sys\n" + "if '--version' in sys.argv:\n" + " raise SystemExit(0)\n" + "if '--write-marker' in sys.argv:\n" + " pathlib.Path(sys.argv[sys.argv.index('--write-marker') + 1]).write_text('ran')\n", + encoding="utf-8", + ) + fake.chmod(0o755) + run = run_live_agent_adapter( + agent="codex", + target=root, + out_dir=root / "codex", + command=[str(fake), "--write-marker", str(marker)], + binary=str(fake), + require_live=True, + policy_mode="advisory", + ) + entries, _warnings = load_ledger_entries(Path(run["managed_run"]["ledger"])) + checks = { + "vendor_native_imported": vendor.get("status") == "pass" and vendor.get("coverage", {}).get("control_position") == "vendor_owned_import", + "vendor_not_invart_enforced": vendor.get("coverage", {}).get("invart_enforced") is False, + "inflated_vendor_claim_fails": inflated_check.get("status") == "fail", + "wrapper_run_is_invart_mediated": any(entry.entry_type == "action" and entry.decision for entry in entries), + "wrapper_run_keeps_autonomy": run.get("status") == "passed" and marker.exists(), + "boundary_text_present": "must not be counted" in vendor.get("claim_boundary", ""), + } + return _suite_result( + "v0.9.12-codex-boundary", + checks, + artifacts={ + "vendor_evidence": vendor.get("artifacts", {}).get("report_json"), + "managed_run": run.get("artifacts", {}).get("report_json"), + }, + ) + + __all__ = [ "run_agent_adapter_contract_benchmark", "run_claude_full_live_adapter_benchmark", "run_claude_reference_adapter_benchmark", "run_conformance_contract_v2_benchmark", + "run_codex_boundary_benchmark", "run_evidence_workspace_gate_benchmark", "run_opencode_real_adapter_benchmark", "run_terminal_agent_managed_wrappers_benchmark", diff --git a/src/invart/evaluation/benchmark_registry.py b/src/invart/evaluation/benchmark_registry.py index 38c315d..e4e000f 100644 --- a/src/invart/evaluation/benchmark_registry.py +++ b/src/invart/evaluation/benchmark_registry.py @@ -40,6 +40,7 @@ {"suite": "v0.9.9-conformance-contract-v2", "version": "v0.9.9", "category": "agent-adapter", "optional_heavy": False, "claim_scope": "adapter_conformance_claim_gate", "evidence_level": "binary_backed_fixture_plus_vendor_import_negative_control"}, {"suite": "v0.9.10-opencode-real-adapter", "version": "v0.9.10", "category": "agent-adapter", "optional_heavy": False, "claim_scope": "local_opencode_managed_wrapper", "evidence_level": "binary_backed_fixture_with_plugin_inventory"}, {"suite": "v0.9.11-terminal-agent-managed-wrappers", "version": "v0.9.11", "category": "compatibility", "optional_heavy": False, "claim_scope": "local_terminal_agent_wrappers", "evidence_level": "binary_backed_fixture_with_config_inventory"}, + {"suite": "v0.9.12-codex-boundary", "version": "v0.9.12", "category": "agent-adapter", "optional_heavy": False, "claim_scope": "codex_vendor_boundary", "evidence_level": "vendor_native_import_plus_managed_wrapper_fixture"}, {"suite": "progressive-external-validation", "version": "pre-release", "category": "external-validation", "optional_heavy": False, "claim_scope": "progressive_external_validation", "evidence_level": "external_progressive_sample"}, {"suite": "real-world-agent-risk-demo", "version": "pre-release", "category": "demo", "optional_heavy": False, "claim_scope": "public_source_mapping", "evidence_level": "public_source_seed_plus_local_demo"}, {"suite": "containerized-risk-demo", "version": "pre-release", "category": "demo", "optional_heavy": False, "claim_scope": "containerized_local_demo", "evidence_level": "per_case_container_artifact_bundle"}, diff --git a/src/invart/evaluation/release_candidate.py b/src/invart/evaluation/release_candidate.py index 6607d85..21617d6 100644 --- a/src/invart/evaluation/release_candidate.py +++ b/src/invart/evaluation/release_candidate.py @@ -57,6 +57,7 @@ "v0.9.9-conformance-contract-v2", "v0.9.10-opencode-real-adapter", "v0.9.11-terminal-agent-managed-wrappers", + "v0.9.12-codex-boundary", "progressive-external-validation", "pre-v1-control-plane", ) diff --git a/src/invart/evaluation/roadmap.py b/src/invart/evaluation/roadmap.py index dbc5047..1a0da9a 100644 --- a/src/invart/evaluation/roadmap.py +++ b/src/invart/evaluation/roadmap.py @@ -902,6 +902,21 @@ def internal_docs_for_capability(docs: list[str]) -> list[str]: external_validation="not_run_optional", next_step="Integrate Codex while keeping Codex-native sandbox and approval facts separate from Invart-owned enforcement.", ), + RoadmapCapability( + version="v0.9.12", + capability_id="codex_deep_adapter_boundary", + title="Codex deep adapter and native-control boundary", + target="Integrate Codex managed-wrapper evidence while importing Codex-native sandbox, approval, network, and credential-boundary facts as vendor-owned evidence rather than Invart enforcement.", + status="implemented", + implementation=["src/invart/surfaces/live_adapter.py", "src/invart/surfaces/vendor_evidence.py", "src/invart/benchmarks/releases_v52_v57.py"], + tests=["test_v0912_codex_vendor_native_facts_do_not_count_as_invart_enforcement", "test_v0912_codex_managed_wrapper_remains_invart_mediated", "v0.9.12-codex-boundary"], + docs=["docs/cli-reference.md", "docs/html/cli-reference.html", "docs/release-history.md", "docs/html/release-history.html", "docs/roadmap.md", "docs/html/roadmap.html"], + product_boundaries=["Local fixture-backed Codex validation checks the boundary contract; installed Codex strict-live validation remains optional.", "Codex-native sandbox, approval, network, and credential facts are vendor-owned evidence unless the action enters Invart mediation.", "Invart wrapper runs remain Invart-mediated and ledger-backed.", "Gate checks reject reports that upgrade vendor-native facts into Invart enforcement."], + claim_scope="codex_vendor_boundary", + evidence_level="vendor_native_import_plus_managed_wrapper_fixture", + external_validation="not_run_optional", + next_step="Add IDE extension bridge and inventory tracks for Cursor, Cline, and Roo without overclaiming discovery-only evidence.", + ), ) diff --git a/src/invart/surfaces/vendor_evidence.py b/src/invart/surfaces/vendor_evidence.py new file mode 100644 index 0000000..dfc548f --- /dev/null +++ b/src/invart/surfaces/vendor_evidence.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from invart.core.artifacts import sha256_file, write_json_artifact +from invart.core.models import utc_now + + +def import_vendor_native_evidence( + *, + agent: str, + source_path: Path, + out_dir: Path, + evidence_kind: str = "native_control", +) -> dict[str, Any]: + source_path = source_path.expanduser().resolve() + out_dir = out_dir.expanduser().resolve() + out_dir.mkdir(parents=True, exist_ok=True) + payload = _load_source(source_path) + controls = _controls_for_agent(agent, payload) + report = { + "schema_version": "invart.vendor_native_evidence.v0.9.12", + "status": "pass" if source_path.exists() else "fail", + "agent": agent, + "evidence_kind": evidence_kind, + "source": { + "path": str(source_path), + "exists": source_path.exists(), + "sha256": sha256_file(source_path) if source_path.exists() and source_path.is_file() else None, + "loaded_at": utc_now(), + }, + "controls": controls, + "coverage": { + "coverage_grade": "vendor_owned", + "control_position": "vendor_owned_import", + "invart_mediated": False, + "invart_enforced": False, + "side_effect_timing": "vendor_native_or_after_the_fact", + }, + "claim_boundary": "Vendor-native sandbox, approval, network, credential, or telemetry facts are imported audit evidence. They must not be counted as Invart-mediated or Invart-enforced coverage unless the action also enters an Invart mediation/enforcement surface.", + "artifacts": {"report_json": str(out_dir / "vendor-native-evidence.json")}, + } + write_json_artifact(out_dir / "vendor-native-evidence.json", report) + return report + + +def validate_vendor_claim_boundary(report: dict[str, Any]) -> dict[str, Any]: + findings = [] + coverage = report.get("coverage", {}) if isinstance(report.get("coverage"), dict) else {} + if coverage.get("control_position") == "vendor_owned_import" and coverage.get("invart_enforced") is True: + findings.append(_finding("vendor.enforcement_inflation", "Vendor-owned evidence cannot be labeled Invart-enforced.")) + if coverage.get("control_position") == "vendor_owned_import" and coverage.get("invart_mediated") is True: + findings.append(_finding("vendor.mediation_inflation", "Vendor-owned evidence cannot be labeled Invart-mediated.")) + return { + "schema_version": "invart.vendor_claim_boundary_check.v0.9.12", + "status": "pass" if not findings else "fail", + "findings": findings, + "summary": {"findings": len(findings)}, + } + + +def _controls_for_agent(agent: str, payload: dict[str, Any]) -> list[dict[str, Any]]: + if agent == "codex": + return [ + _control("sandbox", payload.get("sandbox", "unknown"), "vendor_native"), + _control("approval", payload.get("approval", "unknown"), "vendor_native"), + _control("network_policy", payload.get("network_policy", "unknown"), "vendor_native"), + _control("credential_boundary", payload.get("credential_boundary", "unknown"), "vendor_native"), + ] + return [_control(key, value, "vendor_native") for key, value in sorted(payload.items())] + + +def _control(name: str, value: Any, owner: str) -> dict[str, Any]: + return {"name": name, "value": value, "owner": owner, "invart_owned": False} + + +def _load_source(path: Path) -> dict[str, Any]: + if not path.exists() or not path.is_file(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {"raw_preview": path.read_text(encoding="utf-8")[:400]} + return payload if isinstance(payload, dict) else {"value": payload} + + +def _finding(check_id: str, message: str) -> dict[str, Any]: + return {"check_id": check_id, "severity": "high", "message": message} + + +__all__ = ["import_vendor_native_evidence", "validate_vendor_claim_boundary"] diff --git a/tests/test_integrations.py b/tests/test_integrations.py index 16e0ac4..4979bc6 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -1192,6 +1192,64 @@ def test_v0911_terminal_agent_cli_and_benchmark_are_registered(tmp_path: Path) - assert main(["eval", "benchmark", "--suite", "v0.9.11-terminal-agent-managed-wrappers"]) == 0 +def test_v0912_codex_vendor_native_facts_do_not_count_as_invart_enforcement(tmp_path: Path) -> None: + from invart.surfaces.vendor_evidence import import_vendor_native_evidence, validate_vendor_claim_boundary + + source = tmp_path / "codex-native.json" + source.write_text( + json.dumps({ + "sandbox": "workspace-write", + "approval": "on-request", + "network_policy": "restricted", + "credential_boundary": "redacted-env", + }), + encoding="utf-8", + ) + report = import_vendor_native_evidence(agent="codex", source_path=source, out_dir=tmp_path / "codex-vendor") + assert report["schema_version"] == "invart.vendor_native_evidence.v0.9.12" + assert report["status"] == "pass" + assert report["coverage"]["control_position"] == "vendor_owned_import" + assert report["coverage"]["invart_enforced"] is False + assert {control["name"] for control in report["controls"]} == {"sandbox", "approval", "network_policy", "credential_boundary"} + assert validate_vendor_claim_boundary(report)["status"] == "pass" + inflated = {**report, "coverage": {**report["coverage"], "invart_enforced": True}} + failed = validate_vendor_claim_boundary(inflated) + assert failed["status"] == "fail" + assert any(item["check_id"] == "vendor.enforcement_inflation" for item in failed["findings"]) + + +def test_v0912_codex_managed_wrapper_remains_invart_mediated(tmp_path: Path) -> None: + from invart.surfaces.live_adapter import run_live_agent_adapter + + fake = tmp_path / "fake-codex" + marker = tmp_path / "codex-ran.txt" + fake.write_text( + "#!/usr/bin/env python3\n" + "import pathlib, sys\n" + "if '--version' in sys.argv:\n" + " raise SystemExit(0)\n" + "if '--write-marker' in sys.argv:\n" + " pathlib.Path(sys.argv[sys.argv.index('--write-marker') + 1]).write_text('ran')\n", + encoding="utf-8", + ) + fake.chmod(0o755) + result = run_live_agent_adapter( + agent="codex", + target=tmp_path, + out_dir=tmp_path / "codex-live", + command=[str(fake), "--write-marker", str(marker)], + binary=str(fake), + require_live=True, + policy_mode="advisory", + ) + assert result["status"] == "passed" + assert marker.exists() + assert result["live_evidence"]["control_position"] == "managed_wrapper" + entries, _warnings = load_ledger_entries(Path(result["managed_run"]["ledger"])) + assert any(entry.entry_type == "action" and entry.decision and entry.decision.get("effect") == "allow" for entry in entries) + assert main(["eval", "benchmark", "--suite", "v0.9.12-codex-boundary"]) == 0 + + def test_v09_swe_bench_lite_runner_skips_cleanly_without_dependencies(tmp_path: Path) -> None: out = tmp_path / "swebench-report.json" assert main([