diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 3c2da06..e9c8e95 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -40,10 +40,21 @@ invart replay export --ledger ledger.jsonl --out replay.html invart audit report --ledger ledger.jsonl --out-dir .invart/audit ``` +### Real agent conformance + +```bash +invart adapter profile --kind claude-code +invart real-agent check --agent claude-code --out-dir .invart/real-agent +invart real-agent report --run-dir .invart/real-agent --out .invart/real-agent/report.html +``` + +Use `--require-live` when you want missing local agent binaries to fail the run instead of being recorded as blocked evidence. Fixture-backed runs validate the Invart adapter contract; live runs validate the installed product surface. + ### Evaluation ```bash invart eval list invart eval benchmark --suite full-product-readiness +invart eval benchmark --suite v0.9.3-agent-adapter-contract invart roadmap status --require-full ``` diff --git a/docs/html/cli-reference.html b/docs/html/cli-reference.html index a36c397..95c8693 100644 --- a/docs/html/cli-reference.html +++ b/docs/html/cli-reference.html @@ -15,8 +15,12 @@ invart gate verify --proof proof.json --ledger ledger.jsonl --mode ci

Replay and audit

invart replay export --ledger ledger.jsonl --out replay.html
 invart audit report --ledger ledger.jsonl --out-dir .invart/audit
+

Real agent conformance

invart adapter profile --kind claude-code
+invart real-agent check --agent claude-code --out-dir .invart/real-agent
+invart real-agent report --run-dir .invart/real-agent --out .invart/real-agent/report.html

Use --require-live when missing local binaries should fail instead of being recorded as blocked evidence.

Evaluation

invart eval list
 invart eval benchmark --suite full-product-readiness
+invart eval benchmark --suite v0.9.3-agent-adapter-contract
 invart roadmap status --require-full
diff --git a/docs/html/release-history.html b/docs/html/release-history.html index 326db06..53b41ae 100644 --- a/docs/html/release-history.html +++ b/docs/html/release-history.html @@ -26,6 +26,13 @@

Pre-1.0 Research-Ready Track

v0.51ImplementedSeparate research-ready gate layered on top of product RC readiness. +
+

0.9 Patch Track

+ + + +
VersionStatusFocus
v0.9.3ImplementedAgent adapter contract registry and fixture-backed real-agent conformance foundation. The live mode can be strict, but missing local agent binaries are not reported as successful live validation.
+

Internal History

Detailed historical roadmap and design pages live in internal/history/docs/. They are local-only planning material and are ignored by git for the open-source boundary.

diff --git a/docs/release-history.md b/docs/release-history.md index 5ad6a46..baeb0fb 100644 --- a/docs/release-history.md +++ b/docs/release-history.md @@ -26,6 +26,12 @@ Invart has a long local implementation history. The public docs summarize capabi | v0.50 | Implemented | Product control matrix showing why plugin-only coverage is not full runtime mediation. | | v0.51 | Implemented | Separate research-ready gate layered on top of product RC readiness. | +## 0.9 Patch Track + +| Version | Status | Focus | +| --- | --- | --- | +| v0.9.3 | Implemented | Agent adapter contract registry and fixture-backed real-agent conformance foundation. The live mode can be strict, but missing local agent binaries are not reported as successful live validation. | + ## Internal History Detailed historical roadmap and design pages live in internal/history/docs/. They are local-only planning material and are ignored by git for the open-source boundary. diff --git a/src/invart/benchmarks/registry.py b/src/invart/benchmarks/registry.py index 83a7e14..7038a4c 100644 --- a/src/invart/benchmarks/registry.py +++ b/src/invart/benchmarks/registry.py @@ -55,6 +55,7 @@ run_product_control_matrix_benchmark, run_reviewer_ablation_cost_benchmark, ) +from .releases_v52_v57 import run_agent_adapter_contract_benchmark BenchmarkRunner = Callable[[], dict[str, Any]] @@ -109,6 +110,7 @@ def benchmark_runner_registry() -> dict[str, BenchmarkRunner]: "v0.49-reviewer-ablation-cost": run_reviewer_ablation_cost_benchmark, "v0.50-product-control-matrix": run_product_control_matrix_benchmark, "v0.51-pre-1.0-research-ready-gate": run_pre_1_0_research_ready_gate_benchmark, + "v0.9.3-agent-adapter-contract": run_agent_adapter_contract_benchmark, "progressive-external-validation": run_progressive_external_validation_benchmark, "real-world-agent-risk-demo": run_real_world_risk_benchmark, "containerized-risk-demo": run_container_risk_demo_benchmark, diff --git a/src/invart/benchmarks/releases_v52_v57.py b/src/invart/benchmarks/releases_v52_v57.py new file mode 100644 index 0000000..7790fcd --- /dev/null +++ b/src/invart/benchmarks/releases_v52_v57.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import tempfile +from pathlib import Path + +from .common import _suite_result +from invart.evaluation.real_agent_conformance import run_real_agent_conformance +from invart.surfaces.adapter_profiles import list_adapter_profiles, validate_adapter_profile_truthfulness + + +def run_agent_adapter_contract_benchmark() -> dict[str, object]: + with tempfile.TemporaryDirectory(prefix="invart_v093_") as tmp: + root = Path(tmp) + fake = root / "fake-agent" + fake.write_text("#!/usr/bin/env python3\nimport sys\nsys.exit(0)\n", encoding="utf-8") + fake.chmod(0o755) + profiles = list_adapter_profiles() + validation = validate_adapter_profile_truthfulness(profiles) + conformance = run_real_agent_conformance( + out_dir=root / "conformance", + agents=["claude-code", "codex"], + binary_overrides={"claude-code": str(fake), "codex": str(fake)}, + require_live=True, + ) + by_agent = {profile["agent_id"]: profile for profile in profiles} + checks = { + "profiles_truthful": validation.get("status") == "pass", + "priority_agents_registered": {"claude-code", "codex", "hermes", "openclaw"}.issubset(by_agent), + "cloud_import_not_mediated": by_agent.get("github-copilot-cloud-agent", {}).get("supports_mediation") is False, + "claude_full_requires_evidence": {"ledger", "proof", "evidence_bundle"}.issubset(set(by_agent.get("claude-code", {}).get("required_artifacts", []))), + "fixture_conformance_passed": conformance.get("status") == "pass", + "strict_mode_records_managed_run": all(agent.get("managed_run", {}).get("status") == "pass" for agent in conformance.get("agents", [])), + } + return _suite_result( + "v0.9.3-agent-adapter-contract", + checks, + artifacts=conformance.get("artifacts", {}), + ) + + +__all__ = ["run_agent_adapter_contract_benchmark"] diff --git a/src/invart/commands/integrations.py b/src/invart/commands/integrations.py index 96b6841..f77ceb5 100644 --- a/src/invart/commands/integrations.py +++ b/src/invart/commands/integrations.py @@ -18,6 +18,7 @@ run_official_swe_bench_lite_check, run_swe_bench_lite_check, ) +from invart.evaluation.real_agent_conformance import export_real_agent_report_html, run_real_agent_conformance from invart.surfaces.adapter_profiles import build_adapter_profile from invart.surfaces.enforcement import check_enforcement, run_enforced_command, run_file_write_intercepted, rust_build_check, rust_shim_decision, rust_shim_spec from invart.surfaces.native import install_native_integration, inventory_native_integrations, native_capability_matrix, native_conformance_report, unmanaged_agent_inventory @@ -175,6 +176,40 @@ def handle_external_validation(args: argparse.Namespace) -> int: return 0 if result.get("status") == "pass" else 1 return 2 + +def handle_real_agent(args: argparse.Namespace) -> int: + if args.real_agent_command == "check": + try: + binary_overrides = _parse_agent_binary_overrides(args.binary) + result = run_real_agent_conformance( + out_dir=Path(args.out_dir), + agents=args.agent or None, + binary_overrides=binary_overrides, + require_live=args.require_live, + ) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 + print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if result.get("status") == "pass" else 1 + if args.real_agent_command == "report": + result = export_real_agent_report_html(Path(args.run_dir), Path(args.out)) + print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if result.get("status") == "pass" else 1 + return 2 + + +def _parse_agent_binary_overrides(values: list[str]) -> dict[str, str]: + overrides: dict[str, str] = {} + for value in values: + if "=" not in value: + raise ValueError("--binary must use agent-id=/path/to/binary") + agent, path = value.split("=", 1) + if not agent or not path: + raise ValueError("--binary must use agent-id=/path/to/binary") + overrides[agent] = path + return overrides + def handle_enforce(args: argparse.Namespace) -> int: if args.enforce_command == "check": profile_config = resolved_profile_from_args(args) diff --git a/src/invart/commands/parser_integrations.py b/src/invart/commands/parser_integrations.py index 3b3ba1c..3c6809a 100644 --- a/src/invart/commands/parser_integrations.py +++ b/src/invart/commands/parser_integrations.py @@ -3,8 +3,9 @@ import argparse from invart.evaluation.harness import SWE_BENCH_FULL_DATASET, SWE_BENCH_FULL_EXPECTED_INSTANCES, SWE_BENCH_LITE_DATASET +from invart.surfaces.adapter_profiles import adapter_profile_ids from .common import add_profile_args -from .integrations import handle_adapter, handle_bridge, handle_coverage, handle_enforce, handle_external_validation, handle_graph, handle_harness, handle_launcher, handle_mcp, handle_mediation, handle_native, handle_supervise +from .integrations import handle_adapter, handle_bridge, handle_coverage, handle_enforce, handle_external_validation, handle_graph, handle_harness, handle_launcher, handle_mcp, handle_mediation, handle_native, handle_real_agent, handle_supervise def register_integration_commands(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> None: @@ -30,7 +31,7 @@ def register_integration_commands(subparsers: argparse._SubParsersAction[argpars adapter_package.add_argument("--ledger", required=True) adapter_package.add_argument("--out-dir", required=True) adapter_profile = adapter_sub.add_parser("profile", help="Inspect a hardened adapter profile.") - adapter_profile.add_argument("--kind", choices=("claude-code", "codex", "generic"), default="claude-code") + adapter_profile.add_argument("--kind", choices=adapter_profile_ids(), default="claude-code") adapter_claude_check = adapter_sub.add_parser("claude-code-check", help="Check a real Claude Code binary environment when available.") adapter_claude_check.add_argument("--binary", default="claude") adapter_claude = adapter_sub.add_parser("claude-code", help="Run a command through the Claude Code wrapper/hook bridge.") @@ -102,6 +103,18 @@ def register_integration_commands(subparsers: argparse._SubParsersAction[argpars swe_full.add_argument("--expected-total-instances", type=int, default=SWE_BENCH_FULL_EXPECTED_INSTANCES) swe_full.add_argument("--allow-subset", action="store_true", help="Mark this as a subset smoke run; it will not satisfy full external validation.") + real_agent = subparsers.add_parser("real-agent", help="Validate real or fixture-backed agent products through Invart profiles.") + real_agent.set_defaults(handler=handle_real_agent) + real_agent_sub = real_agent.add_subparsers(dest="real_agent_command", required=True) + real_agent_check = real_agent_sub.add_parser("check", help="Run agent profile and binary-shaped conformance checks.") + real_agent_check.add_argument("--agent", action="append", choices=adapter_profile_ids(), default=[]) + real_agent_check.add_argument("--binary", action="append", default=[], help="Override an agent binary as agent-id=/path/to/binary.") + real_agent_check.add_argument("--require-live", action="store_true", help="Fail when requested real agent binaries are missing.") + real_agent_check.add_argument("--out-dir", default=".invart/real-agent-conformance") + real_agent_report = real_agent_sub.add_parser("report", help="Render a real-agent conformance HTML report from a run directory.") + real_agent_report.add_argument("--run-dir", required=True) + real_agent_report.add_argument("--out", required=True) + enforce = subparsers.add_parser("enforce", help="Evaluate enforcement guard decisions.") enforce.set_defaults(handler=handle_enforce) enforce_sub = enforce.add_subparsers(dest="enforce_command", required=True) diff --git a/src/invart/evaluation/benchmark_registry.py b/src/invart/evaluation/benchmark_registry.py index d35fbb9..a6afd94 100644 --- a/src/invart/evaluation/benchmark_registry.py +++ b/src/invart/evaluation/benchmark_registry.py @@ -31,6 +31,7 @@ {"suite": "v0.49-reviewer-ablation-cost", "version": "v0.49", "category": "reviewer", "optional_heavy": False, "claim_scope": "local_experiment_substrate", "evidence_level": "simulated_agent_trace"}, {"suite": "v0.50-product-control-matrix", "version": "v0.50", "category": "product-boundary", "optional_heavy": False, "claim_scope": "product_comparison", "evidence_level": "documented_capability_matrix"}, {"suite": "v0.51-pre-1.0-research-ready-gate", "version": "v0.51", "category": "release", "optional_heavy": False, "claim_scope": "research_gate", "evidence_level": "local_research_artifact_bundle"}, + {"suite": "v0.9.3-agent-adapter-contract", "version": "v0.9.3", "category": "agent-adapter", "optional_heavy": False, "claim_scope": "local_agent_adapter_contract", "evidence_level": "fixture_backed_conformance"}, {"suite": "progressive-external-validation", "version": "pre-release", "category": "external-validation", "optional_heavy": False, "claim_scope": "progressive_external_validation", "evidence_level": "external_progressive_sample"}, {"suite": "real-world-agent-risk-demo", "version": "pre-release", "category": "demo", "optional_heavy": False, "claim_scope": "public_source_mapping", "evidence_level": "public_source_seed_plus_local_demo"}, {"suite": "containerized-risk-demo", "version": "pre-release", "category": "demo", "optional_heavy": False, "claim_scope": "containerized_local_demo", "evidence_level": "per_case_container_artifact_bundle"}, diff --git a/src/invart/evaluation/real_agent_conformance.py b/src/invart/evaluation/real_agent_conformance.py new file mode 100644 index 0000000..00bfea8 --- /dev/null +++ b/src/invart/evaluation/real_agent_conformance.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import html +import json +import shutil +import subprocess +from pathlib import Path +from typing import Any + +from invart.core.artifacts import stable_json_hash, write_html_artifact, write_json_artifact +from invart.core.models import utc_now +from invart.surfaces.adapter import run_adapter_command +from invart.surfaces.adapter_profiles import get_adapter_profile, list_adapter_profiles, validate_adapter_profile_truthfulness + + +SCHEMA_VERSION = "invart.real_agent_conformance.v0.9.3" +DEFAULT_REQUIRED_AGENTS = ("claude-code", "codex", "hermes", "openclaw") + + +def run_real_agent_conformance( + *, + out_dir: Path, + agents: list[str] | None = None, + binary_overrides: dict[str, str] | None = None, + require_live: bool = False, + target: Path | None = None, +) -> dict[str, Any]: + root = out_dir.expanduser().resolve() + root.mkdir(parents=True, exist_ok=True) + target_root = (target or root / "workspace").expanduser().resolve() + target_root.mkdir(parents=True, exist_ok=True) + selected_agents = list(agents or DEFAULT_REQUIRED_AGENTS) + overrides = dict(binary_overrides or {}) + profile_validation = validate_adapter_profile_truthfulness(list_adapter_profiles()) + rows = [ + _run_agent_check( + agent=agent, + out_dir=root / _safe_id(agent), + target=target_root, + binary_override=overrides.get(agent), + ) + for agent in selected_agents + ] + failed_rows = [ + row + for row in rows + if row["status"] == "failed_run" or (require_live and row["status"] == "blocked_missing_binary") + ] + status = "fail" if failed_rows or profile_validation["status"] != "pass" else "pass" + report_json = root / "real-agent-conformance.json" + report_html = root / "real-agent-conformance.html" + report: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "status": status, + "generated_at": utc_now(), + "required_live": require_live, + "required_agents": selected_agents, + "profile_validation": profile_validation, + "summary": { + "agents": len(rows), + "passed_agents": sum(1 for row in rows if row["status"] == "pass"), + "blocked_missing_binary": sum(1 for row in rows if row["status"] == "blocked_missing_binary"), + "failed_agents": len(failed_rows), + "claim_boundary": "Fixture-backed checks validate Invart's conformance harness. Strict live mode fails when requested real binaries are unavailable or do not produce managed-run evidence.", + }, + "agents": rows, + "artifacts": {"report_json": str(report_json), "report_html": str(report_html)}, + "evidence_hash": stable_json_hash({"agents": rows, "required_live": require_live}), + } + write_json_artifact(report_json, report) + write_html_artifact(report_html, render_real_agent_conformance_html(report)) + return report + + +def export_real_agent_report_html(run_dir: Path, out: Path) -> dict[str, Any]: + report_path = run_dir.expanduser().resolve() / "real-agent-conformance.json" + report = _load_json(report_path) + write_html_artifact(out.expanduser().resolve(), render_real_agent_conformance_html(report)) + return { + "schema_version": "invart.real_agent_conformance_report.v0.9.3", + "status": "pass", + "source": str(report_path), + "out": str(out.expanduser().resolve()), + } + + +def render_real_agent_conformance_html(report: dict[str, Any]) -> str: + rows = [] + for agent in report.get("agents", []): + rows.append( + "" + f"{html.escape(str(agent.get('agent')))}" + f"{html.escape(str(agent.get('status')))}" + f"{html.escape(str(agent.get('coverage', {}).get('coverage_grade')))}" + f"{html.escape(str(agent.get('binary', {}).get('status')))}" + f"{html.escape(str(agent.get('managed_run', {}).get('status')))}" + f"{html.escape(str(agent.get('claim_boundary')))}" + "" + ) + return ( + "Real Agent Conformance" + "
" + f"

Real Agent Conformance

Status: {html.escape(str(report.get('status')))}

" + f"

{html.escape(str(report.get('summary', {}).get('claim_boundary', '')))}

" + "" + f"{''.join(rows)}
AgentStatusCoverage GradeBinaryManaged RunClaim Boundary
" + ) + + +def _run_agent_check(*, agent: str, out_dir: Path, target: Path, binary_override: str | None) -> dict[str, Any]: + profile = get_adapter_profile(agent) + out_dir.mkdir(parents=True, exist_ok=True) + binary = _resolve_binary(profile, binary_override) + row: dict[str, Any] = { + "agent": agent, + "display_name": profile["display_name"], + "status": "blocked_missing_binary", + "binary": binary, + "native_inventory": {"status": "not_run", "reason": "v0.9.3 conformance focuses on binary/profile/managed-run contract"}, + "managed_run": {"status": "not_run", "reason": "binary unavailable"}, + "risk_run": {"status": "not_run", "reason": "risk workflow belongs to later adapter hardening"}, + "coverage": { + "coverage_grade": profile["coverage_grade"], + "supports_mediation": profile["supports_mediation"], + "can_block": profile["can_block"], + "can_pause_resume": profile["can_pause_resume"], + }, + "evidence": {}, + "claim_boundary": profile["claim_boundary"], + "source_urls": profile["source_urls"], + } + if binary["status"] != "found": + return row + + run = run_adapter_command( + target=target, + command=[str(binary["path"]), "--version"], + agent=agent, + goal=f"{agent} conformance version probe", + session_id=f"invart_conformance_{_safe_id(agent)}", + out_dir=out_dir, + capabilities="off", + gate_mode="off", + create_preflight=False, + ) + row["managed_run"] = { + "status": "pass" if run.status == "passed" else "fail", + "returncode": run.returncode, + "ledger": run.ledger, + "proof": run.proof, + } + row["evidence"] = {"ledger": run.ledger, "proof": run.proof} + row["status"] = "pass" if run.status == "passed" else "failed_run" + return row + + +def _resolve_binary(profile: dict[str, Any], override: str | None) -> dict[str, Any]: + candidates = [override] if override else list(profile.get("binary_candidates", [])) + for candidate in candidates: + if not candidate: + continue + resolved = shutil.which(str(candidate)) or _existing_path(candidate) + if not resolved: + continue + version = _probe_version(Path(resolved)) + return { + "status": "found", + "path": resolved, + "version": version.get("stdout") or version.get("stderr") or "", + "returncode": version.get("returncode"), + } + return {"status": "missing", "path": None, "version": "", "returncode": None} + + +def _probe_version(path: Path) -> dict[str, Any]: + try: + completed = subprocess.run([str(path), "--version"], check=False, capture_output=True, text=True, timeout=15) + return { + "returncode": completed.returncode, + "stdout": completed.stdout.strip()[:400], + "stderr": completed.stderr.strip()[:400], + } + except Exception as exc: + return {"returncode": None, "stdout": "", "stderr": str(exc)[:400]} + + +def _existing_path(candidate: str) -> str | None: + path = Path(candidate).expanduser() + return str(path.resolve()) if path.exists() else None + + +def _load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _safe_id(value: str) -> str: + return "".join(char if char.isalnum() else "_" for char in value).strip("_") or "agent" + + +__all__ = [ + "DEFAULT_REQUIRED_AGENTS", + "SCHEMA_VERSION", + "export_real_agent_report_html", + "render_real_agent_conformance_html", + "run_real_agent_conformance", +] diff --git a/src/invart/evaluation/release_candidate.py b/src/invart/evaluation/release_candidate.py index 1acb043..ec02d57 100644 --- a/src/invart/evaluation/release_candidate.py +++ b/src/invart/evaluation/release_candidate.py @@ -47,6 +47,7 @@ "v0.49-reviewer-ablation-cost", "v0.50-product-control-matrix", "v0.51-pre-1.0-research-ready-gate", + "v0.9.3-agent-adapter-contract", "progressive-external-validation", "pre-v1-control-plane", ) diff --git a/src/invart/evaluation/roadmap.py b/src/invart/evaluation/roadmap.py index 402ff1b..78dc425 100644 --- a/src/invart/evaluation/roadmap.py +++ b/src/invart/evaluation/roadmap.py @@ -767,6 +767,21 @@ def internal_docs_for_capability(docs: list[str]) -> list[str]: external_validation="not_run_optional", next_step="Run progressive external validation samples, then attach full external benchmark manifests.", ), + RoadmapCapability( + version="v0.9.3", + capability_id="agent_adapter_contract_foundation", + title="Agent adapter contract and real-agent conformance foundation", + target="Define truthful adapter profiles and fixture-backed real-agent conformance reports for priority agent products.", + status="implemented", + implementation=["src/invart/surfaces/adapter_profiles.py", "src/invart/evaluation/real_agent_conformance.py", "src/invart/commands/parser_integrations.py", "src/invart/commands/integrations.py"], + tests=["test_v093_adapter_profile_registry_reports_truthful_agent_contracts", "test_v093_real_agent_conformance_fixture_and_strict_live_modes", "v0.9.3-agent-adapter-contract"], + docs=["docs/cli-reference.md", "docs/html/cli-reference.html", "docs/release-history.md", "docs/html/release-history.html"], + product_boundaries=["Fixture-backed conformance validates Invart's adapter contract and harness behavior; strict live mode fails when installed agent binaries or live evidence are missing.", "Coverage grades prevent vendor import, plugin-only, and discovery-only surfaces from being reported as full managed mediation."], + claim_scope="local_agent_adapter_contract", + evidence_level="fixture_backed_conformance", + external_validation="not_run_optional", + next_step="Harden Claude Code as the first full reference adapter and attach live conformance traces for each priority product.", + ), ) diff --git a/src/invart/surfaces/adapter_profiles.py b/src/invart/surfaces/adapter_profiles.py index 7af3623..bd8a9e0 100644 --- a/src/invart/surfaces/adapter_profiles.py +++ b/src/invart/surfaces/adapter_profiles.py @@ -1,16 +1,294 @@ from __future__ import annotations import os +from dataclasses import asdict, dataclass, field from typing import Any SECRET_KEY_MARKERS = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL", "AUTH", "SSH") +PROFILE_SCHEMA_VERSION = "invart.adapter_profile.v0.10" +PROFILE_REGISTRY_SCHEMA_VERSION = "invart.agent_adapter_profile_registry.v0.9.3" + + +@dataclass(frozen=True) +class AgentAdapterProfile: + agent_id: str + display_name: str + priority: str + execution_modes: list[str] + native_surfaces: list[str] + event_sources: list[str] + coverage_grade: str + claim_boundary: str + required_artifacts: list[str] + source_urls: list[str] + last_reviewed: str = "2026-06-10" + binary_candidates: list[str] = field(default_factory=list) + supports_mediation: bool = False + can_block: bool = False + can_pause_resume: bool = False + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +_PROFILES: tuple[AgentAdapterProfile, ...] = ( + AgentAdapterProfile( + agent_id="claude-code", + display_name="Claude Code", + priority="p0_reference_full_adapter", + execution_modes=["managed_runtime", "native_event_bridge", "managed_wrapper"], + native_surfaces=["hooks", "permissions", "settings", "mcp", "slash_commands"], + event_sources=["wrapper_command", "session_env", "tool_event_bridge", "hook_jsonl"], + coverage_grade="full_managed_adapter", + claim_boundary="Full pre-1.0 claim only applies when Claude Code is launched through Invart-managed wrapper or hook bridge; direct unmanaged Claude runs remain coverage gaps.", + required_artifacts=["ledger", "proof", "replay", "path_graph", "coverage", "audit", "evidence_bundle"], + source_urls=["https://code.claude.com/docs/en/hooks", "https://code.claude.com/docs/en/permissions"], + binary_candidates=["claude"], + supports_mediation=True, + can_block=True, + can_pause_resume=True, + ), + AgentAdapterProfile( + agent_id="codex", + display_name="OpenAI Codex", + priority="p0_local_wrapper", + execution_modes=["managed_wrapper", "vendor_evidence_import"], + native_surfaces=["sandbox", "approval", "network_policy", "telemetry"], + event_sources=["wrapper_command", "sandbox_policy", "approval_log"], + coverage_grade="managed_wrapper_adapter", + claim_boundary="Invart-mediated claims require routing Codex-like execution through an Invart wrapper; vendor sandbox and approval facts are complementary evidence, not Invart enforcement by themselves.", + required_artifacts=["ledger", "proof", "coverage", "audit", "evidence_bundle"], + source_urls=["https://developers.openai.com/codex/concepts/sandboxing", "https://developers.openai.com/codex/agent-approvals-security"], + binary_candidates=["codex"], + supports_mediation=True, + can_block=True, + can_pause_resume=True, + ), + AgentAdapterProfile( + agent_id="gemini-cli", + display_name="Gemini CLI", + priority="p1_local_wrapper", + execution_modes=["managed_wrapper", "mcp_inventory"], + native_surfaces=["mcp", "settings", "extensions"], + event_sources=["wrapper_command", "mcp_config"], + coverage_grade="managed_wrapper_adapter", + claim_boundary="Invart can mediate Gemini CLI when launched through a managed wrapper; MCP/config discovery alone remains observed or discovered coverage.", + required_artifacts=["ledger", "proof", "coverage", "audit"], + source_urls=["https://github.com/google-gemini/gemini-cli", "https://geminicli.com/docs/tools/mcp-server/"], + binary_candidates=["gemini"], + supports_mediation=True, + can_block=True, + can_pause_resume=True, + ), + AgentAdapterProfile( + agent_id="cursor", + display_name="Cursor", + priority="p1_ide_inventory", + execution_modes=["native_event_bridge", "vendor_evidence_import", "discovery_inventory"], + native_surfaces=["rules", "mcp", "settings", "ide_extension"], + event_sources=["rules_config", "mcp_config", "ide_export"], + coverage_grade="native_event_bridge", + claim_boundary="Cursor IDE and extension surfaces can improve visibility, but Invart does not claim full runtime mediation unless actions enter an Invart bridge or wrapper.", + required_artifacts=["native_inventory", "coverage", "audit"], + source_urls=["https://cursor.com/docs"], + binary_candidates=["cursor"], + supports_mediation=True, + can_block=False, + can_pause_resume=False, + ), + AgentAdapterProfile( + agent_id="opencode", + display_name="OpenCode", + priority="p1_local_wrapper", + execution_modes=["managed_wrapper", "native_event_bridge", "plugin_inventory"], + native_surfaces=["plugins", "agents", "mcp", "config"], + event_sources=["wrapper_command", "plugin_config", "native_hook_payload"], + coverage_grade="native_event_bridge", + claim_boundary="OpenCode plugin and agent surfaces are mediated only when their hook payloads enter Invart; plugin-only logs are not full runtime enforcement.", + required_artifacts=["ledger", "proof", "coverage", "audit"], + source_urls=["https://opencode.ai/docs/agents/", "https://opencode.ai/docs/plugins/"], + binary_candidates=["opencode"], + supports_mediation=True, + can_block=True, + can_pause_resume=True, + ), + AgentAdapterProfile( + agent_id="openclaw", + display_name="OpenClaw", + priority="p0_real_agent_validation", + execution_modes=["managed_wrapper", "vendor_evidence_import", "skill_inventory"], + native_surfaces=["permission_modes", "tools", "skills", "plugins"], + event_sources=["host_exec_policy", "skill_config", "tool_log"], + coverage_grade="vendor_evidence_import", + claim_boundary="OpenClaw permission modes are product-owned control facts until host-exec events are bound to Invart mediation and ledger entries.", + required_artifacts=["native_inventory", "coverage", "audit", "evidence_bundle"], + source_urls=["https://docs.openclaw.ai/tools", "https://docs.openclaw.ai/gateway/security"], + binary_candidates=["openclaw"], + supports_mediation=False, + can_block=False, + can_pause_resume=False, + ), + AgentAdapterProfile( + agent_id="hermes", + display_name="Hermes Agent", + priority="p0_real_agent_validation", + execution_modes=["vendor_evidence_import", "managed_launcher_candidate"], + native_surfaces=["container_backend", "terminal_safety", "credential_filter", "mcp"], + event_sources=["backend_log", "container_policy", "security_report"], + coverage_grade="vendor_evidence_import", + claim_boundary="Hermes security controls are valuable vendor/runtime facts; Invart mediation requires backend events or launches to be routed through Invart.", + required_artifacts=["native_inventory", "coverage", "audit", "evidence_bundle"], + source_urls=["https://hermes-agent.nousresearch.com/docs/user-guide/security/"], + binary_candidates=["hermes"], + supports_mediation=False, + can_block=False, + can_pause_resume=False, + ), + AgentAdapterProfile( + agent_id="cline", + display_name="Cline", + priority="p1_ide_inventory", + execution_modes=["native_event_bridge", "discovery_inventory"], + native_surfaces=["mcp_marketplace", "ide_extension", "settings"], + event_sources=["mcp_config", "extension_config", "task_log"], + coverage_grade="native_event_bridge", + claim_boundary="Cline MCP and IDE extension surfaces can be inventoried or bridged, but Invart must not call them enforced without a mediation response path.", + required_artifacts=["native_inventory", "coverage", "audit"], + source_urls=["https://docs.cline.bot/mcp/mcp-overview", "https://cline.bot/mcp-marketplace"], + binary_candidates=["cline"], + supports_mediation=True, + can_block=False, + can_pause_resume=False, + ), + AgentAdapterProfile( + agent_id="roo-code", + display_name="Roo Code", + priority="p1_ide_inventory", + execution_modes=["native_event_bridge", "discovery_inventory"], + native_surfaces=["mcp", "ide_extension", "settings"], + event_sources=["mcp_config", "extension_config"], + coverage_grade="native_event_bridge", + claim_boundary="Roo Code is treated as an IDE agent surface; Invart requires explicit bridge evidence before claiming mediation.", + required_artifacts=["native_inventory", "coverage", "audit"], + source_urls=["https://github.com/RooVetGit/Roo-Code"], + binary_candidates=["roo"], + supports_mediation=True, + can_block=False, + can_pause_resume=False, + ), + AgentAdapterProfile( + agent_id="github-copilot-cloud-agent", + display_name="GitHub Copilot coding agent", + priority="p1_cloud_import", + execution_modes=["vendor_evidence_import"], + native_surfaces=["cloud_agent", "firewall", "pull_request", "workflow_log"], + event_sources=["github_log", "pull_request_artifact", "firewall_policy"], + coverage_grade="vendor_evidence_import", + claim_boundary="Cloud agent evidence can be imported and audited, but Invart cannot claim local runtime mediation unless it controls the execution boundary.", + required_artifacts=["external_evidence_manifest", "coverage", "audit", "evidence_bundle"], + source_urls=[ + "https://docs.github.com/en/copilot/concepts/agents/cloud-agent/about-cloud-agent", + "https://docs.github.com/en/copilot/how-tos/use-copilot-agents/coding-agent/customize-the-agent-firewall", + ], + binary_candidates=["gh"], + supports_mediation=False, + can_block=False, + can_pause_resume=False, + ), + AgentAdapterProfile( + agent_id="aider", + display_name="Aider", + priority="p1_local_wrapper", + execution_modes=["managed_wrapper"], + native_surfaces=["repo_map", "git_worktree", "shell"], + event_sources=["wrapper_command", "repo_context", "git_diff"], + coverage_grade="managed_wrapper_adapter", + claim_boundary="Aider can be mediated when invoked through Invart's managed wrapper; direct shell execution remains outside Invart coverage.", + required_artifacts=["ledger", "proof", "coverage", "audit"], + source_urls=["https://aider.chat/docs/repomap.html"], + binary_candidates=["aider"], + supports_mediation=True, + can_block=True, + can_pause_resume=True, + ), + AgentAdapterProfile( + agent_id="openai-agents-sdk", + display_name="OpenAI Agents SDK", + priority="p1_framework_import", + execution_modes=["vendor_evidence_import", "framework_trace_import"], + native_surfaces=["guardrails", "human_in_the_loop", "tracing"], + event_sources=["sdk_trace", "approval_interruption", "guardrail_result"], + coverage_grade="vendor_evidence_import", + claim_boundary="SDK traces and guardrails are application-owned facts until an Invart adapter binds them to the ledger and mediation contract.", + required_artifacts=["external_evidence_manifest", "coverage", "audit"], + source_urls=["https://openai.github.io/openai-agents-python/guardrails/", "https://openai.github.io/openai-agents-python/tracing/"], + binary_candidates=["python"], + supports_mediation=False, + can_block=False, + can_pause_resume=False, + ), + AgentAdapterProfile( + agent_id="langgraph", + display_name="LangGraph", + priority="p2_framework_import", + execution_modes=["framework_trace_import"], + native_surfaces=["graph_state", "checkpoints", "tool_calls"], + event_sources=["trace_export", "checkpoint_log", "tool_call_log"], + coverage_grade="vendor_evidence_import", + claim_boundary="LangGraph traces support audit reconstruction, but runtime mediation requires an application-side Invart adapter.", + required_artifacts=["external_evidence_manifest", "coverage", "audit"], + source_urls=["https://langchain-ai.github.io/langgraph/"], + binary_candidates=["python"], + supports_mediation=False, + can_block=False, + can_pause_resume=False, + ), + AgentAdapterProfile( + agent_id="crewai", + display_name="CrewAI", + priority="p2_framework_import", + execution_modes=["framework_trace_import"], + native_surfaces=["crew_flow", "tool_calls", "memory"], + event_sources=["trace_export", "tool_call_log", "flow_log"], + coverage_grade="vendor_evidence_import", + claim_boundary="CrewAI flow artifacts can be imported for audit; Invart cannot claim mediation unless tool calls enter Invart before side effects.", + required_artifacts=["external_evidence_manifest", "coverage", "audit"], + source_urls=["https://docs.crewai.com/"], + binary_candidates=["python"], + supports_mediation=False, + can_block=False, + can_pause_resume=False, + ), + AgentAdapterProfile( + agent_id="generic", + display_name="Generic local command adapter", + priority="fallback", + execution_modes=["managed_wrapper"], + native_surfaces=["shell"], + event_sources=["wrapper_command"], + coverage_grade="managed_wrapper_adapter", + claim_boundary="Generic adapter coverage is limited to what enters Invart's wrapper and cannot be generalized to a named vendor product.", + required_artifacts=["ledger", "proof", "coverage", "audit"], + source_urls=["https://github.com/eric1311/Invart"], + binary_candidates=[], + supports_mediation=True, + can_block=True, + can_pause_resume=True, + ), +) + +_PROFILE_BY_ID = {profile.agent_id: profile for profile in _PROFILES} +_ALIASES = {"roo": "roo-code", "copilot": "github-copilot-cloud-agent"} def build_adapter_profile(kind: str, env: dict[str, str] | None = None) -> dict[str, Any]: env = dict(os.environ if env is None else env) + contract = get_adapter_profile(kind) return { - "schema_version": "invart.adapter_profile.v0.10", + "schema_version": PROFILE_SCHEMA_VERSION, "adapter": kind, + "agent_contract": contract, + **contract, "environment": summarize_environment(env), "process_supervision": { "target": "strong_consistency", @@ -25,6 +303,97 @@ def build_adapter_profile(kind: str, env: dict[str, str] | None = None) -> dict[ } +def adapter_profile_ids() -> list[str]: + return sorted(_PROFILE_BY_ID) + + +def get_adapter_profile(kind: str) -> dict[str, Any]: + normalized = _ALIASES.get(kind, kind) + profile = _PROFILE_BY_ID.get(normalized) + if not profile: + raise ValueError(f"unsupported adapter profile: {kind}") + return profile.to_dict() + + +def list_adapter_profiles() -> list[dict[str, Any]]: + return [profile.to_dict() for profile in _PROFILES if profile.agent_id != "generic"] + + +def adapter_profile_registry() -> dict[str, Any]: + profiles = list_adapter_profiles() + validation = validate_adapter_profile_truthfulness(profiles) + return { + "schema_version": PROFILE_REGISTRY_SCHEMA_VERSION, + "status": validation["status"], + "profiles": profiles, + "validation": validation, + "summary": { + "profiles": len(profiles), + "coverage_grades": _count_by(profiles, "coverage_grade"), + "priorities": _count_by(profiles, "priority"), + }, + } + + +def validate_adapter_profile_truthfulness(profiles: list[dict[str, Any]] | None = None) -> dict[str, Any]: + profiles = profiles or list_adapter_profiles() + required_fields = { + "agent_id", + "display_name", + "priority", + "execution_modes", + "native_surfaces", + "event_sources", + "coverage_grade", + "claim_boundary", + "required_artifacts", + "source_urls", + "last_reviewed", + } + valid_grades = { + "full_managed_adapter", + "managed_wrapper_adapter", + "native_event_bridge", + "vendor_evidence_import", + "discovery_only", + } + checks = { + "required_fields_present": all(required_fields.issubset(profile) and all(profile.get(field) for field in required_fields) for profile in profiles), + "coverage_grades_known": all(profile.get("coverage_grade") in valid_grades for profile in profiles), + "source_urls_https": all(str(url).startswith("https://") for profile in profiles for url in profile.get("source_urls", [])), + "claim_boundaries_present": all(bool(profile.get("claim_boundary")) for profile in profiles), + "full_managed_requires_artifacts": all( + {"ledger", "proof", "evidence_bundle"}.issubset(set(profile.get("required_artifacts", []))) + and "managed_runtime" in set(profile.get("execution_modes", [])) + and profile.get("supports_mediation") is True + and profile.get("can_block") is True + for profile in profiles + if profile.get("coverage_grade") == "full_managed_adapter" + ), + "import_only_not_mediated": all( + profile.get("supports_mediation") is False and profile.get("can_block") is False + for profile in profiles + if profile.get("coverage_grade") == "vendor_evidence_import" + ), + "discovery_only_not_mediated": all( + profile.get("supports_mediation") is False and profile.get("can_block") is False + for profile in profiles + if profile.get("coverage_grade") == "discovery_only" + ), + } + findings = [ + {"check_id": check_id, "status": "fail"} + for check_id, passed in checks.items() + if not passed + ] + return { + "schema_version": "invart.adapter_profile_truthfulness.v0.9.3", + "status": "pass" if not findings else "fail", + "checks": checks, + "findings": findings, + } + + def summarize_environment(env: dict[str, str], *, max_value_length: int = 96) -> dict[str, Any]: items = [] for key in sorted(env): @@ -47,3 +416,23 @@ def _fold(value: str, max_value_length: int) -> str: if len(value) <= max_value_length: return value return value[: max_value_length - 3] + "..." + + +def _count_by(items: list[dict[str, Any]], field_name: str) -> dict[str, int]: + counts: dict[str, int] = {} + for item in items: + key = str(item.get(field_name, "unknown")) + counts[key] = counts.get(key, 0) + 1 + return counts + + +__all__ = [ + "AgentAdapterProfile", + "adapter_profile_ids", + "adapter_profile_registry", + "build_adapter_profile", + "get_adapter_profile", + "list_adapter_profiles", + "summarize_environment", + "validate_adapter_profile_truthfulness", +] diff --git a/src/invart/surfaces/claude_adapter.py b/src/invart/surfaces/claude_adapter.py index 256576d..b546f68 100644 --- a/src/invart/surfaces/claude_adapter.py +++ b/src/invart/surfaces/claude_adapter.py @@ -127,7 +127,7 @@ def check_claude_code_environment(binary: str = "claude") -> dict[str, Any]: completed = subprocess.run([resolved, "--version"], check=False, capture_output=True, text=True, timeout=10) result["conformance"] = { "status": "pass" if completed.returncode == 0 else "warn", - "returncode": returncode, + "returncode": completed.returncode, "stdout": completed.stdout.strip()[:400], "stderr": completed.stderr.strip()[:400], } diff --git a/tests/test_integrations.py b/tests/test_integrations.py index be03dd4..32a548b 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -343,6 +343,117 @@ def test_full_claude_adapter_environment_check_reports_real_binary() -> None: assert result["available"] is True assert result["binary"].endswith("python3") or result["binary"] == "python3" assert "adapter_profile" in result + assert isinstance(result["conformance"]["returncode"], int) + + +def test_v093_adapter_profile_registry_reports_truthful_agent_contracts() -> None: + from invart.surfaces.adapter_profiles import get_adapter_profile, list_adapter_profiles, validate_adapter_profile_truthfulness + + profiles = list_adapter_profiles() + by_agent = {profile["agent_id"]: profile for profile in profiles} + required = { + "claude-code", + "codex", + "gemini-cli", + "cursor", + "opencode", + "openclaw", + "hermes", + "cline", + "roo-code", + "github-copilot-cloud-agent", + "aider", + "openai-agents-sdk", + "langgraph", + "crewai", + } + assert required.issubset(by_agent) + assert by_agent["claude-code"]["coverage_grade"] == "full_managed_adapter" + assert by_agent["github-copilot-cloud-agent"]["coverage_grade"] == "vendor_evidence_import" + assert by_agent["cursor"]["coverage_grade"] in {"native_event_bridge", "discovery_only", "vendor_evidence_import"} + assert all(profile["claim_boundary"] for profile in profiles) + assert all(profile["source_urls"] for profile in profiles) + + validation = validate_adapter_profile_truthfulness(profiles) + assert validation["status"] == "pass" + assert validation["checks"]["full_managed_requires_artifacts"] is True + assert validation["checks"]["import_only_not_mediated"] is True + assert validation["checks"]["discovery_only_not_mediated"] is True + + claude = get_adapter_profile("claude-code") + assert {"ledger", "proof", "evidence_bundle"}.issubset(set(claude["required_artifacts"])) + + +def test_v093_adapter_profile_cli_accepts_priority_agents() -> None: + assert main(["adapter", "profile", "--kind", "gemini-cli"]) == 0 + assert main(["adapter", "profile", "--kind", "github-copilot-cloud-agent"]) == 0 + + +def test_v093_real_agent_conformance_fixture_and_strict_live_modes(tmp_path: Path) -> None: + from invart.evaluation.real_agent_conformance import run_real_agent_conformance + + fake = tmp_path / "fake-agent" + fake.write_text("#!/usr/bin/env python3\nimport sys\nsys.exit(0)\n", encoding="utf-8") + fake.chmod(0o755) + + report = run_real_agent_conformance( + out_dir=tmp_path / "pass", + agents=["claude-code", "codex"], + binary_overrides={"claude-code": str(fake), "codex": str(fake)}, + require_live=True, + ) + assert report["schema_version"] == "invart.real_agent_conformance.v0.9.3" + assert report["status"] == "pass" + assert report["summary"]["passed_agents"] == 2 + assert all(agent["binary"]["status"] == "found" for agent in report["agents"]) + assert all(agent["managed_run"]["status"] == "pass" for agent in report["agents"]) + assert Path(report["artifacts"]["report_json"]).exists() + assert Path(report["artifacts"]["report_html"]).exists() + + advisory_missing = run_real_agent_conformance( + out_dir=tmp_path / "advisory-missing", + agents=["hermes"], + binary_overrides={"hermes": str(tmp_path / "missing-hermes")}, + require_live=False, + ) + assert advisory_missing["status"] == "pass" + assert advisory_missing["agents"][0]["status"] == "blocked_missing_binary" + assert advisory_missing["agents"][0]["claim_boundary"] + + strict_missing = run_real_agent_conformance( + out_dir=tmp_path / "strict-missing", + agents=["hermes"], + binary_overrides={"hermes": str(tmp_path / "missing-hermes")}, + require_live=True, + ) + assert strict_missing["status"] == "fail" + assert strict_missing["agents"][0]["status"] == "blocked_missing_binary" + + +def test_v093_real_agent_cli_and_benchmark(tmp_path: Path) -> None: + fake = tmp_path / "fake-agent" + fake.write_text("#!/usr/bin/env python3\nimport sys\nsys.exit(0)\n", encoding="utf-8") + fake.chmod(0o755) + out = tmp_path / "cli" + + assert main([ + "real-agent", + "check", + "--agent", + "claude-code", + "--agent", + "codex", + "--binary", + f"claude-code={fake}", + "--binary", + f"codex={fake}", + "--require-live", + "--out-dir", + str(out), + ]) == 0 + assert (out / "real-agent-conformance.json").exists() + assert main(["real-agent", "report", "--run-dir", str(out), "--out", str(tmp_path / "report.html")]) == 0 + assert main(["eval", "benchmark", "--suite", "v0.9.3-agent-adapter-contract"]) == 0 def test_v13_adapter_run_uses_file_write_enforcement(tmp_path: Path) -> None: