From 8e0dd9e068178808c5b4e5e3a77c9f8bafea5ede Mon Sep 17 00:00:00 2001 From: Yong Date: Sun, 13 Sep 2026 22:50:21 -0500 Subject: [PATCH 1/2] feat(harness): session-bound confinement attestation with a port-pinned loopback relay policy A second network policy, `loopback`, compiles to outbound on exactly one local TCP port (`loopback_port`, the arm's provider relay) and nothing else; SBPL's `localhost` names every address this host owns, so the pin is what makes the grant exclusive. The capability audit can now run under a session's exact policy: sentinels are placed inside the policy's own roots (caller-named homes or the last directory of each list), registered for removal one by one, reclaimed only when a well-formed owner marker names a dead process, and the retained attestation carries the profile hash and policy id the launch wrapper demands. Under `loopback` the matrix proves the pinned port is reachable, another local port is refused, and an external connect is refused outright (a hang is a failed case, never an abort). The symlink case under a session policy targets the controller's private temp tree; the report replaces hidden roots with digests. wrap_verified_command requires the case set keyed by the compiled network policy and cross-checks the attested network value. Provider bindings accept `relay_port`, the relay binds it, and admission refuses a loopback arm whose relay port differs from its pinned port. Probes never widen an arm's grants (nc or an admitted Python interpreter; printf stays the unlisted control); probe locale pinned to C. MacOSPolicy.from_canonical replaces nine hand-rolled reconstructions; roots must be printable ASCII. CLI: audit-macos --policy [--sentinel-*]; malformed input exits 2 with JSON. Successors cse-causal-study-v23 / cse-causal-study-pilot-v9; registry count -> 58. TRT-5, TRT-6, TRT-7, MON-8, CON-8, CON-5. Refs #347, #567, ADR-66 (PR7 part 2). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01EMTXwWm3ZppWcJvQQw2qbf --- .../declaration.json | 381 ++++++++++ .../freeze-manifest.json | 561 ++++++++++++++ .../cse-causal-study-pilot-v9/protocol.json | 91 +++ .../cse-causal-study-v23/declaration.json | 348 +++++++++ .../cse-causal-study-v23/freeze-manifest.json | 533 +++++++++++++ .../macos-confinement/README.md | 11 + docs/monolithic/matched-session.md | 28 + src/aisle/harness/matched_run.py | 7 +- src/aisle/harness/matched_session.py | 17 +- src/aisle/harness/matched_tools.py | 8 +- src/aisle/harness/monolithic_run_prepare.py | 7 +- src/aisle/harness/provider_relay.py | 15 +- src/aisle/harness/treatment_confinement.py | 705 +++++++++++++++--- src/aisle/harness/typed_graph_audit.py | 7 +- src/aisle/harness/typed_node_host.py | 7 +- src/aisle/harness/typed_validation.py | 7 +- src/aisle/harness/worker_capability.py | 6 +- src/aisle/monolith/worker_config.py | 11 +- tests/accept/test_provider_relay_transport.py | 27 + tests/unit/test_freeze_registry.py | 2 +- tests/unit/test_treatment_confinement.py | 443 +++++++++++ tests/unit/test_typed_graph_preflight.py | 4 +- tests/unit/test_typed_graph_stage.py | 3 +- tests/unit/test_typed_node_host.py | 3 +- tests/unit/test_worker_capability.py | 4 +- tools/matched_campaign.py | 7 +- 26 files changed, 3055 insertions(+), 188 deletions(-) create mode 100644 analysis/freeze/cse-causal-study-pilot-v9/declaration.json create mode 100644 analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json create mode 100644 analysis/freeze/cse-causal-study-pilot-v9/protocol.json create mode 100644 analysis/freeze/cse-causal-study-v23/declaration.json create mode 100644 analysis/freeze/cse-causal-study-v23/freeze-manifest.json diff --git a/analysis/freeze/cse-causal-study-pilot-v9/declaration.json b/analysis/freeze/cse-causal-study-pilot-v9/declaration.json new file mode 100644 index 00000000..e9c7c924 --- /dev/null +++ b/analysis/freeze/cse-causal-study-pilot-v9/declaration.json @@ -0,0 +1,381 @@ +{ + "analysis": { + "scripts": [ + "src/aisle/harness/benchmark_statistics.py", + "src/aisle/harness/causal_study.py", + "src/aisle/harness/pilot_records.py" + ], + "seed": 500001 + }, + "artifacts": { + "agent_adapters": "tools/agent_adapters.py", + "analyzer": "src/aisle/harness/benchmark_statistics.py", + "bnd_calibration_manifest": "analysis/freeze/bnd-task-band-calibration-v11/freeze-manifest.json", + "bnd_perception_audit_report": "analysis/perception-audit/records/bnd-perception-corpus-03/report.json", + "budget_ceiling": "harness/budget.toml", + "dependency_declaration": "pyproject.toml", + "dependency_lock": "uv.lock", + "dora_runtime_selection": "dora-runtime.json", + "frontend_request_authorization_documentation": "docs/benchmark/v1/frontend-request-authorization.md", + "matched_session_decision": "docs/decisions/ADR-62.md", + "matched_session_documentation": "docs/monolithic/matched-session.md", + "matched_source:assets/so101/so101.urdf": "assets/so101/so101.urdf", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "docs/monolithic/allowlist-t1-l2.json", + "matched_source:docs/monolithic/candidates.json": "docs/monolithic/candidates.json", + "matched_source:docs/monolithic/experts-t1-l2.json": "docs/monolithic/experts-t1-l2.json", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "docs/monolithic/interface-map-t1-l2.json", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "docs/monolithic/parity-protocol-t1-l2.json", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "docs/monolithic/treatment-table-t1-l2.json", + "matched_source:env/limits.toml": "env/limits.toml", + "matched_source:experts/monolithic/expert_t1_l2.py": "experts/monolithic/expert_t1_l2.py", + "matched_source:graphs/monolithic_t1_l2.yaml": "graphs/monolithic_t1_l2.yaml", + "matched_source:src/aisle/__init__.py": "src/aisle/__init__.py", + "matched_source:src/aisle/embodiment.py": "src/aisle/embodiment.py", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "src/aisle/harness/_code_mode_protocol/__init__.py", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "src/aisle/harness/_code_mode_protocol/descriptor.pb", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "src/aisle/harness/_code_mode_protocol/protocol.proto", + "matched_source:src/aisle/harness/candidate_launch.py": "src/aisle/harness/candidate_launch.py", + "matched_source:src/aisle/harness/candidates.py": "src/aisle/harness/candidates.py", + "matched_source:src/aisle/harness/cli.py": "src/aisle/harness/cli.py", + "matched_source:src/aisle/harness/code_mode_audit.py": "src/aisle/harness/code_mode_audit.py", + "matched_source:src/aisle/harness/code_mode_authority.py": "src/aisle/harness/code_mode_authority.py", + "matched_source:src/aisle/harness/code_mode_proxy.py": "src/aisle/harness/code_mode_proxy.py", + "matched_source:src/aisle/harness/code_mode_runner.py": "src/aisle/harness/code_mode_runner.py", + "matched_source:src/aisle/harness/common.py": "src/aisle/harness/common.py", + "matched_source:src/aisle/harness/frontend_app_server.py": "src/aisle/harness/frontend_app_server.py", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "src/aisle/harness/frontend_app_server_audit.py", + "matched_source:src/aisle/harness/frontend_dispatch.py": "src/aisle/harness/frontend_dispatch.py", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "src/aisle/harness/frontend_dispatch_audit.py", + "matched_source:src/aisle/harness/frontend_request_audit.py": "src/aisle/harness/frontend_request_audit.py", + "matched_source:src/aisle/harness/frontend_request_authority.py": "src/aisle/harness/frontend_request_authority.py", + "matched_source:src/aisle/harness/guard_divergence.py": "src/aisle/harness/guard_divergence.py", + "matched_source:src/aisle/harness/heldout_eval.py": "src/aisle/harness/heldout_eval.py", + "matched_source:src/aisle/harness/matched_app_server.py": "src/aisle/harness/matched_app_server.py", + "matched_source:src/aisle/harness/matched_collection.py": "src/aisle/harness/matched_collection.py", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "src/aisle/harness/matched_dynamic_run.py", + "matched_source:src/aisle/harness/matched_evidence.py": "src/aisle/harness/matched_evidence.py", + "matched_source:src/aisle/harness/matched_frontend.py": "src/aisle/harness/matched_frontend.py", + "matched_source:src/aisle/harness/matched_run.py": "src/aisle/harness/matched_run.py", + "matched_source:src/aisle/harness/matched_run_launch.py": "src/aisle/harness/matched_run_launch.py", + "matched_source:src/aisle/harness/matched_runtime.py": "src/aisle/harness/matched_runtime.py", + "matched_source:src/aisle/harness/matched_session.py": "src/aisle/harness/matched_session.py", + "matched_source:src/aisle/harness/matched_tool_service.py": "src/aisle/harness/matched_tool_service.py", + "matched_source:src/aisle/harness/matched_tools.py": "src/aisle/harness/matched_tools.py", + "matched_source:src/aisle/harness/monolith.py": "src/aisle/harness/monolith.py", + "matched_source:src/aisle/harness/monolithic.py": "src/aisle/harness/monolithic.py", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "src/aisle/harness/monolithic_run_evidence.py", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "src/aisle/harness/monolithic_run_prepare.py", + "matched_source:src/aisle/harness/pilot_records.py": "src/aisle/harness/pilot_records.py", + "matched_source:src/aisle/harness/provider_relay.py": "src/aisle/harness/provider_relay.py", + "matched_source:src/aisle/harness/provider_response_authority.py": "src/aisle/harness/provider_response_authority.py", + "matched_source:src/aisle/harness/provider_runner.py": "src/aisle/harness/provider_runner.py", + "matched_source:src/aisle/harness/provider_source_audit.py": "src/aisle/harness/provider_source_audit.py", + "matched_source:src/aisle/harness/registry.py": "src/aisle/harness/registry.py", + "matched_source:src/aisle/harness/rollout.py": "src/aisle/harness/rollout.py", + "matched_source:src/aisle/harness/rollout_client.py": "src/aisle/harness/rollout_client.py", + "matched_source:src/aisle/harness/simulator_work.py": "src/aisle/harness/simulator_work.py", + "matched_source:src/aisle/harness/traces.py": "src/aisle/harness/traces.py", + "matched_source:src/aisle/harness/treatment_ambient.py": "src/aisle/harness/treatment_ambient.py", + "matched_source:src/aisle/harness/treatment_confinement.py": "src/aisle/harness/treatment_confinement.py", + "matched_source:src/aisle/harness/treatment_integrity.py": "src/aisle/harness/treatment_integrity.py", + "matched_source:src/aisle/harness/treatment_postflight.py": "src/aisle/harness/treatment_postflight.py", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "src/aisle/harness/typed_execution_bundle.py", + "matched_source:src/aisle/harness/typed_graph_audit.py": "src/aisle/harness/typed_graph_audit.py", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "src/aisle/harness/typed_graph_hosts.py", + "matched_source:src/aisle/harness/typed_graph_stage.py": "src/aisle/harness/typed_graph_stage.py", + "matched_source:src/aisle/harness/typed_node_host.py": "src/aisle/harness/typed_node_host.py", + "matched_source:src/aisle/harness/typed_node_requests.py": "src/aisle/harness/typed_node_requests.py", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "src/aisle/harness/typed_node_supervisor.py", + "matched_source:src/aisle/harness/typed_node_worker.py": "src/aisle/harness/typed_node_worker.py", + "matched_source:src/aisle/harness/typed_run_prepare.py": "src/aisle/harness/typed_run_prepare.py", + "matched_source:src/aisle/harness/typed_snapshot.py": "src/aisle/harness/typed_snapshot.py", + "matched_source:src/aisle/harness/typed_stage_provider.py": "src/aisle/harness/typed_stage_provider.py", + "matched_source:src/aisle/harness/typed_validation.py": "src/aisle/harness/typed_validation.py", + "matched_source:src/aisle/harness/typed_worker_launch.py": "src/aisle/harness/typed_worker_launch.py", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "src/aisle/harness/typed_worker_provisioning.py", + "matched_source:src/aisle/harness/validate.py": "src/aisle/harness/validate.py", + "matched_source:src/aisle/harness/worker_authority_probe.py": "src/aisle/harness/worker_authority_probe.py", + "matched_source:src/aisle/harness/worker_capability.py": "src/aisle/harness/worker_capability.py", + "matched_source:src/aisle/harness/worker_declaration.py": "src/aisle/harness/worker_declaration.py", + "matched_source:src/aisle/harness/worker_network_probe.py": "src/aisle/harness/worker_network_probe.py", + "matched_source:src/aisle/kinematics.py": "src/aisle/kinematics.py", + "matched_source:src/aisle/mobility/guard.py": "src/aisle/mobility/guard.py", + "matched_source:src/aisle/monolith/confinement.py": "src/aisle/monolith/confinement.py", + "matched_source:src/aisle/monolith/primitive_api.py": "src/aisle/monolith/primitive_api.py", + "matched_source:src/aisle/monolith/primitives.py": "src/aisle/monolith/primitives.py", + "matched_source:src/aisle/monolith/proxy.py": "src/aisle/monolith/proxy.py", + "matched_source:src/aisle/monolith/requests.py": "src/aisle/monolith/requests.py", + "matched_source:src/aisle/monolith/supervisor.py": "src/aisle/monolith/supervisor.py", + "matched_source:src/aisle/monolith/wire.py": "src/aisle/monolith/wire.py", + "matched_source:src/aisle/monolith/worker.py": "src/aisle/monolith/worker.py", + "matched_source:src/aisle/monolith/worker_config.py": "src/aisle/monolith/worker_config.py", + "matched_source:src/aisle/monolith/worker_launch.py": "src/aisle/monolith/worker_launch.py", + "matched_source:src/aisle/nodes/budget_guard.py": "src/aisle/nodes/budget_guard.py", + "matched_source:src/aisle/nodes/h6_fault.py": "src/aisle/nodes/h6_fault.py", + "matched_source:src/aisle/nodes/l2_pose.py": "src/aisle/nodes/l2_pose.py", + "matched_source:src/aisle/nodes/label_reader.py": "src/aisle/nodes/label_reader.py", + "matched_source:src/aisle/nodes/monolith_broker.py": "src/aisle/nodes/monolith_broker.py", + "matched_source:src/aisle/nodes/perception_session.py": "src/aisle/nodes/perception_session.py", + "matched_source:src/aisle/scenes/locations.toml": "src/aisle/scenes/locations.toml", + "matched_source:src/aisle/scenes/meds.toml": "src/aisle/scenes/meds.toml", + "matched_source:src/aisle/scenes/pharmacy.py": "src/aisle/scenes/pharmacy.py", + "matched_source:src/aisle/scenes/physics.toml": "src/aisle/scenes/physics.toml", + "matched_source:src/aisle/scenes/planogram.toml": "src/aisle/scenes/planogram.toml", + "matched_source:src/aisle/scenes/store.py": "src/aisle/scenes/store.py", + "matched_source:src/aisle/topics.py": "src/aisle/topics.py", + "matched_source:src/aisle/turn_node.py": "src/aisle/turn_node.py", + "matched_source:src/aisle/turns.py": "src/aisle/turns.py", + "matched_source:src/aisle/verifier/calibration.py": "src/aisle/verifier/calibration.py", + "matched_source:src/aisle/verifier/models.lock": "src/aisle/verifier/models.lock", + "matched_source:src/aisle/verifier/models.py": "src/aisle/verifier/models.py", + "matched_source:src/aisle/verifier/placement.toml": "src/aisle/verifier/placement.toml", + "matched_source:src/aisle/verifier/realistic.py": "src/aisle/verifier/realistic.py", + "matched_source:src/aisle/verifier/stages.py": "src/aisle/verifier/stages.py", + "matched_source:src/aisle/verifier/thresholds.toml": "src/aisle/verifier/thresholds.toml", + "matched_source:tools/campaign.py": "tools/campaign.py", + "matched_source:tools/matched_campaign.py": "tools/matched_campaign.py", + "monolithic_validators": "src/aisle/harness/monolithic.py", + "nested_host_documentation": "docs/monolithic/code-mode-admission.md", + "pilot_assignments_cli": "tools/pilot_assignments.py", + "pilot_decision": "docs/decisions/ADR-66.md", + "pilot_records_cli": "tools/pilot_records.py", + "protocol": "analysis/freeze/cse-causal-study-pilot-v9/protocol.json", + "protocol_template": "analysis/statistics/protocol-template.json", + "provider_admission_documentation": "docs/monolithic/provider-admission.md", + "research_prompt": "harness/CLAUDE.research.md", + "seed_commitment_predecessor": "analysis/freeze/cse-causal-study-pilot-v8/freeze-manifest.json", + "session_runner": "tools/campaign.py", + "session_validators": "src/aisle/harness/causal_study.py", + "simulator_bridge": "src/aisle/nodes/dora_genesis.py", + "spec": "specs/500-typed-monolithic-causal-study.md", + "treatment_integrity": "src/aisle/harness/treatment_integrity.py", + "treatment_postflight": "src/aisle/harness/treatment_postflight.py", + "treatment_table": "analysis/monolithic-control/treatment-table-v1.json" + }, + "budgets": { + "campaign_ceiling": "harness/budget.toml", + "rollouts_per_session": 20, + "session_tokens": 200000, + "session_wall_s": 3600, + "sim_episodes_per_session": 160 + }, + "campaign_id": "cse-causal-study-pilot-v9", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-pilot-v9/protocol.json --purpose power", + "uv run harness stats power --protocol analysis/freeze/cse-causal-study-pilot-v9/protocol.json --output analysis/freeze/cse-causal-study-pilot-v9/power.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json --allow-withheld-seeds" + ], + "decision_rules": { + "alpha": 0.05, + "equivalence_margin": { + "cost_at_equal_quality": { + "acceptance_probability": 0.1, + "heldout_quality": 0.1, + "rule": "CSE-5: both must pass or cost claims stay unconditional", + "sidedness": "two-sided 0.90" + } + }, + "multiplicity": "single primary endpoint", + "power": 0.8, + "primary_rule": "one-sided superiority at alpha 0.05 on the pooled Newcombe interval lower bound > 0 with no reversed stratum; otherwise weakened/rejected/undecidable by the #358 rule (CSE-3)", + "smallest_effect": { + "measure": "risk difference in session success", + "value": 0.25 + }, + "stopping_rule": "fixed n from the frozen power analysis using pilot-only control success; absolute floor 10 randomized assignments per arm per agent-system x task stratum (CSE-8); no optional stopping or extension" + }, + "disposition": "registered 2026-09-13; collection pending. Supersedes the zero-session pilot-v1 lineage with the instrument set the pilot can actually run on (T1-L2 realistic pair, #581 to #586). Named limitations: BND-7 not eligible (recorded failed), external-review and parity gates pending, and the session-bound capability audit not merged (an unattested environment stops every session).", + "endpoints": { + "primary": [ + { + "aggregation": "risk difference typed_dataflow - monolithic; exact arm intervals; Newcombe interval; per agent-system x task stratum before pooling (CSE-1, CSE-17)", + "direction": "higher", + "id": "E-CSE-1", + "outcome": "session_success", + "status": "inferential", + "unit": "agent_session" + } + ], + "secondary": [ + { + "aggregation": "counts by arm", + "direction": "descriptive", + "id": "E-CSE-2", + "outcome": "validation_and_launch_failure_class", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "Kaplan-Meier with censoring", + "direction": "lower", + "id": "E-CSE-3", + "outcome": "time_to_first_accepted_system", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "unconditional session estimands; no at-equal-quality wording unless CSE-5 equivalence passes", + "direction": "lower", + "id": "E-CSE-4", + "outcome": "tokens_tool_calls_wall_active_time_sim_work_rollouts", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "counts", + "direction": "descriptive", + "id": "E-CSE-5", + "outcome": "safety_interventions", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "dropped unless a blinded rubric is ratified before freeze (CSE-18)", + "direction": "n/a", + "id": "E-CSE-6", + "outcome": "artifact_legibility", + "status": "exploratory", + "unit": "agent_session" + } + ] + }, + "exclusions": { + "deviation_policy": "any behaviour-affecting change after registration is a documented deviation record and a new campaign version id; confirmatory data never revises the registered protocol", + "infrastructure": [ + "independent-controller-determined, arm-neutral facts only (CSE-14): host outage, provider interruption before first tool call, launcher crash before the session started" + ], + "rerun_policy": "a permitted rerun receives a new randomized id; the original row is retained with its exclusion reason and never replaced in place", + "treatment_integrity": [ + "contamination sentinel tripped (CSE-12)", + "model alias or CLI drift outside the frozen identity (CSE-6)", + "integrity verdict authored session-side (CSE-11)" + ] + }, + "hypotheses": [ + { + "direction": "typed_dataflow - monolithic > 0", + "id": "H-CSE-1", + "statement": "The bundled typed-dataflow surface raises the probability that a fresh coding-agent session produces a launching system passing the hidden held-out acceptance within budget, versus the equal-capability monolithic surface." + } + ], + "instrument_set": { + "agent_systems": [ + { + "binary": "codex", + "binary_sha256": "b973d440acac501fd2594a43e7ca9ce41e0a65b9dfb28d0d7a7837c99e1261e3", + "frozen": true, + "id": "codex-cli", + "install": "private prefix ~/aisle-private/codex-0.153.4 (the qualified darwin-arm64 binary, not the JS shim); ChatGPT-login credentials copied byte-identically into both arms' private CODEX_HOME, hash recorded only", + "version_observed": "0.153.4" + } + ], + "candidate": { + "id": "t1-l2-realistic", + "monolithic_module": "experts/monolithic/expert_t1_l2.py", + "monolithic_template": "graphs/monolithic_t1_l2.yaml", + "perception": "L2", + "reset": "teleport", + "typed_graph": "graphs/expert_t1_l2.yaml", + "verifier": "realistic (oracle demoted to the held-out scoring tap, BND-3)" + }, + "candidate_table_sha256": "08a1dc56502abc4673cb0b54f01664382214bb473c3ef293b1df7b6c74f64778", + "items": [ + "short_composition:t1-l2-realistic" + ], + "kind": "selected_task" + }, + "integrity_checks": [ + { + "gate": "CON-14 spec approval", + "kind": "external_review", + "owner_role": "project owner", + "pilot_record_2026_09_13": "pending; external review, not a pilot prerequisite (ADR-66 section 2); named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "STA-12 independent statistical review", + "kind": "external_review", + "owner_role": "independent statistician", + "pilot_record_2026_09_13": "pending; external review, not a pilot prerequisite (ADR-66 section 2); named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "MON-1 treatment table and expert parity (#344)", + "kind": "machine_check", + "owner_role": "parity harness; control surface exists, independent authorship and operator evidence remain pending", + "pilot_record_2026_09_13": "pending; the T1-L2 pair (#583) passes the interface, table and module checks but has no operator parity record at an immutable hash; named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "BND-12 task-band freeze (#346)", + "kind": "machine_check", + "owner_role": "task-band auditor", + "pilot_record_2026_09_13": "recorded failed: bnd-task-band-calibration-v11 regenerated the BND-5 perception audit on a 32-seed T1-L2 corpus; 22 of 22 strata below the 0.90 floor (BND-7), led by target_class=ibuprofen refusals; role open (BND-10). ADR-66 section 2: a failing machine gate is a named limitation of this pilot, not a blocker", + "record": "analysis/perception-audit/records/bnd-perception-corpus-03/report.json", + "status": "failed" + }, + { + "gate": "TRT confinement/mutation/agent-path (#353)", + "kind": "machine_check", + "owner_role": "integrity controller", + "pilot_record_2026_09_13": "pending; the session-bound capability audit (loopback relay policy) is not merged; an unattested environment is a stop condition for every session", + "record": null, + "status": "pending" + }, + { + "gate": "seed commitment verification", + "kind": "machine_check", + "owner_role": "holder of the original private seed and salt sources", + "pilot_record_2026_09_13": "not applicable to this registration: it commits a fresh private seed source directly (no inheritance); kept for shape parity and left pending", + "record": null, + "status": "pending" + }, + { + "gate": "MON-8/MON-12 matched-session acceptance (#519)", + "kind": "machine_check", + "owner_role": "matched-session controller; complete acceptance evidence still required", + "pilot_record_2026_09_13": "pending; frontend conformance qualified on the T1 oracle development surface only (#568); the T1-L2 realistic surface runs under the same executor with the same admission checks; named limitation", + "record": null, + "status": "pending" + } + ], + "issue": 347, + "pilot_design": { + "agent_system": "codex-cli 0.153.4 only (the owned provider relay, the ChatGPT-login account header and the pinned conformance profile are Codex-bound)", + "arm_order": "sealed balanced plan (revealed one assignment at a time); no longer a fixed alternation", + "arms": [ + "typed", + "monolithic" + ], + "assignment": "tools/pilot_assignments.py create --protocol analysis/freeze/cse-causal-study-pilot-v8/protocol.json (sealed balanced plan, one typed and one monolithic assignment per temporal block, TRT-8/CSE-7); reveal exactly one assignment before each session; the ledger retains every assignment, started or not", + "below_cse8_floor": "6 per arm is below the CSE-8 floor of 10 by design; this registration can never become a confirmatory estimate (STA-3)", + "decision": "docs/decisions/ADR-66.md (pilot first; agent-drafted under the ADR-65 label, renumbered by #568)", + "executor": "tools/matched_campaign.py admit / run --purpose engineering / collect-run, then the held-out evaluation; retained and unscored, output root ~/aisle-private/raw/cse-causal-study-pilot-v8/", + "records": "tools/pilot_records.py derives one aisle.stats.records.v1 row per ledger assignment; harness stats validate --purpose analysis; harness stats analyze; harness stats power from observed control success (CSE-8)", + "reporting": "analysis/reports/cse_pilot_v8.md; results enter docs/claim-evidence.yaml only as CSE-8 power inputs under the unrun confirmatory claim", + "rerun_policy": "no reruns; a failed attempt is retained under a new session id, never overwritten", + "session_success": "controller-run held-out evaluation of the retained final snapshot on the 32 private seeds after the agent process ends, oracle as the held-out scorer (BND-3); success iff at least 28 of 32 oracle successes; a launch failure counts as a failed seed (CSE-2, CSE-9)", + "sessions_per_arm": 6, + "stop_conditions": [ + "wrong_object outcome", + "guard bypass", + "unattested environment", + "relay bypass (provider traffic outside the owned relay)", + "any oracle field on a policy input" + ], + "task_stratum": "short_composition on candidate t1-l2-realistic (T1 franka, L2 perception, realistic verifier); bnd-task-band-calibration-v11 records it as not perception-eligible (BND-7), carried as a machine-gate limitation per ADR-66" + }, + "purpose": "pilot", + "schema_version": "aisle.freeze.declaration.v1", + "seed_commitment": { + "inherited_from": "analysis/freeze/cse-causal-study-pilot-v8/freeze-manifest.json", + "rule": "32 distinct integers in [1000,9999) drawn once from the OS CSPRNG on 2026-09-13, retained only under ~/aisle-private/freeze/cse-causal-study-pilot-v8/, disjoint by value from every private seed list committed so far (including cse-causal-study-pilot-v1 and the cse-causal-study-v1 confirmatory commitment) and from the committed development seed lists; never inherited by a confirmatory declaration (ADR-66); revealed after the pilot closes. The same directory holds the controller-private 256-bit randomization seed for the sealed assignment plan (TRT-8); its commitment is recorded in the assignment ledger, never here.", + "salt_source": "~/aisle-private/freeze/cse-causal-study-pilot-v8/salt.bin", + "seeds_source": "~/aisle-private/freeze/cse-causal-study-pilot-v8/seeds.json" + }, + "spec": "500", + "superseded": "cse-causal-study-pilot-v8 (session-bound confinement attestation: loopback network policy for the provider relay and the capability audit run under the session's exact policy (TRT-5/TRT-7); seed commitment, collection rules and all pending gates preserved)" +} diff --git a/analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json b/analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json new file mode 100644 index 00000000..e4be77c1 --- /dev/null +++ b/analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json @@ -0,0 +1,561 @@ +{ + "analysis_script_hashes": { + "src/aisle/harness/benchmark_statistics.py": "sha256:22cde3e9d7c5cc80874c7eec30aa2898e30476d6c939c94ab3009e8b12af641e", + "src/aisle/harness/causal_study.py": "sha256:1b1c53d6389881c146726d865a741a681f3d5039c9bf99558a53ee368250971e", + "src/aisle/harness/pilot_records.py": "sha256:68ac7ef9afd449d8527e2ffe99a9ef926e06253c466f3421bc1848e9aa094b70" + }, + "analysis_seed": 500001, + "artifact_hashes": { + "agent_adapters": "sha256:848f8f7da5ad978e53712991a688eb74f56c7a77deabee8afae717ad712666ca", + "analyzer": "sha256:22cde3e9d7c5cc80874c7eec30aa2898e30476d6c939c94ab3009e8b12af641e", + "bnd_calibration_manifest": "sha256:8615de2078337065bf6045c3e4bb0f92924987606768fdb699a8ce58ce8f8b57", + "bnd_perception_audit_report": "sha256:27134904b4b739423085a9b3c9b23678658d39f7a48e00cd2fbeaedea8c7a4d8", + "budget_ceiling": "sha256:bcbb79aa146afed1d877da18093175ae144b4607db50119022ef4f8173122cda", + "dependency_declaration": "sha256:add4e718001da7678b404bdf37ea8d9889f563324ffba9b03fe65c46b172ee4c", + "dependency_lock": "sha256:9b8a82a17ba68c192f045405feed3ae7046af9568bddcb398292c20f0ed6f5a8", + "dora_runtime_selection": "sha256:f9eb7ffe285a0f0ed26d50303e31cf1e94d2f93ed62b599071dff5a183c15f53", + "frontend_request_authorization_documentation": "sha256:a1d4e65c0e467af4bd77922b95b12f26ee6478932eb5dd460de34b81249ca86b", + "matched_session_decision": "sha256:d88e0ef18c92cedff7b74dd3ffafed14d92affaec65beb9ddfec2801b365dc61", + "matched_session_documentation": "sha256:ff9aded2c28cfd857478095b044361062492a8dbf79a143fc9415a9f1545a22f", + "matched_source:assets/so101/so101.urdf": "sha256:3a65d2d35e68a8d2f0c2cc176d19b884506543c93ba72980145b80abe276022c", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "sha256:a284af4a06649763c7fac25eefbdd43ed068f00aab09f2aaaa17066e7b705ac5", + "matched_source:docs/monolithic/candidates.json": "sha256:08a1dc56502abc4673cb0b54f01664382214bb473c3ef293b1df7b6c74f64778", + "matched_source:docs/monolithic/experts-t1-l2.json": "sha256:263ea1a08c76f0c7ce34212a3cee4a71a0c4bbb67f8f11a319781e84cdde3cdf", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "sha256:b80c35f849afc3e28986710134c21c68247024c507d89c3a031513f36d2949e7", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "sha256:a1df92be604e4b0f65890640bca8f2b36f199b888ca1f8ef3df80ad13f2d764c", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "sha256:b19cc6366e3f773d822736812b56f5a1e00a78fc7bdddfe893cce80a3e236e2d", + "matched_source:env/limits.toml": "sha256:1ab1f4fd5d52e7949b81d21933daaa2c66e80ef07fdedf8217280468e0142edc", + "matched_source:experts/monolithic/expert_t1_l2.py": "sha256:93ade82db523fe808312f9d83c88ab3464eee303be1bfc3f824880bf1de7924c", + "matched_source:graphs/monolithic_t1_l2.yaml": "sha256:ff89c18e9466eef497836c95f8c01743f06fba124ac509c32ecda8d07b676ff8", + "matched_source:src/aisle/__init__.py": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "matched_source:src/aisle/embodiment.py": "sha256:d17c204492d680175cbaf365aaf164e2a6049da23ce125a9b9f21606b79f4cca", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "sha256:7e7fb09527b647c2bb13787011de8f9a4412e8b6f304e89efa107162d8150e82", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "sha256:459465e666c6a76a70e902c10c58905dfd8417b6c2ca3e038be99624e3542540", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "sha256:406d8f47d8d0b95a9d4ae104890e5e52764c9597e9271f75ff434a4ce24748d0", + "matched_source:src/aisle/harness/candidate_launch.py": "sha256:654eb90264703e3ce3ab599bb8e143047eaee288c8852c5b09c74e3147039270", + "matched_source:src/aisle/harness/candidates.py": "sha256:d1b441a30288a6c0500aaa765a24d4aeecca4f8d52e96f633befc50c84ecf5f5", + "matched_source:src/aisle/harness/cli.py": "sha256:c855f059d28409ffaa14be99ad3fc0bdc16fc33cf1a3c9584a75d968bd59d72d", + "matched_source:src/aisle/harness/code_mode_audit.py": "sha256:24638163f205408da1c6cf20a1b77c80a0d740a3f8ed44696eb90baf7de61f6a", + "matched_source:src/aisle/harness/code_mode_authority.py": "sha256:217ea025f10a4dc02b4b0ff2ff733adf4a695d1098b3c8ce0d2b27b8a140853d", + "matched_source:src/aisle/harness/code_mode_proxy.py": "sha256:333eee561d610489d2f48f9359ad94a3d9d8ff729fbae0fd3dec4acf64be2cb2", + "matched_source:src/aisle/harness/code_mode_runner.py": "sha256:4f91577dcfc6c239bf4111bd6735a64e97d6e22b7ca6287389abae09023cc25d", + "matched_source:src/aisle/harness/common.py": "sha256:ef62bf518a99c8c4154908bc9cf99e68402e6ae6c8e3fb5ec30c3f5fce9b5fac", + "matched_source:src/aisle/harness/frontend_app_server.py": "sha256:d21b9bf218efc3066399b8b22bbe3892e772433da315a9ecd40e2773369e9283", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "sha256:fb3cf5b8a115eb93e3f292dae694a4318cbe43e9552347fbc25ea8e43102ddbb", + "matched_source:src/aisle/harness/frontend_dispatch.py": "sha256:6bff2abbacca6c332abdb5c5cb4cb3d688d8b00535e3f428633619bf91cd30c8", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "sha256:dc75024bd52c21c89a6112da847875a1d0ebf44f53a3704705e0bb8c81b9bc4a", + "matched_source:src/aisle/harness/frontend_request_audit.py": "sha256:a4bc60b2ce1968ada1776324b72ad87025b93d366046029445fdf9b2e92f8830", + "matched_source:src/aisle/harness/frontend_request_authority.py": "sha256:0a061f1528b8bb2071dd1ba7e1d0bbee178b2c982bd6b688b916fb612522e163", + "matched_source:src/aisle/harness/guard_divergence.py": "sha256:af2c4755cabe46fabba913aee4819e05c10caaae79105615d50f8cf4567dad14", + "matched_source:src/aisle/harness/heldout_eval.py": "sha256:56c83164b04ae1b4623e8fd7a0579dd0a9caa7d5dbb891448643464a07893962", + "matched_source:src/aisle/harness/matched_app_server.py": "sha256:b7177caca8f1e4d3664b6cef76539b572c2ea6604fe0799c629bc50a52951b52", + "matched_source:src/aisle/harness/matched_collection.py": "sha256:b347127344a4c15fbb73fa9f3ac7c711a3d55b0fa368a11f76683bb1b5bbea14", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "sha256:b7f2b81397bcd232710408b9c6c9f463b3928e37f117d7310d519728f9b4b2c9", + "matched_source:src/aisle/harness/matched_evidence.py": "sha256:6d65476a9a923fd445ea9dad0334e521378353454b873ef59e2b0ff848c650f7", + "matched_source:src/aisle/harness/matched_frontend.py": "sha256:a4b9827af7dbf36f4af5228ca8c87ea36d2d32f8cffc7fcb8fea1923bcc09401", + "matched_source:src/aisle/harness/matched_run.py": "sha256:825fec659b8bdbb104d6fa07fa2adf29f4b23be74c1f3ec894a8ffcab6b68b64", + "matched_source:src/aisle/harness/matched_run_launch.py": "sha256:8701f034d0bd9873aa7981e3db47849d50bd4465f085e617e6e3193aac070539", + "matched_source:src/aisle/harness/matched_runtime.py": "sha256:c4d81e8309fcf5212315013dbf8e83347a65080e1221f37a0491757834ac2797", + "matched_source:src/aisle/harness/matched_session.py": "sha256:344ea0bc92705ef23f550045f1d4b26d55c7e2cf894133354c22068804887b21", + "matched_source:src/aisle/harness/matched_tool_service.py": "sha256:1bf6685addbeee541961e1053b2eea68cba66ce110667d8e7720398d49191a17", + "matched_source:src/aisle/harness/matched_tools.py": "sha256:eff206d64477d6a91f88c4889e2934a17bba254612c7dc72d542e799e11ef948", + "matched_source:src/aisle/harness/monolith.py": "sha256:a12428eab867b869d902733e43626669b58f87218f0c40d1713712c094ef7414", + "matched_source:src/aisle/harness/monolithic.py": "sha256:1983ed8c26b2371c993c571f33f5c7e4ee81ea2597f9468b43d53c9970465ccf", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "sha256:4c0cb658f0148eb5d06ad69dceafd39148a2d4787e085cb624047c365c87bb24", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "sha256:23f2646571d1ed67c5b8e09e7b0b10bcc6cfa996b05b3161fe39f859d88c15ba", + "matched_source:src/aisle/harness/pilot_records.py": "sha256:68ac7ef9afd449d8527e2ffe99a9ef926e06253c466f3421bc1848e9aa094b70", + "matched_source:src/aisle/harness/provider_relay.py": "sha256:b353c3c40ea03dd63b46264fb60623ae535a7e45aec867261420ba404c9fe19c", + "matched_source:src/aisle/harness/provider_response_authority.py": "sha256:9562a8de28967e2eb83910ea5062dc491c8588a6bb4a2a04407f44d59f2251fe", + "matched_source:src/aisle/harness/provider_runner.py": "sha256:2c39d11d67cc4da2f9a2c732e86ba4e9a4b8da88570bdb1ecd1364d4668831cb", + "matched_source:src/aisle/harness/provider_source_audit.py": "sha256:e3be1f9d70288b4f596427a39ab4274399e9fdc74e87043e3c1249596fe1872b", + "matched_source:src/aisle/harness/registry.py": "sha256:214da7bb8db26c80c79262fa79c40a624d8a9981463d38a9c4aa30bc989bc1fb", + "matched_source:src/aisle/harness/rollout.py": "sha256:371789bd5093ef5c9e5ebb92177de6470caf5ec93ceeb59627411852fa62684a", + "matched_source:src/aisle/harness/rollout_client.py": "sha256:18808db0aa2df356d1a7cc9a538585aa9dcc653abd120e761649f1dff2c27c31", + "matched_source:src/aisle/harness/simulator_work.py": "sha256:2ade87416ab8c5d8902e9734af850b7613ee9ee42f9ca672c9482625e15d3d91", + "matched_source:src/aisle/harness/traces.py": "sha256:c2e6a717a56ce47e296bdc0115b15de9361b72084638b4f3bc66f7e7d032546b", + "matched_source:src/aisle/harness/treatment_ambient.py": "sha256:13dce400a649bc0d01749661517138f8ad1f8cde106caceaedb9f33af6f6d856", + "matched_source:src/aisle/harness/treatment_confinement.py": "sha256:665b50b294d9947c706a6e6c817217600c519e0c375da87464dbc5c4eb4253df", + "matched_source:src/aisle/harness/treatment_integrity.py": "sha256:b37162c6f295c5686da9b3089eb3bd96b69f68f01f41da0a784c4017658b10a6", + "matched_source:src/aisle/harness/treatment_postflight.py": "sha256:0bf01138ef0c277c6fcd728fcf25f4f13795de679fb41ab58b868b81f6c2f0ca", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "sha256:607945d5a51e3a139ecd35a8dceaf77a2027b0f842a577587616e0490b35c663", + "matched_source:src/aisle/harness/typed_graph_audit.py": "sha256:ca315a2717668f6e1029c0940471df6320eb23af859a0bf41220180bbcae656c", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "sha256:0376209def99645318d1476144a6c00e8d5d7e823868ff6033f1d1f572f609d3", + "matched_source:src/aisle/harness/typed_graph_stage.py": "sha256:0ea37ad396fb3321978c5f8f63d869a8ba0a652f7377cad60f13060b5a5ccbfb", + "matched_source:src/aisle/harness/typed_node_host.py": "sha256:56a5f0ab8159071897aad22d49aae2ab1785e6de9a186412b7b5566db334d056", + "matched_source:src/aisle/harness/typed_node_requests.py": "sha256:c3712381bc7383beef1e3304ce05f6a4fe27fb16d9b7e97e7f51f8d7cb6de78b", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "sha256:d964517b767bdd087771bf9233c05de61604cf5f705124d6b83be05feb0cb577", + "matched_source:src/aisle/harness/typed_node_worker.py": "sha256:890cb125e12b05afa3609186ceae6d1d45d1d765f653e2c0b7345b00b2c24a89", + "matched_source:src/aisle/harness/typed_run_prepare.py": "sha256:787ba2edc04efa6df752fdba2112ff50e34d2061acb74471c52c78e813382063", + "matched_source:src/aisle/harness/typed_snapshot.py": "sha256:96003b648f09d1348375ca72709537dd80aa4ddecd60e87465011d221d3200ec", + "matched_source:src/aisle/harness/typed_stage_provider.py": "sha256:afe7e37ef08020f5bf19630ef9117af2bdf9d8fe68ceba7f4a352f7c61c8c9ff", + "matched_source:src/aisle/harness/typed_validation.py": "sha256:bf51adce9f8fe107a2586bb4ffe2aec0cff684a235b6fde5837a95223f37ffa2", + "matched_source:src/aisle/harness/typed_worker_launch.py": "sha256:0759d03e0d26bfa2b38e11771a2ba79a2d7b0f8ca816cd10f084fdfbaa7c93d0", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "sha256:0e594beec5a07829044900d4a93fcde8e2e0e9d50e1e0f7e19809f9d2845b3ba", + "matched_source:src/aisle/harness/validate.py": "sha256:ed82967e800af6ec4d1f5336374343df38a5ba437771dd94e24b06c7dc64f481", + "matched_source:src/aisle/harness/worker_authority_probe.py": "sha256:9ee0638062e062c500c7103664eecd541e5d68056bb83368223c09f145dad6ee", + "matched_source:src/aisle/harness/worker_capability.py": "sha256:272c82452cd6f81912593b4584729cb7bb0e8f528d662cb3021e8dfc6ce70e83", + "matched_source:src/aisle/harness/worker_declaration.py": "sha256:15f42f83cae62a8e917f010b3e1571dda81a3cc7e7e3d630ed2b697fb0e8c0c2", + "matched_source:src/aisle/harness/worker_network_probe.py": "sha256:eb0d75fcbd4e8242eb61701f07e02e83ee6e7d9053247df5572f699969bcceb2", + "matched_source:src/aisle/kinematics.py": "sha256:0656cb4c9c4769365c42fc175deefb69d7a7a3dda77991329b1cedf33f7d5aaa", + "matched_source:src/aisle/mobility/guard.py": "sha256:3d0d4ddd757a8ae1840807511f8816f85feff0f2103fc490a8d29db6fa3663d0", + "matched_source:src/aisle/monolith/confinement.py": "sha256:f207d08bac411ae0cb9a5986b3f5f4d55e55d1407e0c0e6889ec5d489b232969", + "matched_source:src/aisle/monolith/primitive_api.py": "sha256:d5a344125ca3f434e489325b6bc7e20422082d18abee8ba7cd4deccda901327a", + "matched_source:src/aisle/monolith/primitives.py": "sha256:e4b3ef279708ef5fbd6e114f524a6c727f3c9d39a41d56dbfbe73d26e3ac515b", + "matched_source:src/aisle/monolith/proxy.py": "sha256:191b4ce164ea431ccbe728ad9ca5b482a09894093ee389643e099c3774df4ca4", + "matched_source:src/aisle/monolith/requests.py": "sha256:04151bf8838eda75be2a9d0c91240607283fb826712b90a8f99eab5d83d30185", + "matched_source:src/aisle/monolith/supervisor.py": "sha256:7dbe215dc5bddd12d414d1b88e896000fc64a9c71746230518eb2f47f1ef1051", + "matched_source:src/aisle/monolith/wire.py": "sha256:3418587b67e3ce2481e68c2809aaafc8e17f244246d33b055ee05f06ac6177a6", + "matched_source:src/aisle/monolith/worker.py": "sha256:931e42442b9f69157eab17cb73f1fa235eb053da7d843915ef889df5f84fb2d7", + "matched_source:src/aisle/monolith/worker_config.py": "sha256:55a26fdfa8b701470e50d54ffa9384d0ed89a0dc157867ac7e0005671e623312", + "matched_source:src/aisle/monolith/worker_launch.py": "sha256:0e0d053037e1dc9aef27c0248b8adce21c29546ca38ca7ebff6fd0c41c14475b", + "matched_source:src/aisle/nodes/budget_guard.py": "sha256:e15bc5f70b295e7dfd32f72e4a5cf72dffca9da50e9e8f6bb18360fb38069abf", + "matched_source:src/aisle/nodes/h6_fault.py": "sha256:c93b8e1607b7627adef9d971753727b9c602be44ae539def9188344332a63485", + "matched_source:src/aisle/nodes/l2_pose.py": "sha256:d2889dac35af94cdf89a280cf734f672098ce95a4a347e1b9797545c2269fda2", + "matched_source:src/aisle/nodes/label_reader.py": "sha256:aeb51080744779f68fb56d6ff408993e92e3e0b929b7d353af7521537f252517", + "matched_source:src/aisle/nodes/monolith_broker.py": "sha256:6fb86d2fdcae8d9ac0314e4cab3100cd925c35a96f74249006392e042847c048", + "matched_source:src/aisle/nodes/perception_session.py": "sha256:cf0fd80a43a338711456d7db546167d082fb6ecc461a828f3cb0e2d42701fc9e", + "matched_source:src/aisle/scenes/locations.toml": "sha256:baea316611ce2f05751ab3058b4b46acbdd9801d8734faca0272c06b2565bcca", + "matched_source:src/aisle/scenes/meds.toml": "sha256:3b25c5eb6586b311fbe8aa3ffc9fdab76cb1b60c9781c3a7a31b12ab6b61dc6b", + "matched_source:src/aisle/scenes/pharmacy.py": "sha256:642e4a57c9225804b44d4e6909fad6514475598cf002dff300b9ce4455225981", + "matched_source:src/aisle/scenes/physics.toml": "sha256:7a400f5cb01a4a93e2a1daa07fa3b2fafd6598643e15965d779c924480f7c36c", + "matched_source:src/aisle/scenes/planogram.toml": "sha256:c579e207de210ed102ff81e15bf0877a91ce0d4ac31ebce2c31e484322f0e576", + "matched_source:src/aisle/scenes/store.py": "sha256:ea79ce1c91d2085ac920fa1ffe87c0c662c875e0e36da5866c1bc60ae5093a66", + "matched_source:src/aisle/topics.py": "sha256:431a0e0bee4c4c3d83f09f9452eb481f238ec0b8cab11b7370bebc117beeb26f", + "matched_source:src/aisle/turn_node.py": "sha256:0c6cd11cd802ebf8934a671caeb4b8a590e5f43b23f03c52515f51eac4288705", + "matched_source:src/aisle/turns.py": "sha256:f0014e3d8ea6b07d4effd10c5bf0b7de08b89ecec1e164580509ad25694b36a4", + "matched_source:src/aisle/verifier/calibration.py": "sha256:a00b91406d27a3b6123701a7a7270258633ad9906c28dbb6f1110fbb07b59568", + "matched_source:src/aisle/verifier/models.lock": "sha256:3fa7271740f3ecd361213f45de140452f41216b80600e343e2daa9f2a413eab5", + "matched_source:src/aisle/verifier/models.py": "sha256:1b2905181bc757f8390666c2a4c7303f674a6d8a31232e8feae582b5cc24909f", + "matched_source:src/aisle/verifier/placement.toml": "sha256:8d8f37165cf4bb4214486597689d40d5df8fb36e2bb8f15e65f6be8801d3717f", + "matched_source:src/aisle/verifier/realistic.py": "sha256:769f6295d3c15a5be02da8ec815a401728059608f2ae92c904d5bec0dd18d156", + "matched_source:src/aisle/verifier/stages.py": "sha256:63848f9e4c0e59020461e3350bf8ca2a8d0ae93cc45cd43c5786c7b6f95a443c", + "matched_source:src/aisle/verifier/thresholds.toml": "sha256:567ac0544a4c382d3ce6547d8df868e6ca37c056b146cf0d1ee11934b1be16d3", + "matched_source:tools/campaign.py": "sha256:cb33ab83f9125ea9708a73f639ca38ad539fbc77e6c5b03e0b49dc2e66abd8f0", + "matched_source:tools/matched_campaign.py": "sha256:71e648035529877ee68b26eb2a4ccf5c0126e7872e3e9f6134785a1b63554609", + "monolithic_validators": "sha256:1983ed8c26b2371c993c571f33f5c7e4ee81ea2597f9468b43d53c9970465ccf", + "nested_host_documentation": "sha256:1c4777e4d8a29882ae5332f96df6b175423e7e5d1f279357fdb1286bace12959", + "pilot_assignments_cli": "sha256:bafcfdf4f79d5d0abd7442355454ea7ca476928afd57ed7c83f38da0c3726cc8", + "pilot_decision": "sha256:8a2f932b7ff6e87c833baa0fd85f2e5d892bed4bd7287e05488eaa8652a85966", + "pilot_records_cli": "sha256:51438c6e14fca6368e1b91ebce77fd13c6bcc821edb61666553750a91d3ab75f", + "protocol": "sha256:bc428999047c944a58b079f55ecce10c0e1008f11924f58838a3d36b045f31f5", + "protocol_template": "sha256:9d636f3dc462a2e051ee0b3ded565aff1c5be86331e5b1b6692988509ccf088e", + "provider_admission_documentation": "sha256:856851935d15d8cb0c5653548ae9312441263cce9a3300e36d0863f5bdb96cf4", + "research_prompt": "sha256:e736bbd6caa7236dbfafe92ae42c0c7f13088764af1b753f6685d398c111b7c0", + "seed_commitment_predecessor": "sha256:be280f2fc8f06277b2d7d604108a3f0c8c53334081cc6d41b73523e790ef6f27", + "session_runner": "sha256:cb33ab83f9125ea9708a73f639ca38ad539fbc77e6c5b03e0b49dc2e66abd8f0", + "session_validators": "sha256:1b1c53d6389881c146726d865a741a681f3d5039c9bf99558a53ee368250971e", + "simulator_bridge": "sha256:196837073251bedefd0365d220aa94407612f7670ed0b1b39c7319d63a1c18f8", + "spec": "sha256:f5096c587cd9687859f2dc80ec807d1bbf959cb4e05c152154e71e47bdc1f593", + "treatment_integrity": "sha256:b37162c6f295c5686da9b3089eb3bd96b69f68f01f41da0a784c4017658b10a6", + "treatment_postflight": "sha256:0bf01138ef0c277c6fcd728fcf25f4f13795de679fb41ab58b868b81f6c2f0ca", + "treatment_table": "sha256:0f860aefee6272ae5ca92f0a6d704d9f2dd924d931eed97d0b666f09361217d8" + }, + "campaign_id": "cse-causal-study-pilot-v9", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-pilot-v9/protocol.json --purpose power", + "uv run harness stats power --protocol analysis/freeze/cse-causal-study-pilot-v9/protocol.json --output analysis/freeze/cse-causal-study-pilot-v9/power.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json --allow-withheld-seeds" + ], + "declaration": { + "analysis": { + "scripts": [ + "src/aisle/harness/benchmark_statistics.py", + "src/aisle/harness/causal_study.py", + "src/aisle/harness/pilot_records.py" + ], + "seed": 500001 + }, + "artifacts": { + "agent_adapters": "tools/agent_adapters.py", + "analyzer": "src/aisle/harness/benchmark_statistics.py", + "bnd_calibration_manifest": "analysis/freeze/bnd-task-band-calibration-v11/freeze-manifest.json", + "bnd_perception_audit_report": "analysis/perception-audit/records/bnd-perception-corpus-03/report.json", + "budget_ceiling": "harness/budget.toml", + "dependency_declaration": "pyproject.toml", + "dependency_lock": "uv.lock", + "dora_runtime_selection": "dora-runtime.json", + "frontend_request_authorization_documentation": "docs/benchmark/v1/frontend-request-authorization.md", + "matched_session_decision": "docs/decisions/ADR-62.md", + "matched_session_documentation": "docs/monolithic/matched-session.md", + "matched_source:assets/so101/so101.urdf": "assets/so101/so101.urdf", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "docs/monolithic/allowlist-t1-l2.json", + "matched_source:docs/monolithic/candidates.json": "docs/monolithic/candidates.json", + "matched_source:docs/monolithic/experts-t1-l2.json": "docs/monolithic/experts-t1-l2.json", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "docs/monolithic/interface-map-t1-l2.json", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "docs/monolithic/parity-protocol-t1-l2.json", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "docs/monolithic/treatment-table-t1-l2.json", + "matched_source:env/limits.toml": "env/limits.toml", + "matched_source:experts/monolithic/expert_t1_l2.py": "experts/monolithic/expert_t1_l2.py", + "matched_source:graphs/monolithic_t1_l2.yaml": "graphs/monolithic_t1_l2.yaml", + "matched_source:src/aisle/__init__.py": "src/aisle/__init__.py", + "matched_source:src/aisle/embodiment.py": "src/aisle/embodiment.py", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "src/aisle/harness/_code_mode_protocol/__init__.py", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "src/aisle/harness/_code_mode_protocol/descriptor.pb", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "src/aisle/harness/_code_mode_protocol/protocol.proto", + "matched_source:src/aisle/harness/candidate_launch.py": "src/aisle/harness/candidate_launch.py", + "matched_source:src/aisle/harness/candidates.py": "src/aisle/harness/candidates.py", + "matched_source:src/aisle/harness/cli.py": "src/aisle/harness/cli.py", + "matched_source:src/aisle/harness/code_mode_audit.py": "src/aisle/harness/code_mode_audit.py", + "matched_source:src/aisle/harness/code_mode_authority.py": "src/aisle/harness/code_mode_authority.py", + "matched_source:src/aisle/harness/code_mode_proxy.py": "src/aisle/harness/code_mode_proxy.py", + "matched_source:src/aisle/harness/code_mode_runner.py": "src/aisle/harness/code_mode_runner.py", + "matched_source:src/aisle/harness/common.py": "src/aisle/harness/common.py", + "matched_source:src/aisle/harness/frontend_app_server.py": "src/aisle/harness/frontend_app_server.py", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "src/aisle/harness/frontend_app_server_audit.py", + "matched_source:src/aisle/harness/frontend_dispatch.py": "src/aisle/harness/frontend_dispatch.py", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "src/aisle/harness/frontend_dispatch_audit.py", + "matched_source:src/aisle/harness/frontend_request_audit.py": "src/aisle/harness/frontend_request_audit.py", + "matched_source:src/aisle/harness/frontend_request_authority.py": "src/aisle/harness/frontend_request_authority.py", + "matched_source:src/aisle/harness/guard_divergence.py": "src/aisle/harness/guard_divergence.py", + "matched_source:src/aisle/harness/heldout_eval.py": "src/aisle/harness/heldout_eval.py", + "matched_source:src/aisle/harness/matched_app_server.py": "src/aisle/harness/matched_app_server.py", + "matched_source:src/aisle/harness/matched_collection.py": "src/aisle/harness/matched_collection.py", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "src/aisle/harness/matched_dynamic_run.py", + "matched_source:src/aisle/harness/matched_evidence.py": "src/aisle/harness/matched_evidence.py", + "matched_source:src/aisle/harness/matched_frontend.py": "src/aisle/harness/matched_frontend.py", + "matched_source:src/aisle/harness/matched_run.py": "src/aisle/harness/matched_run.py", + "matched_source:src/aisle/harness/matched_run_launch.py": "src/aisle/harness/matched_run_launch.py", + "matched_source:src/aisle/harness/matched_runtime.py": "src/aisle/harness/matched_runtime.py", + "matched_source:src/aisle/harness/matched_session.py": "src/aisle/harness/matched_session.py", + "matched_source:src/aisle/harness/matched_tool_service.py": "src/aisle/harness/matched_tool_service.py", + "matched_source:src/aisle/harness/matched_tools.py": "src/aisle/harness/matched_tools.py", + "matched_source:src/aisle/harness/monolith.py": "src/aisle/harness/monolith.py", + "matched_source:src/aisle/harness/monolithic.py": "src/aisle/harness/monolithic.py", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "src/aisle/harness/monolithic_run_evidence.py", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "src/aisle/harness/monolithic_run_prepare.py", + "matched_source:src/aisle/harness/pilot_records.py": "src/aisle/harness/pilot_records.py", + "matched_source:src/aisle/harness/provider_relay.py": "src/aisle/harness/provider_relay.py", + "matched_source:src/aisle/harness/provider_response_authority.py": "src/aisle/harness/provider_response_authority.py", + "matched_source:src/aisle/harness/provider_runner.py": "src/aisle/harness/provider_runner.py", + "matched_source:src/aisle/harness/provider_source_audit.py": "src/aisle/harness/provider_source_audit.py", + "matched_source:src/aisle/harness/registry.py": "src/aisle/harness/registry.py", + "matched_source:src/aisle/harness/rollout.py": "src/aisle/harness/rollout.py", + "matched_source:src/aisle/harness/rollout_client.py": "src/aisle/harness/rollout_client.py", + "matched_source:src/aisle/harness/simulator_work.py": "src/aisle/harness/simulator_work.py", + "matched_source:src/aisle/harness/traces.py": "src/aisle/harness/traces.py", + "matched_source:src/aisle/harness/treatment_ambient.py": "src/aisle/harness/treatment_ambient.py", + "matched_source:src/aisle/harness/treatment_confinement.py": "src/aisle/harness/treatment_confinement.py", + "matched_source:src/aisle/harness/treatment_integrity.py": "src/aisle/harness/treatment_integrity.py", + "matched_source:src/aisle/harness/treatment_postflight.py": "src/aisle/harness/treatment_postflight.py", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "src/aisle/harness/typed_execution_bundle.py", + "matched_source:src/aisle/harness/typed_graph_audit.py": "src/aisle/harness/typed_graph_audit.py", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "src/aisle/harness/typed_graph_hosts.py", + "matched_source:src/aisle/harness/typed_graph_stage.py": "src/aisle/harness/typed_graph_stage.py", + "matched_source:src/aisle/harness/typed_node_host.py": "src/aisle/harness/typed_node_host.py", + "matched_source:src/aisle/harness/typed_node_requests.py": "src/aisle/harness/typed_node_requests.py", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "src/aisle/harness/typed_node_supervisor.py", + "matched_source:src/aisle/harness/typed_node_worker.py": "src/aisle/harness/typed_node_worker.py", + "matched_source:src/aisle/harness/typed_run_prepare.py": "src/aisle/harness/typed_run_prepare.py", + "matched_source:src/aisle/harness/typed_snapshot.py": "src/aisle/harness/typed_snapshot.py", + "matched_source:src/aisle/harness/typed_stage_provider.py": "src/aisle/harness/typed_stage_provider.py", + "matched_source:src/aisle/harness/typed_validation.py": "src/aisle/harness/typed_validation.py", + "matched_source:src/aisle/harness/typed_worker_launch.py": "src/aisle/harness/typed_worker_launch.py", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "src/aisle/harness/typed_worker_provisioning.py", + "matched_source:src/aisle/harness/validate.py": "src/aisle/harness/validate.py", + "matched_source:src/aisle/harness/worker_authority_probe.py": "src/aisle/harness/worker_authority_probe.py", + "matched_source:src/aisle/harness/worker_capability.py": "src/aisle/harness/worker_capability.py", + "matched_source:src/aisle/harness/worker_declaration.py": "src/aisle/harness/worker_declaration.py", + "matched_source:src/aisle/harness/worker_network_probe.py": "src/aisle/harness/worker_network_probe.py", + "matched_source:src/aisle/kinematics.py": "src/aisle/kinematics.py", + "matched_source:src/aisle/mobility/guard.py": "src/aisle/mobility/guard.py", + "matched_source:src/aisle/monolith/confinement.py": "src/aisle/monolith/confinement.py", + "matched_source:src/aisle/monolith/primitive_api.py": "src/aisle/monolith/primitive_api.py", + "matched_source:src/aisle/monolith/primitives.py": "src/aisle/monolith/primitives.py", + "matched_source:src/aisle/monolith/proxy.py": "src/aisle/monolith/proxy.py", + "matched_source:src/aisle/monolith/requests.py": "src/aisle/monolith/requests.py", + "matched_source:src/aisle/monolith/supervisor.py": "src/aisle/monolith/supervisor.py", + "matched_source:src/aisle/monolith/wire.py": "src/aisle/monolith/wire.py", + "matched_source:src/aisle/monolith/worker.py": "src/aisle/monolith/worker.py", + "matched_source:src/aisle/monolith/worker_config.py": "src/aisle/monolith/worker_config.py", + "matched_source:src/aisle/monolith/worker_launch.py": "src/aisle/monolith/worker_launch.py", + "matched_source:src/aisle/nodes/budget_guard.py": "src/aisle/nodes/budget_guard.py", + "matched_source:src/aisle/nodes/h6_fault.py": "src/aisle/nodes/h6_fault.py", + "matched_source:src/aisle/nodes/l2_pose.py": "src/aisle/nodes/l2_pose.py", + "matched_source:src/aisle/nodes/label_reader.py": "src/aisle/nodes/label_reader.py", + "matched_source:src/aisle/nodes/monolith_broker.py": "src/aisle/nodes/monolith_broker.py", + "matched_source:src/aisle/nodes/perception_session.py": "src/aisle/nodes/perception_session.py", + "matched_source:src/aisle/scenes/locations.toml": "src/aisle/scenes/locations.toml", + "matched_source:src/aisle/scenes/meds.toml": "src/aisle/scenes/meds.toml", + "matched_source:src/aisle/scenes/pharmacy.py": "src/aisle/scenes/pharmacy.py", + "matched_source:src/aisle/scenes/physics.toml": "src/aisle/scenes/physics.toml", + "matched_source:src/aisle/scenes/planogram.toml": "src/aisle/scenes/planogram.toml", + "matched_source:src/aisle/scenes/store.py": "src/aisle/scenes/store.py", + "matched_source:src/aisle/topics.py": "src/aisle/topics.py", + "matched_source:src/aisle/turn_node.py": "src/aisle/turn_node.py", + "matched_source:src/aisle/turns.py": "src/aisle/turns.py", + "matched_source:src/aisle/verifier/calibration.py": "src/aisle/verifier/calibration.py", + "matched_source:src/aisle/verifier/models.lock": "src/aisle/verifier/models.lock", + "matched_source:src/aisle/verifier/models.py": "src/aisle/verifier/models.py", + "matched_source:src/aisle/verifier/placement.toml": "src/aisle/verifier/placement.toml", + "matched_source:src/aisle/verifier/realistic.py": "src/aisle/verifier/realistic.py", + "matched_source:src/aisle/verifier/stages.py": "src/aisle/verifier/stages.py", + "matched_source:src/aisle/verifier/thresholds.toml": "src/aisle/verifier/thresholds.toml", + "matched_source:tools/campaign.py": "tools/campaign.py", + "matched_source:tools/matched_campaign.py": "tools/matched_campaign.py", + "monolithic_validators": "src/aisle/harness/monolithic.py", + "nested_host_documentation": "docs/monolithic/code-mode-admission.md", + "pilot_assignments_cli": "tools/pilot_assignments.py", + "pilot_decision": "docs/decisions/ADR-66.md", + "pilot_records_cli": "tools/pilot_records.py", + "protocol": "analysis/freeze/cse-causal-study-pilot-v9/protocol.json", + "protocol_template": "analysis/statistics/protocol-template.json", + "provider_admission_documentation": "docs/monolithic/provider-admission.md", + "research_prompt": "harness/CLAUDE.research.md", + "seed_commitment_predecessor": "analysis/freeze/cse-causal-study-pilot-v8/freeze-manifest.json", + "session_runner": "tools/campaign.py", + "session_validators": "src/aisle/harness/causal_study.py", + "simulator_bridge": "src/aisle/nodes/dora_genesis.py", + "spec": "specs/500-typed-monolithic-causal-study.md", + "treatment_integrity": "src/aisle/harness/treatment_integrity.py", + "treatment_postflight": "src/aisle/harness/treatment_postflight.py", + "treatment_table": "analysis/monolithic-control/treatment-table-v1.json" + }, + "budgets": { + "campaign_ceiling": "harness/budget.toml", + "rollouts_per_session": 20, + "session_tokens": 200000, + "session_wall_s": 3600, + "sim_episodes_per_session": 160 + }, + "campaign_id": "cse-causal-study-pilot-v9", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-pilot-v9/protocol.json --purpose power", + "uv run harness stats power --protocol analysis/freeze/cse-causal-study-pilot-v9/protocol.json --output analysis/freeze/cse-causal-study-pilot-v9/power.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json --allow-withheld-seeds" + ], + "decision_rules": { + "alpha": 0.05, + "equivalence_margin": { + "cost_at_equal_quality": { + "acceptance_probability": 0.1, + "heldout_quality": 0.1, + "rule": "CSE-5: both must pass or cost claims stay unconditional", + "sidedness": "two-sided 0.90" + } + }, + "multiplicity": "single primary endpoint", + "power": 0.8, + "primary_rule": "one-sided superiority at alpha 0.05 on the pooled Newcombe interval lower bound > 0 with no reversed stratum; otherwise weakened/rejected/undecidable by the #358 rule (CSE-3)", + "smallest_effect": { + "measure": "risk difference in session success", + "value": 0.25 + }, + "stopping_rule": "fixed n from the frozen power analysis using pilot-only control success; absolute floor 10 randomized assignments per arm per agent-system x task stratum (CSE-8); no optional stopping or extension" + }, + "disposition": "registered 2026-09-13; collection pending. Supersedes the zero-session pilot-v1 lineage with the instrument set the pilot can actually run on (T1-L2 realistic pair, #581 to #586). Named limitations: BND-7 not eligible (recorded failed), external-review and parity gates pending, and the session-bound capability audit not merged (an unattested environment stops every session).", + "endpoints": { + "primary": [ + { + "aggregation": "risk difference typed_dataflow - monolithic; exact arm intervals; Newcombe interval; per agent-system x task stratum before pooling (CSE-1, CSE-17)", + "direction": "higher", + "id": "E-CSE-1", + "outcome": "session_success", + "status": "inferential", + "unit": "agent_session" + } + ], + "secondary": [ + { + "aggregation": "counts by arm", + "direction": "descriptive", + "id": "E-CSE-2", + "outcome": "validation_and_launch_failure_class", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "Kaplan-Meier with censoring", + "direction": "lower", + "id": "E-CSE-3", + "outcome": "time_to_first_accepted_system", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "unconditional session estimands; no at-equal-quality wording unless CSE-5 equivalence passes", + "direction": "lower", + "id": "E-CSE-4", + "outcome": "tokens_tool_calls_wall_active_time_sim_work_rollouts", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "counts", + "direction": "descriptive", + "id": "E-CSE-5", + "outcome": "safety_interventions", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "dropped unless a blinded rubric is ratified before freeze (CSE-18)", + "direction": "n/a", + "id": "E-CSE-6", + "outcome": "artifact_legibility", + "status": "exploratory", + "unit": "agent_session" + } + ] + }, + "exclusions": { + "deviation_policy": "any behaviour-affecting change after registration is a documented deviation record and a new campaign version id; confirmatory data never revises the registered protocol", + "infrastructure": [ + "independent-controller-determined, arm-neutral facts only (CSE-14): host outage, provider interruption before first tool call, launcher crash before the session started" + ], + "rerun_policy": "a permitted rerun receives a new randomized id; the original row is retained with its exclusion reason and never replaced in place", + "treatment_integrity": [ + "contamination sentinel tripped (CSE-12)", + "model alias or CLI drift outside the frozen identity (CSE-6)", + "integrity verdict authored session-side (CSE-11)" + ] + }, + "hypotheses": [ + { + "direction": "typed_dataflow - monolithic > 0", + "id": "H-CSE-1", + "statement": "The bundled typed-dataflow surface raises the probability that a fresh coding-agent session produces a launching system passing the hidden held-out acceptance within budget, versus the equal-capability monolithic surface." + } + ], + "instrument_set": { + "agent_systems": [ + { + "binary": "codex", + "binary_sha256": "b973d440acac501fd2594a43e7ca9ce41e0a65b9dfb28d0d7a7837c99e1261e3", + "frozen": true, + "id": "codex-cli", + "install": "private prefix ~/aisle-private/codex-0.153.4 (the qualified darwin-arm64 binary, not the JS shim); ChatGPT-login credentials copied byte-identically into both arms' private CODEX_HOME, hash recorded only", + "version_observed": "0.153.4" + } + ], + "candidate": { + "id": "t1-l2-realistic", + "monolithic_module": "experts/monolithic/expert_t1_l2.py", + "monolithic_template": "graphs/monolithic_t1_l2.yaml", + "perception": "L2", + "reset": "teleport", + "typed_graph": "graphs/expert_t1_l2.yaml", + "verifier": "realistic (oracle demoted to the held-out scoring tap, BND-3)" + }, + "candidate_table_sha256": "08a1dc56502abc4673cb0b54f01664382214bb473c3ef293b1df7b6c74f64778", + "items": [ + "short_composition:t1-l2-realistic" + ], + "kind": "selected_task" + }, + "integrity_checks": [ + { + "gate": "CON-14 spec approval", + "kind": "external_review", + "owner_role": "project owner", + "pilot_record_2026_09_13": "pending; external review, not a pilot prerequisite (ADR-66 section 2); named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "STA-12 independent statistical review", + "kind": "external_review", + "owner_role": "independent statistician", + "pilot_record_2026_09_13": "pending; external review, not a pilot prerequisite (ADR-66 section 2); named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "MON-1 treatment table and expert parity (#344)", + "kind": "machine_check", + "owner_role": "parity harness; control surface exists, independent authorship and operator evidence remain pending", + "pilot_record_2026_09_13": "pending; the T1-L2 pair (#583) passes the interface, table and module checks but has no operator parity record at an immutable hash; named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "BND-12 task-band freeze (#346)", + "kind": "machine_check", + "owner_role": "task-band auditor", + "pilot_record_2026_09_13": "recorded failed: bnd-task-band-calibration-v11 regenerated the BND-5 perception audit on a 32-seed T1-L2 corpus; 22 of 22 strata below the 0.90 floor (BND-7), led by target_class=ibuprofen refusals; role open (BND-10). ADR-66 section 2: a failing machine gate is a named limitation of this pilot, not a blocker", + "record": "analysis/perception-audit/records/bnd-perception-corpus-03/report.json", + "status": "failed" + }, + { + "gate": "TRT confinement/mutation/agent-path (#353)", + "kind": "machine_check", + "owner_role": "integrity controller", + "pilot_record_2026_09_13": "pending; the session-bound capability audit (loopback relay policy) is not merged; an unattested environment is a stop condition for every session", + "record": null, + "status": "pending" + }, + { + "gate": "seed commitment verification", + "kind": "machine_check", + "owner_role": "holder of the original private seed and salt sources", + "pilot_record_2026_09_13": "not applicable to this registration: it commits a fresh private seed source directly (no inheritance); kept for shape parity and left pending", + "record": null, + "status": "pending" + }, + { + "gate": "MON-8/MON-12 matched-session acceptance (#519)", + "kind": "machine_check", + "owner_role": "matched-session controller; complete acceptance evidence still required", + "pilot_record_2026_09_13": "pending; frontend conformance qualified on the T1 oracle development surface only (#568); the T1-L2 realistic surface runs under the same executor with the same admission checks; named limitation", + "record": null, + "status": "pending" + } + ], + "issue": 347, + "pilot_design": { + "agent_system": "codex-cli 0.153.4 only (the owned provider relay, the ChatGPT-login account header and the pinned conformance profile are Codex-bound)", + "arm_order": "sealed balanced plan (revealed one assignment at a time); no longer a fixed alternation", + "arms": [ + "typed", + "monolithic" + ], + "assignment": "tools/pilot_assignments.py create --protocol analysis/freeze/cse-causal-study-pilot-v8/protocol.json (sealed balanced plan, one typed and one monolithic assignment per temporal block, TRT-8/CSE-7); reveal exactly one assignment before each session; the ledger retains every assignment, started or not", + "below_cse8_floor": "6 per arm is below the CSE-8 floor of 10 by design; this registration can never become a confirmatory estimate (STA-3)", + "decision": "docs/decisions/ADR-66.md (pilot first; agent-drafted under the ADR-65 label, renumbered by #568)", + "executor": "tools/matched_campaign.py admit / run --purpose engineering / collect-run, then the held-out evaluation; retained and unscored, output root ~/aisle-private/raw/cse-causal-study-pilot-v8/", + "records": "tools/pilot_records.py derives one aisle.stats.records.v1 row per ledger assignment; harness stats validate --purpose analysis; harness stats analyze; harness stats power from observed control success (CSE-8)", + "reporting": "analysis/reports/cse_pilot_v8.md; results enter docs/claim-evidence.yaml only as CSE-8 power inputs under the unrun confirmatory claim", + "rerun_policy": "no reruns; a failed attempt is retained under a new session id, never overwritten", + "session_success": "controller-run held-out evaluation of the retained final snapshot on the 32 private seeds after the agent process ends, oracle as the held-out scorer (BND-3); success iff at least 28 of 32 oracle successes; a launch failure counts as a failed seed (CSE-2, CSE-9)", + "sessions_per_arm": 6, + "stop_conditions": [ + "wrong_object outcome", + "guard bypass", + "unattested environment", + "relay bypass (provider traffic outside the owned relay)", + "any oracle field on a policy input" + ], + "task_stratum": "short_composition on candidate t1-l2-realistic (T1 franka, L2 perception, realistic verifier); bnd-task-band-calibration-v11 records it as not perception-eligible (BND-7), carried as a machine-gate limitation per ADR-66" + }, + "purpose": "pilot", + "schema_version": "aisle.freeze.declaration.v1", + "seed_commitment": { + "inherited_from": "analysis/freeze/cse-causal-study-pilot-v8/freeze-manifest.json", + "rule": "32 distinct integers in [1000,9999) drawn once from the OS CSPRNG on 2026-09-13, retained only under ~/aisle-private/freeze/cse-causal-study-pilot-v8/, disjoint by value from every private seed list committed so far (including cse-causal-study-pilot-v1 and the cse-causal-study-v1 confirmatory commitment) and from the committed development seed lists; never inherited by a confirmatory declaration (ADR-66); revealed after the pilot closes. The same directory holds the controller-private 256-bit randomization seed for the sealed assignment plan (TRT-8); its commitment is recorded in the assignment ledger, never here.", + "salt_source": "~/aisle-private/freeze/cse-causal-study-pilot-v8/salt.bin", + "seeds_source": "~/aisle-private/freeze/cse-causal-study-pilot-v8/seeds.json" + }, + "spec": "500", + "superseded": "cse-causal-study-pilot-v8 (session-bound confinement attestation: loopback network policy for the provider relay and the capability audit run under the session's exact policy (TRT-5/TRT-7); seed commitment, collection rules and all pending gates preserved)" + }, + "declaration_sha256": "sha256:b15c524ed8cee3e8214e89310a106c65c2664be44883687dcd6115ebfcd09a94", + "frozen": false, + "gate_record_hashes": {}, + "git_head": "c9876f4a8985214ba4a9441ecf2bdea4209c3035", + "issue": 347, + "pending_gates": [ + "BND-12 task-band freeze (#346)", + "CON-14 spec approval", + "MON-1 treatment table and expert parity (#344)", + "MON-8/MON-12 matched-session acceptance (#519)", + "STA-12 independent statistical review", + "TRT confinement/mutation/agent-path (#353)", + "seed commitment verification" + ], + "purpose": "pilot", + "schema_version": "aisle.freeze.manifest.v1", + "seed_commitment": "sha256:bc4a2c141eded05f77bee114330c2f9d1c5112e26618f2feb9694cf60dd56143", + "spec": "500", + "status": "registered_pending_review", + "timestamp": null, + "timestamp_source": null +} diff --git a/analysis/freeze/cse-causal-study-pilot-v9/protocol.json b/analysis/freeze/cse-causal-study-pilot-v9/protocol.json new file mode 100644 index 00000000..a3b6af63 --- /dev/null +++ b/analysis/freeze/cse-causal-study-pilot-v9/protocol.json @@ -0,0 +1,91 @@ +{ + "allocation_ratio": 1.0, + "alpha": 0.05, + "analysis_seed": 500001, + "artifact_claim_threshold": 0.9, + "bootstrap": { + "confidence_level": 0.95, + "replicates": 10000 + }, + "campaign_id": "cse-causal-study-pilot-v9", + "campaign_phase": "pilot", + "control_arm": "monolithic", + "decision": { + "kind": "superiority", + "margin": null, + "rationale": "equivalence and non-inferiority are not the primary decision for this template" + }, + "exclusion_rules": [ + "exclude only a predeclared infrastructure failure independent of treatment outcome", + "retain every exclusion row and reason in the raw records", + "no reruns: a failed attempt is retained under a new session id and never overwritten" + ], + "experimental_unit": "session", + "inclusion_rules": [ + "retain and classify every randomized assignment", + "include a started session unless a predeclared infrastructure rule applies", + "a session's outcome is only its retained held-out evidence (28 of 32 oracle successes); an agent process that exits nonzero within budget is a failed session, not an exclusion" + ], + "outcome": { + "direction": "higher", + "field": "session_success", + "rule": { + "kind": "heldout_oracle_successes", + "n_seeds": 32, + "source": "heldout/heldout-evidence.json per session (controller-run, oracle as the held-out scorer)", + "threshold": 28 + }, + "type": "binary" + }, + "power": { + "assumption_note": "control_probability 0.40 is the pre-registration ASSUMPTION carried from cse-causal-study-v1; this pilot protocol exists to REPLACE it with pilot-only control success (CSE-8). Pilot rows are calibration inputs only and never enter a confirmatory estimate (STA-3). 6 sessions per arm is below the CSE-8 floor of 10 by design (ADR-66).", + "control_probability": 0.4, + "sensitivity_effects": [ + 0.15, + 0.2, + 0.25, + 0.3 + ], + "test_sidedness": "one-sided", + "treatment_probability": 0.65 + }, + "primary_estimand": { + "contrast": "typed_dataflow minus monolithic", + "population": "autonomous coding-agent sessions on the frozen task band", + "summary": "risk_difference" + }, + "protocol_id": "cse-causal-study-pilot-v9-protocol-r0", + "randomization": { + "plan": "sealed balanced plan over temporal blocks (tools/pilot_assignments.py); one typed_dataflow and one monolithic assignment per block", + "seed": "controller-private 256-bit seed under ~/aisle-private/freeze/cse-causal-study-pilot-v8/ (never written into any artifact); the ledger records its commitment" + }, + "schema_version": "aisle.stats.protocol.v1", + "smallest_effect": { + "measure": "risk_difference", + "value": 0.25 + }, + "stopping_rule": { + "max_sessions_per_arm": 6, + "type": "fixed" + }, + "strata": [ + "agent_system", + "task" + ], + "survival": { + "censor_time_field": "wall_s", + "event_time_field": "accepted_time_s" + }, + "target_power": 0.8, + "treatment_arm": "typed_dataflow", + "treatment_arms": [ + "monolithic", + "typed_dataflow" + ], + "zero_event": { + "confidence_level": 0.95, + "event_field": "safety_events", + "exposure_field": "commands", + "exposure_unit": "command" + } +} diff --git a/analysis/freeze/cse-causal-study-v23/declaration.json b/analysis/freeze/cse-causal-study-v23/declaration.json new file mode 100644 index 00000000..dce6cd6e --- /dev/null +++ b/analysis/freeze/cse-causal-study-v23/declaration.json @@ -0,0 +1,348 @@ +{ + "analysis": { + "scripts": [ + "src/aisle/harness/benchmark_statistics.py", + "src/aisle/harness/causal_study.py", + "src/aisle/harness/pilot_records.py" + ], + "seed": 500001 + }, + "artifacts": { + "agent_adapters": "tools/agent_adapters.py", + "analyzer": "src/aisle/harness/benchmark_statistics.py", + "budget_ceiling": "harness/budget.toml", + "dependency_declaration": "pyproject.toml", + "dependency_lock": "uv.lock", + "dora_runtime_selection": "dora-runtime.json", + "frontend_conformance_decision": "docs/decisions/ADR-65.md", + "frontend_request_authorization_documentation": "docs/benchmark/v1/frontend-request-authorization.md", + "matched_session_decision": "docs/decisions/ADR-62.md", + "matched_session_documentation": "docs/monolithic/matched-session.md", + "matched_source:assets/so101/so101.urdf": "assets/so101/so101.urdf", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "docs/monolithic/allowlist-t1-l2.json", + "matched_source:docs/monolithic/candidates.json": "docs/monolithic/candidates.json", + "matched_source:docs/monolithic/experts-t1-l2.json": "docs/monolithic/experts-t1-l2.json", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "docs/monolithic/interface-map-t1-l2.json", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "docs/monolithic/parity-protocol-t1-l2.json", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "docs/monolithic/treatment-table-t1-l2.json", + "matched_source:env/limits.toml": "env/limits.toml", + "matched_source:experts/monolithic/expert_t1_l2.py": "experts/monolithic/expert_t1_l2.py", + "matched_source:graphs/monolithic_t1_l2.yaml": "graphs/monolithic_t1_l2.yaml", + "matched_source:src/aisle/__init__.py": "src/aisle/__init__.py", + "matched_source:src/aisle/embodiment.py": "src/aisle/embodiment.py", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "src/aisle/harness/_code_mode_protocol/__init__.py", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "src/aisle/harness/_code_mode_protocol/descriptor.pb", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "src/aisle/harness/_code_mode_protocol/protocol.proto", + "matched_source:src/aisle/harness/candidate_launch.py": "src/aisle/harness/candidate_launch.py", + "matched_source:src/aisle/harness/candidates.py": "src/aisle/harness/candidates.py", + "matched_source:src/aisle/harness/cli.py": "src/aisle/harness/cli.py", + "matched_source:src/aisle/harness/code_mode_audit.py": "src/aisle/harness/code_mode_audit.py", + "matched_source:src/aisle/harness/code_mode_authority.py": "src/aisle/harness/code_mode_authority.py", + "matched_source:src/aisle/harness/code_mode_proxy.py": "src/aisle/harness/code_mode_proxy.py", + "matched_source:src/aisle/harness/code_mode_runner.py": "src/aisle/harness/code_mode_runner.py", + "matched_source:src/aisle/harness/common.py": "src/aisle/harness/common.py", + "matched_source:src/aisle/harness/frontend_app_server.py": "src/aisle/harness/frontend_app_server.py", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "src/aisle/harness/frontend_app_server_audit.py", + "matched_source:src/aisle/harness/frontend_conformance.py": "src/aisle/harness/frontend_conformance.py", + "matched_source:src/aisle/harness/frontend_dispatch.py": "src/aisle/harness/frontend_dispatch.py", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "src/aisle/harness/frontend_dispatch_audit.py", + "matched_source:src/aisle/harness/frontend_effects.py": "src/aisle/harness/frontend_effects.py", + "matched_source:src/aisle/harness/frontend_qualification.py": "src/aisle/harness/frontend_qualification.py", + "matched_source:src/aisle/harness/frontend_request_audit.py": "src/aisle/harness/frontend_request_audit.py", + "matched_source:src/aisle/harness/frontend_request_authority.py": "src/aisle/harness/frontend_request_authority.py", + "matched_source:src/aisle/harness/guard_divergence.py": "src/aisle/harness/guard_divergence.py", + "matched_source:src/aisle/harness/heldout_eval.py": "src/aisle/harness/heldout_eval.py", + "matched_source:src/aisle/harness/matched_app_server.py": "src/aisle/harness/matched_app_server.py", + "matched_source:src/aisle/harness/matched_collection.py": "src/aisle/harness/matched_collection.py", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "src/aisle/harness/matched_dynamic_run.py", + "matched_source:src/aisle/harness/matched_evidence.py": "src/aisle/harness/matched_evidence.py", + "matched_source:src/aisle/harness/matched_frontend.py": "src/aisle/harness/matched_frontend.py", + "matched_source:src/aisle/harness/matched_run.py": "src/aisle/harness/matched_run.py", + "matched_source:src/aisle/harness/matched_run_launch.py": "src/aisle/harness/matched_run_launch.py", + "matched_source:src/aisle/harness/matched_runtime.py": "src/aisle/harness/matched_runtime.py", + "matched_source:src/aisle/harness/matched_session.py": "src/aisle/harness/matched_session.py", + "matched_source:src/aisle/harness/matched_tool_service.py": "src/aisle/harness/matched_tool_service.py", + "matched_source:src/aisle/harness/matched_tools.py": "src/aisle/harness/matched_tools.py", + "matched_source:src/aisle/harness/mcp_harness_authority.py": "src/aisle/harness/mcp_harness_authority.py", + "matched_source:src/aisle/harness/mcp_harness_source.py": "src/aisle/harness/mcp_harness_source.py", + "matched_source:src/aisle/harness/mcp_harness_transport.py": "src/aisle/harness/mcp_harness_transport.py", + "matched_source:src/aisle/harness/monolith.py": "src/aisle/harness/monolith.py", + "matched_source:src/aisle/harness/monolithic.py": "src/aisle/harness/monolithic.py", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "src/aisle/harness/monolithic_run_evidence.py", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "src/aisle/harness/monolithic_run_prepare.py", + "matched_source:src/aisle/harness/pilot_records.py": "src/aisle/harness/pilot_records.py", + "matched_source:src/aisle/harness/provider_hosted.py": "src/aisle/harness/provider_hosted.py", + "matched_source:src/aisle/harness/provider_relay.py": "src/aisle/harness/provider_relay.py", + "matched_source:src/aisle/harness/provider_response_authority.py": "src/aisle/harness/provider_response_authority.py", + "matched_source:src/aisle/harness/provider_runner.py": "src/aisle/harness/provider_runner.py", + "matched_source:src/aisle/harness/provider_source_audit.py": "src/aisle/harness/provider_source_audit.py", + "matched_source:src/aisle/harness/registry.py": "src/aisle/harness/registry.py", + "matched_source:src/aisle/harness/rollout.py": "src/aisle/harness/rollout.py", + "matched_source:src/aisle/harness/rollout_client.py": "src/aisle/harness/rollout_client.py", + "matched_source:src/aisle/harness/simulator_work.py": "src/aisle/harness/simulator_work.py", + "matched_source:src/aisle/harness/traces.py": "src/aisle/harness/traces.py", + "matched_source:src/aisle/harness/treatment_ambient.py": "src/aisle/harness/treatment_ambient.py", + "matched_source:src/aisle/harness/treatment_confinement.py": "src/aisle/harness/treatment_confinement.py", + "matched_source:src/aisle/harness/treatment_integrity.py": "src/aisle/harness/treatment_integrity.py", + "matched_source:src/aisle/harness/treatment_postflight.py": "src/aisle/harness/treatment_postflight.py", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "src/aisle/harness/typed_execution_bundle.py", + "matched_source:src/aisle/harness/typed_graph_audit.py": "src/aisle/harness/typed_graph_audit.py", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "src/aisle/harness/typed_graph_hosts.py", + "matched_source:src/aisle/harness/typed_graph_stage.py": "src/aisle/harness/typed_graph_stage.py", + "matched_source:src/aisle/harness/typed_node_host.py": "src/aisle/harness/typed_node_host.py", + "matched_source:src/aisle/harness/typed_node_requests.py": "src/aisle/harness/typed_node_requests.py", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "src/aisle/harness/typed_node_supervisor.py", + "matched_source:src/aisle/harness/typed_node_worker.py": "src/aisle/harness/typed_node_worker.py", + "matched_source:src/aisle/harness/typed_run_prepare.py": "src/aisle/harness/typed_run_prepare.py", + "matched_source:src/aisle/harness/typed_snapshot.py": "src/aisle/harness/typed_snapshot.py", + "matched_source:src/aisle/harness/typed_stage_provider.py": "src/aisle/harness/typed_stage_provider.py", + "matched_source:src/aisle/harness/typed_validation.py": "src/aisle/harness/typed_validation.py", + "matched_source:src/aisle/harness/typed_worker_launch.py": "src/aisle/harness/typed_worker_launch.py", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "src/aisle/harness/typed_worker_provisioning.py", + "matched_source:src/aisle/harness/validate.py": "src/aisle/harness/validate.py", + "matched_source:src/aisle/harness/worker_authority_probe.py": "src/aisle/harness/worker_authority_probe.py", + "matched_source:src/aisle/harness/worker_capability.py": "src/aisle/harness/worker_capability.py", + "matched_source:src/aisle/harness/worker_declaration.py": "src/aisle/harness/worker_declaration.py", + "matched_source:src/aisle/harness/worker_network_probe.py": "src/aisle/harness/worker_network_probe.py", + "matched_source:src/aisle/kinematics.py": "src/aisle/kinematics.py", + "matched_source:src/aisle/mobility/guard.py": "src/aisle/mobility/guard.py", + "matched_source:src/aisle/monolith/confinement.py": "src/aisle/monolith/confinement.py", + "matched_source:src/aisle/monolith/primitive_api.py": "src/aisle/monolith/primitive_api.py", + "matched_source:src/aisle/monolith/primitives.py": "src/aisle/monolith/primitives.py", + "matched_source:src/aisle/monolith/proxy.py": "src/aisle/monolith/proxy.py", + "matched_source:src/aisle/monolith/requests.py": "src/aisle/monolith/requests.py", + "matched_source:src/aisle/monolith/supervisor.py": "src/aisle/monolith/supervisor.py", + "matched_source:src/aisle/monolith/wire.py": "src/aisle/monolith/wire.py", + "matched_source:src/aisle/monolith/worker.py": "src/aisle/monolith/worker.py", + "matched_source:src/aisle/monolith/worker_config.py": "src/aisle/monolith/worker_config.py", + "matched_source:src/aisle/monolith/worker_launch.py": "src/aisle/monolith/worker_launch.py", + "matched_source:src/aisle/nodes/budget_guard.py": "src/aisle/nodes/budget_guard.py", + "matched_source:src/aisle/nodes/h6_fault.py": "src/aisle/nodes/h6_fault.py", + "matched_source:src/aisle/nodes/l2_pose.py": "src/aisle/nodes/l2_pose.py", + "matched_source:src/aisle/nodes/label_reader.py": "src/aisle/nodes/label_reader.py", + "matched_source:src/aisle/nodes/monolith_broker.py": "src/aisle/nodes/monolith_broker.py", + "matched_source:src/aisle/nodes/perception_session.py": "src/aisle/nodes/perception_session.py", + "matched_source:src/aisle/scenes/locations.toml": "src/aisle/scenes/locations.toml", + "matched_source:src/aisle/scenes/meds.toml": "src/aisle/scenes/meds.toml", + "matched_source:src/aisle/scenes/pharmacy.py": "src/aisle/scenes/pharmacy.py", + "matched_source:src/aisle/scenes/physics.toml": "src/aisle/scenes/physics.toml", + "matched_source:src/aisle/scenes/planogram.toml": "src/aisle/scenes/planogram.toml", + "matched_source:src/aisle/scenes/store.py": "src/aisle/scenes/store.py", + "matched_source:src/aisle/topics.py": "src/aisle/topics.py", + "matched_source:src/aisle/turn_node.py": "src/aisle/turn_node.py", + "matched_source:src/aisle/turns.py": "src/aisle/turns.py", + "matched_source:src/aisle/verifier/calibration.py": "src/aisle/verifier/calibration.py", + "matched_source:src/aisle/verifier/models.lock": "src/aisle/verifier/models.lock", + "matched_source:src/aisle/verifier/models.py": "src/aisle/verifier/models.py", + "matched_source:src/aisle/verifier/placement.toml": "src/aisle/verifier/placement.toml", + "matched_source:src/aisle/verifier/realistic.py": "src/aisle/verifier/realistic.py", + "matched_source:src/aisle/verifier/stages.py": "src/aisle/verifier/stages.py", + "matched_source:src/aisle/verifier/thresholds.toml": "src/aisle/verifier/thresholds.toml", + "matched_source:tools/campaign.py": "tools/campaign.py", + "matched_source:tools/matched_campaign.py": "tools/matched_campaign.py", + "monolithic_validators": "src/aisle/harness/monolithic.py", + "nested_host_documentation": "docs/monolithic/code-mode-admission.md", + "pilot_assignments_cli": "tools/pilot_assignments.py", + "pilot_records_cli": "tools/pilot_records.py", + "protocol": "analysis/freeze/cse-causal-study-v1/protocol.json", + "protocol_template": "analysis/statistics/protocol-template.json", + "provider_admission_documentation": "docs/monolithic/provider-admission.md", + "research_prompt": "harness/CLAUDE.research.md", + "seed_commitment_predecessor": "analysis/freeze/cse-causal-study-v22/freeze-manifest.json", + "session_runner": "tools/campaign.py", + "session_validators": "src/aisle/harness/causal_study.py", + "simulator_bridge": "src/aisle/nodes/dora_genesis.py", + "spec": "specs/500-typed-monolithic-causal-study.md", + "treatment_integrity": "src/aisle/harness/treatment_integrity.py", + "treatment_postflight": "src/aisle/harness/treatment_postflight.py", + "treatment_table": "analysis/monolithic-control/treatment-table-v1.json" + }, + "budgets": { + "campaign_ceiling": "harness/budget.toml", + "rollouts_per_session": 20, + "session_tokens": 200000, + "session_wall_s": 3600, + "sim_episodes_per_session": 160 + }, + "campaign_id": "cse-causal-study-v23", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose power", + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose freeze --output /tmp/cse-causal-study-v23-freeze-refusal.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-v23/freeze-manifest.json --allow-withheld-seeds" + ], + "decision_rules": { + "alpha": 0.05, + "equivalence_margin": { + "cost_at_equal_quality": { + "acceptance_probability": 0.1, + "heldout_quality": 0.1, + "rule": "CSE-5: both must pass or cost claims stay unconditional", + "sidedness": "two-sided 0.90" + } + }, + "multiplicity": "single primary endpoint", + "power": 0.8, + "primary_rule": "one-sided superiority at alpha 0.05 on the pooled Newcombe interval lower bound > 0 with no reversed stratum; otherwise weakened/rejected/undecidable by the #358 rule (CSE-3)", + "smallest_effect": { + "measure": "risk difference in session success", + "value": 0.25 + }, + "stopping_rule": "fixed n from the frozen power analysis using pilot-only control success; absolute floor 10 randomized assignments per arm per agent-system x task stratum (CSE-8); no optional stopping or extension" + }, + "endpoints": { + "primary": [ + { + "aggregation": "risk difference typed_dataflow - monolithic; exact arm intervals; Newcombe interval; per agent-system x task stratum before pooling (CSE-1, CSE-17)", + "direction": "higher", + "id": "E-CSE-1", + "outcome": "session_success", + "status": "inferential", + "unit": "agent_session" + } + ], + "secondary": [ + { + "aggregation": "counts by arm", + "direction": "descriptive", + "id": "E-CSE-2", + "outcome": "validation_and_launch_failure_class", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "Kaplan-Meier with censoring", + "direction": "lower", + "id": "E-CSE-3", + "outcome": "time_to_first_accepted_system", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "unconditional session estimands; no at-equal-quality wording unless CSE-5 equivalence passes", + "direction": "lower", + "id": "E-CSE-4", + "outcome": "tokens_tool_calls_wall_active_time_sim_work_rollouts", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "counts", + "direction": "descriptive", + "id": "E-CSE-5", + "outcome": "safety_interventions", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "dropped unless a blinded rubric is ratified before freeze (CSE-18)", + "direction": "n/a", + "id": "E-CSE-6", + "outcome": "artifact_legibility", + "status": "exploratory", + "unit": "agent_session" + } + ] + }, + "exclusions": { + "deviation_policy": "any behaviour-affecting change after registration is a documented deviation record and a new campaign version id; confirmatory data never revises the registered protocol", + "infrastructure": [ + "independent-controller-determined, arm-neutral facts only (CSE-14): host outage, provider interruption before first tool call, launcher crash before the session started" + ], + "rerun_policy": "a permitted rerun receives a new randomized id; the original row is retained with its exclusion reason and never replaced in place", + "treatment_integrity": [ + "contamination sentinel tripped (CSE-12)", + "model alias or CLI drift outside the frozen identity (CSE-6)", + "integrity verdict authored session-side (CSE-11)" + ] + }, + "hypotheses": [ + { + "direction": "typed_dataflow - monolithic > 0", + "id": "H-CSE-1", + "statement": "The bundled typed-dataflow surface raises the probability that a fresh coding-agent session produces a launching system passing the hidden held-out acceptance within budget, versus the equal-capability monolithic surface." + } + ], + "instrument_set": { + "agent_systems": [ + { + "binary": "claude", + "frozen": false, + "id": "claude-code", + "version_observed": "2.1.261" + }, + { + "binary": "codex", + "frozen": false, + "id": "codex-cli", + "version_observed": "0.151.0" + } + ], + "items": [ + "short_composition:TBD-by-bnd-task-band-calibration-v1", + "engineering:TBD-by-bnd-task-band-calibration-v1" + ], + "kind": "selected_task" + }, + "integrity_checks": [ + { + "gate": "CON-14 spec approval", + "kind": "external_review", + "owner_role": "project owner", + "record": null, + "status": "pending" + }, + { + "gate": "STA-12 independent statistical review", + "kind": "external_review", + "owner_role": "independent statistician", + "record": null, + "status": "pending" + }, + { + "gate": "MON-1 treatment table and expert parity (#344)", + "kind": "machine_check", + "owner_role": "parity harness; control surface exists, independent authorship and operator evidence remain pending", + "record": null, + "status": "pending" + }, + { + "gate": "BND-12 task-band freeze (#346)", + "kind": "machine_check", + "owner_role": "task-band auditor", + "record": null, + "status": "pending" + }, + { + "gate": "TRT confinement/mutation/agent-path (#353)", + "kind": "machine_check", + "owner_role": "integrity controller", + "record": null, + "status": "pending" + }, + { + "gate": "seed commitment verification", + "kind": "machine_check", + "owner_role": "holder of the original private seed and salt sources", + "record": null, + "status": "pending" + }, + { + "gate": "MON-8/MON-12 matched-session acceptance (#519)", + "kind": "machine_check", + "owner_role": "matched-session controller; complete acceptance evidence still required", + "record": null, + "status": "pending" + } + ], + "issue": 347, + "purpose": "confirmatory", + "schema_version": "aisle.freeze.declaration.v1", + "seed_commitment": { + "inherited_from": "analysis/freeze/cse-causal-study-v22/freeze-manifest.json", + "rule": "32 distinct integers in [1000,9999) drawn once from the OS CSPRNG, retained only under ~/aisle-private/freeze/cse-causal-study-v1/, disjoint by value from every committed dev/pilot seed list; revealed after the campaign closes", + "salt_source": "~/aisle-private/freeze/cse-causal-study-v1/salt.bin", + "seeds_source": "~/aisle-private/freeze/cse-causal-study-v1/seeds.json" + }, + "spec": "500", + "superseded": "cse-causal-study-v22 (session-bound confinement attestation: loopback network policy for the provider relay and the capability audit run under the session's exact policy (TRT-5/TRT-7); seed commitment, collection rules and all pending gates preserved)" +} diff --git a/analysis/freeze/cse-causal-study-v23/freeze-manifest.json b/analysis/freeze/cse-causal-study-v23/freeze-manifest.json new file mode 100644 index 00000000..a0d8cc9d --- /dev/null +++ b/analysis/freeze/cse-causal-study-v23/freeze-manifest.json @@ -0,0 +1,533 @@ +{ + "analysis_script_hashes": { + "src/aisle/harness/benchmark_statistics.py": "sha256:22cde3e9d7c5cc80874c7eec30aa2898e30476d6c939c94ab3009e8b12af641e", + "src/aisle/harness/causal_study.py": "sha256:1b1c53d6389881c146726d865a741a681f3d5039c9bf99558a53ee368250971e", + "src/aisle/harness/pilot_records.py": "sha256:68ac7ef9afd449d8527e2ffe99a9ef926e06253c466f3421bc1848e9aa094b70" + }, + "analysis_seed": 500001, + "artifact_hashes": { + "agent_adapters": "sha256:848f8f7da5ad978e53712991a688eb74f56c7a77deabee8afae717ad712666ca", + "analyzer": "sha256:22cde3e9d7c5cc80874c7eec30aa2898e30476d6c939c94ab3009e8b12af641e", + "budget_ceiling": "sha256:bcbb79aa146afed1d877da18093175ae144b4607db50119022ef4f8173122cda", + "dependency_declaration": "sha256:add4e718001da7678b404bdf37ea8d9889f563324ffba9b03fe65c46b172ee4c", + "dependency_lock": "sha256:9b8a82a17ba68c192f045405feed3ae7046af9568bddcb398292c20f0ed6f5a8", + "dora_runtime_selection": "sha256:f9eb7ffe285a0f0ed26d50303e31cf1e94d2f93ed62b599071dff5a183c15f53", + "frontend_conformance_decision": "sha256:5059582dc735ddb85f329c647070b42bcc689b2836e198d305f381e74f89edc2", + "frontend_request_authorization_documentation": "sha256:a1d4e65c0e467af4bd77922b95b12f26ee6478932eb5dd460de34b81249ca86b", + "matched_session_decision": "sha256:d88e0ef18c92cedff7b74dd3ffafed14d92affaec65beb9ddfec2801b365dc61", + "matched_session_documentation": "sha256:ff9aded2c28cfd857478095b044361062492a8dbf79a143fc9415a9f1545a22f", + "matched_source:assets/so101/so101.urdf": "sha256:3a65d2d35e68a8d2f0c2cc176d19b884506543c93ba72980145b80abe276022c", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "sha256:a284af4a06649763c7fac25eefbdd43ed068f00aab09f2aaaa17066e7b705ac5", + "matched_source:docs/monolithic/candidates.json": "sha256:08a1dc56502abc4673cb0b54f01664382214bb473c3ef293b1df7b6c74f64778", + "matched_source:docs/monolithic/experts-t1-l2.json": "sha256:263ea1a08c76f0c7ce34212a3cee4a71a0c4bbb67f8f11a319781e84cdde3cdf", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "sha256:b80c35f849afc3e28986710134c21c68247024c507d89c3a031513f36d2949e7", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "sha256:a1df92be604e4b0f65890640bca8f2b36f199b888ca1f8ef3df80ad13f2d764c", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "sha256:b19cc6366e3f773d822736812b56f5a1e00a78fc7bdddfe893cce80a3e236e2d", + "matched_source:env/limits.toml": "sha256:1ab1f4fd5d52e7949b81d21933daaa2c66e80ef07fdedf8217280468e0142edc", + "matched_source:experts/monolithic/expert_t1_l2.py": "sha256:93ade82db523fe808312f9d83c88ab3464eee303be1bfc3f824880bf1de7924c", + "matched_source:graphs/monolithic_t1_l2.yaml": "sha256:ff89c18e9466eef497836c95f8c01743f06fba124ac509c32ecda8d07b676ff8", + "matched_source:src/aisle/__init__.py": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "matched_source:src/aisle/embodiment.py": "sha256:d17c204492d680175cbaf365aaf164e2a6049da23ce125a9b9f21606b79f4cca", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "sha256:7e7fb09527b647c2bb13787011de8f9a4412e8b6f304e89efa107162d8150e82", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "sha256:459465e666c6a76a70e902c10c58905dfd8417b6c2ca3e038be99624e3542540", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "sha256:406d8f47d8d0b95a9d4ae104890e5e52764c9597e9271f75ff434a4ce24748d0", + "matched_source:src/aisle/harness/candidate_launch.py": "sha256:654eb90264703e3ce3ab599bb8e143047eaee288c8852c5b09c74e3147039270", + "matched_source:src/aisle/harness/candidates.py": "sha256:d1b441a30288a6c0500aaa765a24d4aeecca4f8d52e96f633befc50c84ecf5f5", + "matched_source:src/aisle/harness/cli.py": "sha256:c855f059d28409ffaa14be99ad3fc0bdc16fc33cf1a3c9584a75d968bd59d72d", + "matched_source:src/aisle/harness/code_mode_audit.py": "sha256:24638163f205408da1c6cf20a1b77c80a0d740a3f8ed44696eb90baf7de61f6a", + "matched_source:src/aisle/harness/code_mode_authority.py": "sha256:217ea025f10a4dc02b4b0ff2ff733adf4a695d1098b3c8ce0d2b27b8a140853d", + "matched_source:src/aisle/harness/code_mode_proxy.py": "sha256:333eee561d610489d2f48f9359ad94a3d9d8ff729fbae0fd3dec4acf64be2cb2", + "matched_source:src/aisle/harness/code_mode_runner.py": "sha256:4f91577dcfc6c239bf4111bd6735a64e97d6e22b7ca6287389abae09023cc25d", + "matched_source:src/aisle/harness/common.py": "sha256:ef62bf518a99c8c4154908bc9cf99e68402e6ae6c8e3fb5ec30c3f5fce9b5fac", + "matched_source:src/aisle/harness/frontend_app_server.py": "sha256:d21b9bf218efc3066399b8b22bbe3892e772433da315a9ecd40e2773369e9283", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "sha256:fb3cf5b8a115eb93e3f292dae694a4318cbe43e9552347fbc25ea8e43102ddbb", + "matched_source:src/aisle/harness/frontend_conformance.py": "sha256:54f13041eaa2eb8679b33a7c9e441e300d93818901cc8819503425cd025aa02f", + "matched_source:src/aisle/harness/frontend_dispatch.py": "sha256:6bff2abbacca6c332abdb5c5cb4cb3d688d8b00535e3f428633619bf91cd30c8", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "sha256:dc75024bd52c21c89a6112da847875a1d0ebf44f53a3704705e0bb8c81b9bc4a", + "matched_source:src/aisle/harness/frontend_effects.py": "sha256:bc539cecad349d27f68b1b2d6e435142c1119310b2129fb5a7c5854aa899d80e", + "matched_source:src/aisle/harness/frontend_qualification.py": "sha256:6c416f992cbc1418fb6fad52c9d0ab6bc3ff2e6fefd218f1bcf62b8500131d45", + "matched_source:src/aisle/harness/frontend_request_audit.py": "sha256:a4bc60b2ce1968ada1776324b72ad87025b93d366046029445fdf9b2e92f8830", + "matched_source:src/aisle/harness/frontend_request_authority.py": "sha256:0a061f1528b8bb2071dd1ba7e1d0bbee178b2c982bd6b688b916fb612522e163", + "matched_source:src/aisle/harness/guard_divergence.py": "sha256:af2c4755cabe46fabba913aee4819e05c10caaae79105615d50f8cf4567dad14", + "matched_source:src/aisle/harness/heldout_eval.py": "sha256:56c83164b04ae1b4623e8fd7a0579dd0a9caa7d5dbb891448643464a07893962", + "matched_source:src/aisle/harness/matched_app_server.py": "sha256:b7177caca8f1e4d3664b6cef76539b572c2ea6604fe0799c629bc50a52951b52", + "matched_source:src/aisle/harness/matched_collection.py": "sha256:b347127344a4c15fbb73fa9f3ac7c711a3d55b0fa368a11f76683bb1b5bbea14", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "sha256:b7f2b81397bcd232710408b9c6c9f463b3928e37f117d7310d519728f9b4b2c9", + "matched_source:src/aisle/harness/matched_evidence.py": "sha256:6d65476a9a923fd445ea9dad0334e521378353454b873ef59e2b0ff848c650f7", + "matched_source:src/aisle/harness/matched_frontend.py": "sha256:a4b9827af7dbf36f4af5228ca8c87ea36d2d32f8cffc7fcb8fea1923bcc09401", + "matched_source:src/aisle/harness/matched_run.py": "sha256:825fec659b8bdbb104d6fa07fa2adf29f4b23be74c1f3ec894a8ffcab6b68b64", + "matched_source:src/aisle/harness/matched_run_launch.py": "sha256:8701f034d0bd9873aa7981e3db47849d50bd4465f085e617e6e3193aac070539", + "matched_source:src/aisle/harness/matched_runtime.py": "sha256:c4d81e8309fcf5212315013dbf8e83347a65080e1221f37a0491757834ac2797", + "matched_source:src/aisle/harness/matched_session.py": "sha256:344ea0bc92705ef23f550045f1d4b26d55c7e2cf894133354c22068804887b21", + "matched_source:src/aisle/harness/matched_tool_service.py": "sha256:1bf6685addbeee541961e1053b2eea68cba66ce110667d8e7720398d49191a17", + "matched_source:src/aisle/harness/matched_tools.py": "sha256:eff206d64477d6a91f88c4889e2934a17bba254612c7dc72d542e799e11ef948", + "matched_source:src/aisle/harness/mcp_harness_authority.py": "sha256:f5760ef7a4e05a1444ee6ae5da3d9c37d0859cf767e2d96862b4d21803d646eb", + "matched_source:src/aisle/harness/mcp_harness_source.py": "sha256:f5a57dd19fc1c2966a8988ff925bf744384a4bf1368760427465340c9167990e", + "matched_source:src/aisle/harness/mcp_harness_transport.py": "sha256:a0c713cd82628109cd7324ff3deeaa33a66d37819a7c517b273a32e5e82cdaf3", + "matched_source:src/aisle/harness/monolith.py": "sha256:a12428eab867b869d902733e43626669b58f87218f0c40d1713712c094ef7414", + "matched_source:src/aisle/harness/monolithic.py": "sha256:1983ed8c26b2371c993c571f33f5c7e4ee81ea2597f9468b43d53c9970465ccf", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "sha256:4c0cb658f0148eb5d06ad69dceafd39148a2d4787e085cb624047c365c87bb24", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "sha256:23f2646571d1ed67c5b8e09e7b0b10bcc6cfa996b05b3161fe39f859d88c15ba", + "matched_source:src/aisle/harness/pilot_records.py": "sha256:68ac7ef9afd449d8527e2ffe99a9ef926e06253c466f3421bc1848e9aa094b70", + "matched_source:src/aisle/harness/provider_hosted.py": "sha256:82a3c8dd10449a07970e39322f649d6680e55fb935ee0dbdc67381b136e81068", + "matched_source:src/aisle/harness/provider_relay.py": "sha256:b353c3c40ea03dd63b46264fb60623ae535a7e45aec867261420ba404c9fe19c", + "matched_source:src/aisle/harness/provider_response_authority.py": "sha256:9562a8de28967e2eb83910ea5062dc491c8588a6bb4a2a04407f44d59f2251fe", + "matched_source:src/aisle/harness/provider_runner.py": "sha256:2c39d11d67cc4da2f9a2c732e86ba4e9a4b8da88570bdb1ecd1364d4668831cb", + "matched_source:src/aisle/harness/provider_source_audit.py": "sha256:e3be1f9d70288b4f596427a39ab4274399e9fdc74e87043e3c1249596fe1872b", + "matched_source:src/aisle/harness/registry.py": "sha256:214da7bb8db26c80c79262fa79c40a624d8a9981463d38a9c4aa30bc989bc1fb", + "matched_source:src/aisle/harness/rollout.py": "sha256:371789bd5093ef5c9e5ebb92177de6470caf5ec93ceeb59627411852fa62684a", + "matched_source:src/aisle/harness/rollout_client.py": "sha256:18808db0aa2df356d1a7cc9a538585aa9dcc653abd120e761649f1dff2c27c31", + "matched_source:src/aisle/harness/simulator_work.py": "sha256:2ade87416ab8c5d8902e9734af850b7613ee9ee42f9ca672c9482625e15d3d91", + "matched_source:src/aisle/harness/traces.py": "sha256:c2e6a717a56ce47e296bdc0115b15de9361b72084638b4f3bc66f7e7d032546b", + "matched_source:src/aisle/harness/treatment_ambient.py": "sha256:13dce400a649bc0d01749661517138f8ad1f8cde106caceaedb9f33af6f6d856", + "matched_source:src/aisle/harness/treatment_confinement.py": "sha256:665b50b294d9947c706a6e6c817217600c519e0c375da87464dbc5c4eb4253df", + "matched_source:src/aisle/harness/treatment_integrity.py": "sha256:b37162c6f295c5686da9b3089eb3bd96b69f68f01f41da0a784c4017658b10a6", + "matched_source:src/aisle/harness/treatment_postflight.py": "sha256:0bf01138ef0c277c6fcd728fcf25f4f13795de679fb41ab58b868b81f6c2f0ca", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "sha256:607945d5a51e3a139ecd35a8dceaf77a2027b0f842a577587616e0490b35c663", + "matched_source:src/aisle/harness/typed_graph_audit.py": "sha256:ca315a2717668f6e1029c0940471df6320eb23af859a0bf41220180bbcae656c", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "sha256:0376209def99645318d1476144a6c00e8d5d7e823868ff6033f1d1f572f609d3", + "matched_source:src/aisle/harness/typed_graph_stage.py": "sha256:0ea37ad396fb3321978c5f8f63d869a8ba0a652f7377cad60f13060b5a5ccbfb", + "matched_source:src/aisle/harness/typed_node_host.py": "sha256:56a5f0ab8159071897aad22d49aae2ab1785e6de9a186412b7b5566db334d056", + "matched_source:src/aisle/harness/typed_node_requests.py": "sha256:c3712381bc7383beef1e3304ce05f6a4fe27fb16d9b7e97e7f51f8d7cb6de78b", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "sha256:d964517b767bdd087771bf9233c05de61604cf5f705124d6b83be05feb0cb577", + "matched_source:src/aisle/harness/typed_node_worker.py": "sha256:890cb125e12b05afa3609186ceae6d1d45d1d765f653e2c0b7345b00b2c24a89", + "matched_source:src/aisle/harness/typed_run_prepare.py": "sha256:787ba2edc04efa6df752fdba2112ff50e34d2061acb74471c52c78e813382063", + "matched_source:src/aisle/harness/typed_snapshot.py": "sha256:96003b648f09d1348375ca72709537dd80aa4ddecd60e87465011d221d3200ec", + "matched_source:src/aisle/harness/typed_stage_provider.py": "sha256:afe7e37ef08020f5bf19630ef9117af2bdf9d8fe68ceba7f4a352f7c61c8c9ff", + "matched_source:src/aisle/harness/typed_validation.py": "sha256:bf51adce9f8fe107a2586bb4ffe2aec0cff684a235b6fde5837a95223f37ffa2", + "matched_source:src/aisle/harness/typed_worker_launch.py": "sha256:0759d03e0d26bfa2b38e11771a2ba79a2d7b0f8ca816cd10f084fdfbaa7c93d0", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "sha256:0e594beec5a07829044900d4a93fcde8e2e0e9d50e1e0f7e19809f9d2845b3ba", + "matched_source:src/aisle/harness/validate.py": "sha256:ed82967e800af6ec4d1f5336374343df38a5ba437771dd94e24b06c7dc64f481", + "matched_source:src/aisle/harness/worker_authority_probe.py": "sha256:9ee0638062e062c500c7103664eecd541e5d68056bb83368223c09f145dad6ee", + "matched_source:src/aisle/harness/worker_capability.py": "sha256:272c82452cd6f81912593b4584729cb7bb0e8f528d662cb3021e8dfc6ce70e83", + "matched_source:src/aisle/harness/worker_declaration.py": "sha256:15f42f83cae62a8e917f010b3e1571dda81a3cc7e7e3d630ed2b697fb0e8c0c2", + "matched_source:src/aisle/harness/worker_network_probe.py": "sha256:eb0d75fcbd4e8242eb61701f07e02e83ee6e7d9053247df5572f699969bcceb2", + "matched_source:src/aisle/kinematics.py": "sha256:0656cb4c9c4769365c42fc175deefb69d7a7a3dda77991329b1cedf33f7d5aaa", + "matched_source:src/aisle/mobility/guard.py": "sha256:3d0d4ddd757a8ae1840807511f8816f85feff0f2103fc490a8d29db6fa3663d0", + "matched_source:src/aisle/monolith/confinement.py": "sha256:f207d08bac411ae0cb9a5986b3f5f4d55e55d1407e0c0e6889ec5d489b232969", + "matched_source:src/aisle/monolith/primitive_api.py": "sha256:d5a344125ca3f434e489325b6bc7e20422082d18abee8ba7cd4deccda901327a", + "matched_source:src/aisle/monolith/primitives.py": "sha256:e4b3ef279708ef5fbd6e114f524a6c727f3c9d39a41d56dbfbe73d26e3ac515b", + "matched_source:src/aisle/monolith/proxy.py": "sha256:191b4ce164ea431ccbe728ad9ca5b482a09894093ee389643e099c3774df4ca4", + "matched_source:src/aisle/monolith/requests.py": "sha256:04151bf8838eda75be2a9d0c91240607283fb826712b90a8f99eab5d83d30185", + "matched_source:src/aisle/monolith/supervisor.py": "sha256:7dbe215dc5bddd12d414d1b88e896000fc64a9c71746230518eb2f47f1ef1051", + "matched_source:src/aisle/monolith/wire.py": "sha256:3418587b67e3ce2481e68c2809aaafc8e17f244246d33b055ee05f06ac6177a6", + "matched_source:src/aisle/monolith/worker.py": "sha256:931e42442b9f69157eab17cb73f1fa235eb053da7d843915ef889df5f84fb2d7", + "matched_source:src/aisle/monolith/worker_config.py": "sha256:55a26fdfa8b701470e50d54ffa9384d0ed89a0dc157867ac7e0005671e623312", + "matched_source:src/aisle/monolith/worker_launch.py": "sha256:0e0d053037e1dc9aef27c0248b8adce21c29546ca38ca7ebff6fd0c41c14475b", + "matched_source:src/aisle/nodes/budget_guard.py": "sha256:e15bc5f70b295e7dfd32f72e4a5cf72dffca9da50e9e8f6bb18360fb38069abf", + "matched_source:src/aisle/nodes/h6_fault.py": "sha256:c93b8e1607b7627adef9d971753727b9c602be44ae539def9188344332a63485", + "matched_source:src/aisle/nodes/l2_pose.py": "sha256:d2889dac35af94cdf89a280cf734f672098ce95a4a347e1b9797545c2269fda2", + "matched_source:src/aisle/nodes/label_reader.py": "sha256:aeb51080744779f68fb56d6ff408993e92e3e0b929b7d353af7521537f252517", + "matched_source:src/aisle/nodes/monolith_broker.py": "sha256:6fb86d2fdcae8d9ac0314e4cab3100cd925c35a96f74249006392e042847c048", + "matched_source:src/aisle/nodes/perception_session.py": "sha256:cf0fd80a43a338711456d7db546167d082fb6ecc461a828f3cb0e2d42701fc9e", + "matched_source:src/aisle/scenes/locations.toml": "sha256:baea316611ce2f05751ab3058b4b46acbdd9801d8734faca0272c06b2565bcca", + "matched_source:src/aisle/scenes/meds.toml": "sha256:3b25c5eb6586b311fbe8aa3ffc9fdab76cb1b60c9781c3a7a31b12ab6b61dc6b", + "matched_source:src/aisle/scenes/pharmacy.py": "sha256:642e4a57c9225804b44d4e6909fad6514475598cf002dff300b9ce4455225981", + "matched_source:src/aisle/scenes/physics.toml": "sha256:7a400f5cb01a4a93e2a1daa07fa3b2fafd6598643e15965d779c924480f7c36c", + "matched_source:src/aisle/scenes/planogram.toml": "sha256:c579e207de210ed102ff81e15bf0877a91ce0d4ac31ebce2c31e484322f0e576", + "matched_source:src/aisle/scenes/store.py": "sha256:ea79ce1c91d2085ac920fa1ffe87c0c662c875e0e36da5866c1bc60ae5093a66", + "matched_source:src/aisle/topics.py": "sha256:431a0e0bee4c4c3d83f09f9452eb481f238ec0b8cab11b7370bebc117beeb26f", + "matched_source:src/aisle/turn_node.py": "sha256:0c6cd11cd802ebf8934a671caeb4b8a590e5f43b23f03c52515f51eac4288705", + "matched_source:src/aisle/turns.py": "sha256:f0014e3d8ea6b07d4effd10c5bf0b7de08b89ecec1e164580509ad25694b36a4", + "matched_source:src/aisle/verifier/calibration.py": "sha256:a00b91406d27a3b6123701a7a7270258633ad9906c28dbb6f1110fbb07b59568", + "matched_source:src/aisle/verifier/models.lock": "sha256:3fa7271740f3ecd361213f45de140452f41216b80600e343e2daa9f2a413eab5", + "matched_source:src/aisle/verifier/models.py": "sha256:1b2905181bc757f8390666c2a4c7303f674a6d8a31232e8feae582b5cc24909f", + "matched_source:src/aisle/verifier/placement.toml": "sha256:8d8f37165cf4bb4214486597689d40d5df8fb36e2bb8f15e65f6be8801d3717f", + "matched_source:src/aisle/verifier/realistic.py": "sha256:769f6295d3c15a5be02da8ec815a401728059608f2ae92c904d5bec0dd18d156", + "matched_source:src/aisle/verifier/stages.py": "sha256:63848f9e4c0e59020461e3350bf8ca2a8d0ae93cc45cd43c5786c7b6f95a443c", + "matched_source:src/aisle/verifier/thresholds.toml": "sha256:567ac0544a4c382d3ce6547d8df868e6ca37c056b146cf0d1ee11934b1be16d3", + "matched_source:tools/campaign.py": "sha256:cb33ab83f9125ea9708a73f639ca38ad539fbc77e6c5b03e0b49dc2e66abd8f0", + "matched_source:tools/matched_campaign.py": "sha256:71e648035529877ee68b26eb2a4ccf5c0126e7872e3e9f6134785a1b63554609", + "monolithic_validators": "sha256:1983ed8c26b2371c993c571f33f5c7e4ee81ea2597f9468b43d53c9970465ccf", + "nested_host_documentation": "sha256:1c4777e4d8a29882ae5332f96df6b175423e7e5d1f279357fdb1286bace12959", + "pilot_assignments_cli": "sha256:bafcfdf4f79d5d0abd7442355454ea7ca476928afd57ed7c83f38da0c3726cc8", + "pilot_records_cli": "sha256:51438c6e14fca6368e1b91ebce77fd13c6bcc821edb61666553750a91d3ab75f", + "protocol": "sha256:459ce9961e3f5c7975ca38463e1807ca447f4463b0c9ad3e8eab1831b06b1d58", + "protocol_template": "sha256:9d636f3dc462a2e051ee0b3ded565aff1c5be86331e5b1b6692988509ccf088e", + "provider_admission_documentation": "sha256:856851935d15d8cb0c5653548ae9312441263cce9a3300e36d0863f5bdb96cf4", + "research_prompt": "sha256:e736bbd6caa7236dbfafe92ae42c0c7f13088764af1b753f6685d398c111b7c0", + "seed_commitment_predecessor": "sha256:0963d4cb2edf8d389aee19cfba1e1406ce7c9f73802352ecdf2a0642360794e0", + "session_runner": "sha256:cb33ab83f9125ea9708a73f639ca38ad539fbc77e6c5b03e0b49dc2e66abd8f0", + "session_validators": "sha256:1b1c53d6389881c146726d865a741a681f3d5039c9bf99558a53ee368250971e", + "simulator_bridge": "sha256:196837073251bedefd0365d220aa94407612f7670ed0b1b39c7319d63a1c18f8", + "spec": "sha256:f5096c587cd9687859f2dc80ec807d1bbf959cb4e05c152154e71e47bdc1f593", + "treatment_integrity": "sha256:b37162c6f295c5686da9b3089eb3bd96b69f68f01f41da0a784c4017658b10a6", + "treatment_postflight": "sha256:0bf01138ef0c277c6fcd728fcf25f4f13795de679fb41ab58b868b81f6c2f0ca", + "treatment_table": "sha256:0f860aefee6272ae5ca92f0a6d704d9f2dd924d931eed97d0b666f09361217d8" + }, + "campaign_id": "cse-causal-study-v23", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose power", + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose freeze --output /tmp/cse-causal-study-v23-freeze-refusal.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-v23/freeze-manifest.json --allow-withheld-seeds" + ], + "declaration": { + "analysis": { + "scripts": [ + "src/aisle/harness/benchmark_statistics.py", + "src/aisle/harness/causal_study.py", + "src/aisle/harness/pilot_records.py" + ], + "seed": 500001 + }, + "artifacts": { + "agent_adapters": "tools/agent_adapters.py", + "analyzer": "src/aisle/harness/benchmark_statistics.py", + "budget_ceiling": "harness/budget.toml", + "dependency_declaration": "pyproject.toml", + "dependency_lock": "uv.lock", + "dora_runtime_selection": "dora-runtime.json", + "frontend_conformance_decision": "docs/decisions/ADR-65.md", + "frontend_request_authorization_documentation": "docs/benchmark/v1/frontend-request-authorization.md", + "matched_session_decision": "docs/decisions/ADR-62.md", + "matched_session_documentation": "docs/monolithic/matched-session.md", + "matched_source:assets/so101/so101.urdf": "assets/so101/so101.urdf", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "docs/monolithic/allowlist-t1-l2.json", + "matched_source:docs/monolithic/candidates.json": "docs/monolithic/candidates.json", + "matched_source:docs/monolithic/experts-t1-l2.json": "docs/monolithic/experts-t1-l2.json", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "docs/monolithic/interface-map-t1-l2.json", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "docs/monolithic/parity-protocol-t1-l2.json", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "docs/monolithic/treatment-table-t1-l2.json", + "matched_source:env/limits.toml": "env/limits.toml", + "matched_source:experts/monolithic/expert_t1_l2.py": "experts/monolithic/expert_t1_l2.py", + "matched_source:graphs/monolithic_t1_l2.yaml": "graphs/monolithic_t1_l2.yaml", + "matched_source:src/aisle/__init__.py": "src/aisle/__init__.py", + "matched_source:src/aisle/embodiment.py": "src/aisle/embodiment.py", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "src/aisle/harness/_code_mode_protocol/__init__.py", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "src/aisle/harness/_code_mode_protocol/descriptor.pb", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "src/aisle/harness/_code_mode_protocol/protocol.proto", + "matched_source:src/aisle/harness/candidate_launch.py": "src/aisle/harness/candidate_launch.py", + "matched_source:src/aisle/harness/candidates.py": "src/aisle/harness/candidates.py", + "matched_source:src/aisle/harness/cli.py": "src/aisle/harness/cli.py", + "matched_source:src/aisle/harness/code_mode_audit.py": "src/aisle/harness/code_mode_audit.py", + "matched_source:src/aisle/harness/code_mode_authority.py": "src/aisle/harness/code_mode_authority.py", + "matched_source:src/aisle/harness/code_mode_proxy.py": "src/aisle/harness/code_mode_proxy.py", + "matched_source:src/aisle/harness/code_mode_runner.py": "src/aisle/harness/code_mode_runner.py", + "matched_source:src/aisle/harness/common.py": "src/aisle/harness/common.py", + "matched_source:src/aisle/harness/frontend_app_server.py": "src/aisle/harness/frontend_app_server.py", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "src/aisle/harness/frontend_app_server_audit.py", + "matched_source:src/aisle/harness/frontend_conformance.py": "src/aisle/harness/frontend_conformance.py", + "matched_source:src/aisle/harness/frontend_dispatch.py": "src/aisle/harness/frontend_dispatch.py", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "src/aisle/harness/frontend_dispatch_audit.py", + "matched_source:src/aisle/harness/frontend_effects.py": "src/aisle/harness/frontend_effects.py", + "matched_source:src/aisle/harness/frontend_qualification.py": "src/aisle/harness/frontend_qualification.py", + "matched_source:src/aisle/harness/frontend_request_audit.py": "src/aisle/harness/frontend_request_audit.py", + "matched_source:src/aisle/harness/frontend_request_authority.py": "src/aisle/harness/frontend_request_authority.py", + "matched_source:src/aisle/harness/guard_divergence.py": "src/aisle/harness/guard_divergence.py", + "matched_source:src/aisle/harness/heldout_eval.py": "src/aisle/harness/heldout_eval.py", + "matched_source:src/aisle/harness/matched_app_server.py": "src/aisle/harness/matched_app_server.py", + "matched_source:src/aisle/harness/matched_collection.py": "src/aisle/harness/matched_collection.py", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "src/aisle/harness/matched_dynamic_run.py", + "matched_source:src/aisle/harness/matched_evidence.py": "src/aisle/harness/matched_evidence.py", + "matched_source:src/aisle/harness/matched_frontend.py": "src/aisle/harness/matched_frontend.py", + "matched_source:src/aisle/harness/matched_run.py": "src/aisle/harness/matched_run.py", + "matched_source:src/aisle/harness/matched_run_launch.py": "src/aisle/harness/matched_run_launch.py", + "matched_source:src/aisle/harness/matched_runtime.py": "src/aisle/harness/matched_runtime.py", + "matched_source:src/aisle/harness/matched_session.py": "src/aisle/harness/matched_session.py", + "matched_source:src/aisle/harness/matched_tool_service.py": "src/aisle/harness/matched_tool_service.py", + "matched_source:src/aisle/harness/matched_tools.py": "src/aisle/harness/matched_tools.py", + "matched_source:src/aisle/harness/mcp_harness_authority.py": "src/aisle/harness/mcp_harness_authority.py", + "matched_source:src/aisle/harness/mcp_harness_source.py": "src/aisle/harness/mcp_harness_source.py", + "matched_source:src/aisle/harness/mcp_harness_transport.py": "src/aisle/harness/mcp_harness_transport.py", + "matched_source:src/aisle/harness/monolith.py": "src/aisle/harness/monolith.py", + "matched_source:src/aisle/harness/monolithic.py": "src/aisle/harness/monolithic.py", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "src/aisle/harness/monolithic_run_evidence.py", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "src/aisle/harness/monolithic_run_prepare.py", + "matched_source:src/aisle/harness/pilot_records.py": "src/aisle/harness/pilot_records.py", + "matched_source:src/aisle/harness/provider_hosted.py": "src/aisle/harness/provider_hosted.py", + "matched_source:src/aisle/harness/provider_relay.py": "src/aisle/harness/provider_relay.py", + "matched_source:src/aisle/harness/provider_response_authority.py": "src/aisle/harness/provider_response_authority.py", + "matched_source:src/aisle/harness/provider_runner.py": "src/aisle/harness/provider_runner.py", + "matched_source:src/aisle/harness/provider_source_audit.py": "src/aisle/harness/provider_source_audit.py", + "matched_source:src/aisle/harness/registry.py": "src/aisle/harness/registry.py", + "matched_source:src/aisle/harness/rollout.py": "src/aisle/harness/rollout.py", + "matched_source:src/aisle/harness/rollout_client.py": "src/aisle/harness/rollout_client.py", + "matched_source:src/aisle/harness/simulator_work.py": "src/aisle/harness/simulator_work.py", + "matched_source:src/aisle/harness/traces.py": "src/aisle/harness/traces.py", + "matched_source:src/aisle/harness/treatment_ambient.py": "src/aisle/harness/treatment_ambient.py", + "matched_source:src/aisle/harness/treatment_confinement.py": "src/aisle/harness/treatment_confinement.py", + "matched_source:src/aisle/harness/treatment_integrity.py": "src/aisle/harness/treatment_integrity.py", + "matched_source:src/aisle/harness/treatment_postflight.py": "src/aisle/harness/treatment_postflight.py", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "src/aisle/harness/typed_execution_bundle.py", + "matched_source:src/aisle/harness/typed_graph_audit.py": "src/aisle/harness/typed_graph_audit.py", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "src/aisle/harness/typed_graph_hosts.py", + "matched_source:src/aisle/harness/typed_graph_stage.py": "src/aisle/harness/typed_graph_stage.py", + "matched_source:src/aisle/harness/typed_node_host.py": "src/aisle/harness/typed_node_host.py", + "matched_source:src/aisle/harness/typed_node_requests.py": "src/aisle/harness/typed_node_requests.py", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "src/aisle/harness/typed_node_supervisor.py", + "matched_source:src/aisle/harness/typed_node_worker.py": "src/aisle/harness/typed_node_worker.py", + "matched_source:src/aisle/harness/typed_run_prepare.py": "src/aisle/harness/typed_run_prepare.py", + "matched_source:src/aisle/harness/typed_snapshot.py": "src/aisle/harness/typed_snapshot.py", + "matched_source:src/aisle/harness/typed_stage_provider.py": "src/aisle/harness/typed_stage_provider.py", + "matched_source:src/aisle/harness/typed_validation.py": "src/aisle/harness/typed_validation.py", + "matched_source:src/aisle/harness/typed_worker_launch.py": "src/aisle/harness/typed_worker_launch.py", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "src/aisle/harness/typed_worker_provisioning.py", + "matched_source:src/aisle/harness/validate.py": "src/aisle/harness/validate.py", + "matched_source:src/aisle/harness/worker_authority_probe.py": "src/aisle/harness/worker_authority_probe.py", + "matched_source:src/aisle/harness/worker_capability.py": "src/aisle/harness/worker_capability.py", + "matched_source:src/aisle/harness/worker_declaration.py": "src/aisle/harness/worker_declaration.py", + "matched_source:src/aisle/harness/worker_network_probe.py": "src/aisle/harness/worker_network_probe.py", + "matched_source:src/aisle/kinematics.py": "src/aisle/kinematics.py", + "matched_source:src/aisle/mobility/guard.py": "src/aisle/mobility/guard.py", + "matched_source:src/aisle/monolith/confinement.py": "src/aisle/monolith/confinement.py", + "matched_source:src/aisle/monolith/primitive_api.py": "src/aisle/monolith/primitive_api.py", + "matched_source:src/aisle/monolith/primitives.py": "src/aisle/monolith/primitives.py", + "matched_source:src/aisle/monolith/proxy.py": "src/aisle/monolith/proxy.py", + "matched_source:src/aisle/monolith/requests.py": "src/aisle/monolith/requests.py", + "matched_source:src/aisle/monolith/supervisor.py": "src/aisle/monolith/supervisor.py", + "matched_source:src/aisle/monolith/wire.py": "src/aisle/monolith/wire.py", + "matched_source:src/aisle/monolith/worker.py": "src/aisle/monolith/worker.py", + "matched_source:src/aisle/monolith/worker_config.py": "src/aisle/monolith/worker_config.py", + "matched_source:src/aisle/monolith/worker_launch.py": "src/aisle/monolith/worker_launch.py", + "matched_source:src/aisle/nodes/budget_guard.py": "src/aisle/nodes/budget_guard.py", + "matched_source:src/aisle/nodes/h6_fault.py": "src/aisle/nodes/h6_fault.py", + "matched_source:src/aisle/nodes/l2_pose.py": "src/aisle/nodes/l2_pose.py", + "matched_source:src/aisle/nodes/label_reader.py": "src/aisle/nodes/label_reader.py", + "matched_source:src/aisle/nodes/monolith_broker.py": "src/aisle/nodes/monolith_broker.py", + "matched_source:src/aisle/nodes/perception_session.py": "src/aisle/nodes/perception_session.py", + "matched_source:src/aisle/scenes/locations.toml": "src/aisle/scenes/locations.toml", + "matched_source:src/aisle/scenes/meds.toml": "src/aisle/scenes/meds.toml", + "matched_source:src/aisle/scenes/pharmacy.py": "src/aisle/scenes/pharmacy.py", + "matched_source:src/aisle/scenes/physics.toml": "src/aisle/scenes/physics.toml", + "matched_source:src/aisle/scenes/planogram.toml": "src/aisle/scenes/planogram.toml", + "matched_source:src/aisle/scenes/store.py": "src/aisle/scenes/store.py", + "matched_source:src/aisle/topics.py": "src/aisle/topics.py", + "matched_source:src/aisle/turn_node.py": "src/aisle/turn_node.py", + "matched_source:src/aisle/turns.py": "src/aisle/turns.py", + "matched_source:src/aisle/verifier/calibration.py": "src/aisle/verifier/calibration.py", + "matched_source:src/aisle/verifier/models.lock": "src/aisle/verifier/models.lock", + "matched_source:src/aisle/verifier/models.py": "src/aisle/verifier/models.py", + "matched_source:src/aisle/verifier/placement.toml": "src/aisle/verifier/placement.toml", + "matched_source:src/aisle/verifier/realistic.py": "src/aisle/verifier/realistic.py", + "matched_source:src/aisle/verifier/stages.py": "src/aisle/verifier/stages.py", + "matched_source:src/aisle/verifier/thresholds.toml": "src/aisle/verifier/thresholds.toml", + "matched_source:tools/campaign.py": "tools/campaign.py", + "matched_source:tools/matched_campaign.py": "tools/matched_campaign.py", + "monolithic_validators": "src/aisle/harness/monolithic.py", + "nested_host_documentation": "docs/monolithic/code-mode-admission.md", + "pilot_assignments_cli": "tools/pilot_assignments.py", + "pilot_records_cli": "tools/pilot_records.py", + "protocol": "analysis/freeze/cse-causal-study-v1/protocol.json", + "protocol_template": "analysis/statistics/protocol-template.json", + "provider_admission_documentation": "docs/monolithic/provider-admission.md", + "research_prompt": "harness/CLAUDE.research.md", + "seed_commitment_predecessor": "analysis/freeze/cse-causal-study-v22/freeze-manifest.json", + "session_runner": "tools/campaign.py", + "session_validators": "src/aisle/harness/causal_study.py", + "simulator_bridge": "src/aisle/nodes/dora_genesis.py", + "spec": "specs/500-typed-monolithic-causal-study.md", + "treatment_integrity": "src/aisle/harness/treatment_integrity.py", + "treatment_postflight": "src/aisle/harness/treatment_postflight.py", + "treatment_table": "analysis/monolithic-control/treatment-table-v1.json" + }, + "budgets": { + "campaign_ceiling": "harness/budget.toml", + "rollouts_per_session": 20, + "session_tokens": 200000, + "session_wall_s": 3600, + "sim_episodes_per_session": 160 + }, + "campaign_id": "cse-causal-study-v23", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose power", + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose freeze --output /tmp/cse-causal-study-v23-freeze-refusal.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-v23/freeze-manifest.json --allow-withheld-seeds" + ], + "decision_rules": { + "alpha": 0.05, + "equivalence_margin": { + "cost_at_equal_quality": { + "acceptance_probability": 0.1, + "heldout_quality": 0.1, + "rule": "CSE-5: both must pass or cost claims stay unconditional", + "sidedness": "two-sided 0.90" + } + }, + "multiplicity": "single primary endpoint", + "power": 0.8, + "primary_rule": "one-sided superiority at alpha 0.05 on the pooled Newcombe interval lower bound > 0 with no reversed stratum; otherwise weakened/rejected/undecidable by the #358 rule (CSE-3)", + "smallest_effect": { + "measure": "risk difference in session success", + "value": 0.25 + }, + "stopping_rule": "fixed n from the frozen power analysis using pilot-only control success; absolute floor 10 randomized assignments per arm per agent-system x task stratum (CSE-8); no optional stopping or extension" + }, + "endpoints": { + "primary": [ + { + "aggregation": "risk difference typed_dataflow - monolithic; exact arm intervals; Newcombe interval; per agent-system x task stratum before pooling (CSE-1, CSE-17)", + "direction": "higher", + "id": "E-CSE-1", + "outcome": "session_success", + "status": "inferential", + "unit": "agent_session" + } + ], + "secondary": [ + { + "aggregation": "counts by arm", + "direction": "descriptive", + "id": "E-CSE-2", + "outcome": "validation_and_launch_failure_class", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "Kaplan-Meier with censoring", + "direction": "lower", + "id": "E-CSE-3", + "outcome": "time_to_first_accepted_system", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "unconditional session estimands; no at-equal-quality wording unless CSE-5 equivalence passes", + "direction": "lower", + "id": "E-CSE-4", + "outcome": "tokens_tool_calls_wall_active_time_sim_work_rollouts", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "counts", + "direction": "descriptive", + "id": "E-CSE-5", + "outcome": "safety_interventions", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "dropped unless a blinded rubric is ratified before freeze (CSE-18)", + "direction": "n/a", + "id": "E-CSE-6", + "outcome": "artifact_legibility", + "status": "exploratory", + "unit": "agent_session" + } + ] + }, + "exclusions": { + "deviation_policy": "any behaviour-affecting change after registration is a documented deviation record and a new campaign version id; confirmatory data never revises the registered protocol", + "infrastructure": [ + "independent-controller-determined, arm-neutral facts only (CSE-14): host outage, provider interruption before first tool call, launcher crash before the session started" + ], + "rerun_policy": "a permitted rerun receives a new randomized id; the original row is retained with its exclusion reason and never replaced in place", + "treatment_integrity": [ + "contamination sentinel tripped (CSE-12)", + "model alias or CLI drift outside the frozen identity (CSE-6)", + "integrity verdict authored session-side (CSE-11)" + ] + }, + "hypotheses": [ + { + "direction": "typed_dataflow - monolithic > 0", + "id": "H-CSE-1", + "statement": "The bundled typed-dataflow surface raises the probability that a fresh coding-agent session produces a launching system passing the hidden held-out acceptance within budget, versus the equal-capability monolithic surface." + } + ], + "instrument_set": { + "agent_systems": [ + { + "binary": "claude", + "frozen": false, + "id": "claude-code", + "version_observed": "2.1.261" + }, + { + "binary": "codex", + "frozen": false, + "id": "codex-cli", + "version_observed": "0.151.0" + } + ], + "items": [ + "short_composition:TBD-by-bnd-task-band-calibration-v1", + "engineering:TBD-by-bnd-task-band-calibration-v1" + ], + "kind": "selected_task" + }, + "integrity_checks": [ + { + "gate": "CON-14 spec approval", + "kind": "external_review", + "owner_role": "project owner", + "record": null, + "status": "pending" + }, + { + "gate": "STA-12 independent statistical review", + "kind": "external_review", + "owner_role": "independent statistician", + "record": null, + "status": "pending" + }, + { + "gate": "MON-1 treatment table and expert parity (#344)", + "kind": "machine_check", + "owner_role": "parity harness; control surface exists, independent authorship and operator evidence remain pending", + "record": null, + "status": "pending" + }, + { + "gate": "BND-12 task-band freeze (#346)", + "kind": "machine_check", + "owner_role": "task-band auditor", + "record": null, + "status": "pending" + }, + { + "gate": "TRT confinement/mutation/agent-path (#353)", + "kind": "machine_check", + "owner_role": "integrity controller", + "record": null, + "status": "pending" + }, + { + "gate": "seed commitment verification", + "kind": "machine_check", + "owner_role": "holder of the original private seed and salt sources", + "record": null, + "status": "pending" + }, + { + "gate": "MON-8/MON-12 matched-session acceptance (#519)", + "kind": "machine_check", + "owner_role": "matched-session controller; complete acceptance evidence still required", + "record": null, + "status": "pending" + } + ], + "issue": 347, + "purpose": "confirmatory", + "schema_version": "aisle.freeze.declaration.v1", + "seed_commitment": { + "inherited_from": "analysis/freeze/cse-causal-study-v22/freeze-manifest.json", + "rule": "32 distinct integers in [1000,9999) drawn once from the OS CSPRNG, retained only under ~/aisle-private/freeze/cse-causal-study-v1/, disjoint by value from every committed dev/pilot seed list; revealed after the campaign closes", + "salt_source": "~/aisle-private/freeze/cse-causal-study-v1/salt.bin", + "seeds_source": "~/aisle-private/freeze/cse-causal-study-v1/seeds.json" + }, + "spec": "500", + "superseded": "cse-causal-study-v22 (session-bound confinement attestation: loopback network policy for the provider relay and the capability audit run under the session's exact policy (TRT-5/TRT-7); seed commitment, collection rules and all pending gates preserved)" + }, + "declaration_sha256": "sha256:d669fffb2a31ff76e4e15947d80643a8ac980689ba720095b59c295bbe8b3eba", + "frozen": false, + "gate_record_hashes": {}, + "git_head": "c9876f4a8985214ba4a9441ecf2bdea4209c3035", + "issue": 347, + "pending_gates": [ + "BND-12 task-band freeze (#346)", + "CON-14 spec approval", + "MON-1 treatment table and expert parity (#344)", + "MON-8/MON-12 matched-session acceptance (#519)", + "STA-12 independent statistical review", + "TRT confinement/mutation/agent-path (#353)", + "seed commitment verification" + ], + "purpose": "confirmatory", + "schema_version": "aisle.freeze.manifest.v1", + "seed_commitment": "sha256:f3f4e5efe4d573e1159d93e7b53a35a50620606e3dc72d0b5d5057611a2b7522", + "spec": "500", + "status": "registered_pending_review", + "timestamp": null, + "timestamp_source": null +} diff --git a/analysis/treatment-integrity/macos-confinement/README.md b/analysis/treatment-integrity/macos-confinement/README.md index bb34ffe5..ba9ea666 100644 --- a/analysis/treatment-integrity/macos-confinement/README.md +++ b/analysis/treatment-integrity/macos-confinement/README.md @@ -23,3 +23,14 @@ adapter hash, imported `system.sb` hash, compiled-profile hash, and policy id. Temporary synthetic hidden bytes and the compiled profile are removed after the audit; only their hashes, exposure booleans, and role-scoped policy metadata are retained. + +Since the session-bound attestation (2026-09-14), `audit-macos --policy +` runs the same matrix under a session's exact policy +(`MacOSPolicy.canonical_dict` JSON) so the retained record's profile hash and +policy id are the ones `wrap_verified_command` demands; `policy_source` in the +report says whether the synthetic or a session policy was audited, and the +report's hidden roots are digests, not paths. A `loopback` policy pins one +relay port and replaces the `tcp_read` denial with a `loopback_tcp_read` allow +control on that port, a `foreign_loopback_tcp_read` denial on another local +port, and an `external_tcp_read` denial. See +`docs/monolithic/matched-session.md`. diff --git a/docs/monolithic/matched-session.md b/docs/monolithic/matched-session.md index 51a631cd..9be90a37 100644 --- a/docs/monolithic/matched-session.md +++ b/docs/monolithic/matched-session.md @@ -71,6 +71,34 @@ process record, including on a nonzero process exit. Final snapshots and postflight failures remain in `matched-session.json` alongside the original launcher failure. +The `attestation` must have been produced under the arm's exact confinement +policy, or the launch wrapper refuses it: `python -m +aisle.harness.treatment_confinement audit-macos --policy +--output ` runs the full deny/allow matrix under that policy +(its sentinels are placed inside the policy's own visible, output and hidden +roots and removed afterwards) and retains the profile hash and policy id the +wrapper demands (TRT-5/TRT-7). The policy must admit the audit's probes +(`/bin/bash`, `/bin/cat`, the Apple developer Git, and either `/usr/bin/nc` +or a Python interpreter for the Unix-socket case) and must not admit its +unlisted-executable control (`/usr/bin/printf`). By default each sentinel goes +into the last directory of the matching root list; `--sentinel-visible`, +`--sentinel-output` and `--sentinel-hidden` name controller-private +directories instead so the audit never writes into another arm's readable +view. The retained report replaces hidden-root paths with their digests +(`policy_id` still binds the full policy). A sentinel left behind by an audit +process that died uncleanly is reclaimed on the next run; any other occupant +refuses. Two network policies compile: `deny-external` (no network) and +`loopback`, which pins ONE local TCP port (`loopback_port`) so a confined +frontend can reach exactly its provider relay; the launch binding's +`provider.relay_port` must equal it (the relay binds that port) and admission +refuses a mismatch. Under `loopback` the attestation carries a +`loopback_tcp_read` allow case on the pinned port, a `foreign_loopback_tcp_read` +case proving another local port is refused, and an `external_tcp_read` case +proving an external connect is refused outright. SBPL's `localhost` names every +address this host owns, so the pin, not the host, is what makes the grant +exclusive. A session whose attestation was not produced this way is +unattested, which the pilot registration names as a stop condition. + Deterministic integration tests run real child processes for both arms through an explicitly synthetic pass-through adapter. Those tests cover the process boundary and failure retention. Their synthetic access logs and capability diff --git a/src/aisle/harness/matched_run.py b/src/aisle/harness/matched_run.py index 83080c35..33b1b747 100644 --- a/src/aisle/harness/matched_run.py +++ b/src/aisle/harness/matched_run.py @@ -101,12 +101,7 @@ def _worker_binding(launch, config, config_path): raise ValueError("worker runtime differs from bound run runtime") if launch["python_sha256"] != hashlib.sha256(worker_interpreter().read_bytes()).hexdigest(): raise ValueError("worker and controller interpreter identities differ") - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in launch["policy"].items() - } - ) + policy = MacOSPolicy.from_canonical(launch["policy"]) readable = (*policy.visible_roots, *policy.runtime_read_roots, *policy.output_roots) if any(config_path.is_relative_to(p) for p in readable) or not any( config_path.is_relative_to(p) for p in policy.hidden_roots diff --git a/src/aisle/harness/matched_session.py b/src/aisle/harness/matched_session.py index d0397bdf..12aef5d0 100644 --- a/src/aisle/harness/matched_session.py +++ b/src/aisle/harness/matched_session.py @@ -315,13 +315,10 @@ def _permission_ids( for arm in sorted(ARMS): other = next(name for name in ARMS if name != arm) view, other_view = Path(views[arm]).resolve(), Path(views[other]).resolve() - declared = bindings[arm]["policy"] - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in declared.items() - } - ) + try: + policy = MacOSPolicy.from_canonical(bindings[arm]["policy"]) + except ConfinementError as exc: + raise AdmissionError(f"invalid confinement policy: {exc}") from exc editable = {view / p for p in manifests[arm]["repository"]["editable_allowlist"]} if set(policy.output_roots) != editable | {scratch[arm]}: raise AdmissionError("confinement write authority differs from declared edit grants") @@ -410,12 +407,18 @@ def _verify_launches( ) if "provider" in launch: from aisle.harness.provider_relay import verify_provider + from aisle.harness.treatment_confinement import verify_relay_port try: verify_provider(launch["provider"]) ceiling = candidates[arm]["budget"].get("frontend_tool_ceiling") if type(ceiling) is not int or ceiling <= 0: raise ValueError("provider requires a shared frontend dispatch ceiling") + bindings = (execution or {}).get("confinement_bindings") or {} + verify_relay_port( + bindings.get(arm, {}).get("policy") if isinstance(bindings, dict) else None, + launch["provider"], + ) except (ValueError, TypeError) as exc: raise AdmissionError("invalid provider binding: " + str(exc)) from exc if "code_mode_host" in launch: diff --git a/src/aisle/harness/matched_tools.py b/src/aisle/harness/matched_tools.py index 8d5f50c2..9e7cfbd7 100644 --- a/src/aisle/harness/matched_tools.py +++ b/src/aisle/harness/matched_tools.py @@ -616,12 +616,8 @@ def _dispatch( raise AdmissionError( "bound typed runtime requires a validation launch binding" ) - declared = current["confinement_bindings"][self.arm]["policy"] - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in declared.items() - } + policy = MacOSPolicy.from_canonical( + current["confinement_bindings"][self.arm]["policy"] ) compiled = compile_macos_profile(policy) python_hash = hashlib.sha256(self.python.read_bytes()).hexdigest() diff --git a/src/aisle/harness/monolithic_run_prepare.py b/src/aisle/harness/monolithic_run_prepare.py index c65bb539..b38fad94 100644 --- a/src/aisle/harness/monolithic_run_prepare.py +++ b/src/aisle/harness/monolithic_run_prepare.py @@ -33,12 +33,7 @@ def prepare_monolithic_run( if launch["runtime_record"] != runtime or launch["attestation"]["adapter"]["sha256"] != adapter: raise ValueError("worker declaration differs from admitted runtime or adapter") bundle, output = Path(launch["bundle"]).absolute(), Path(output).absolute() - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in launch["policy"].items() - } - ) + policy = MacOSPolicy.from_canonical(launch["policy"]) protected = [ Path(controller_root), *(Path(p) for p in views.values()), diff --git a/src/aisle/harness/provider_relay.py b/src/aisle/harness/provider_relay.py index e9eeca53..cc93f689 100644 --- a/src/aisle/harness/provider_relay.py +++ b/src/aisle/harness/provider_relay.py @@ -33,11 +33,17 @@ def verify_provider(binding): """Require an explicit upstream; local HTTP is for operator-owned fixtures only.""" _require( type(binding) is dict - and set(binding) - {"hosted_tool_contract"} - in ({"base_url"}, {"base_url", "requires_openai_auth"}), + and {"base_url"} <= set(binding) + and set(binding) + <= {"base_url", "requires_openai_auth", "hosted_tool_contract", "relay_port"}, "invalid provider binding", ) _require(type(binding.get("requires_openai_auth", True)) is bool, "invalid provider auth mode") + relay_port = binding.get("relay_port", 0) + _require( + type(relay_port) is int and not isinstance(relay_port, bool) and 0 <= relay_port < 65536, + "invalid relay port", + ) value = binding["base_url"] _require(type(value) is str and 0 < len(value) <= 2048, "invalid provider URL") url = urlsplit(value) @@ -86,6 +92,7 @@ class ProviderRelay: def __init__(self, binding, *, dispatch, output, timeout_s, delegated_tools=()): self.url = verify_provider(binding) self.hosted_contract = binding.get("hosted_tool_contract") + self._binding_port = binding.get("relay_port", 0) _require( type(timeout_s) in (int, float) and 0 < timeout_s < 86400, "invalid provider deadline" ) @@ -380,7 +387,9 @@ def do_POST(self): relay._handle(self) try: - self._server = Server(("127.0.0.1", 0), Handler) + # A pinned relay_port is the one loopback port a `loopback` + # confinement policy admits; 0 keeps the ephemeral default. + self._server = Server(("127.0.0.1", self._binding_port), Handler) self.address = self._server.server_address self._retain( "listener.json", diff --git a/src/aisle/harness/treatment_confinement.py b/src/aisle/harness/treatment_confinement.py index 064f6a6f..c458c106 100644 --- a/src/aisle/harness/treatment_confinement.py +++ b/src/aisle/harness/treatment_confinement.py @@ -9,11 +9,13 @@ from __future__ import annotations import argparse +import contextlib import hashlib import json import os import platform import shlex +import shutil import socket import subprocess import sys @@ -29,29 +31,81 @@ SANDBOX_EXEC = Path("/usr/bin/sandbox-exec") SYSTEM_PROFILE = Path("/System/Library/Sandbox/Profiles/system.sb") _HASH_LENGTH = 64 -_REQUIRED_CASE_IDS = { - "absolute_hidden_read", - "alternate_worktree_hidden_read", - "declared_output_write", - "git_object_hidden_read", - "hidden_write", - "parent_traversal_hidden_read", - "subprocess_hidden_read", - "subprocess_visible_read", - "symlink_hidden_read", - "unrestricted_alternate_worktree_baseline", - "unrestricted_git_object_baseline", - "unrestricted_hidden_baseline", - "visible_git_object_read", - "visible_read", - "unrestricted_tcp_baseline", - "tcp_read", - "unrestricted_unix_socket_baseline", - "unix_socket_network_control", - "unix_socket_read", - "unrestricted_exec_baseline", - "unlisted_executable", -} +#: the two network policies a controller profile may compile (TRT-5/TRT-7): +#: `deny-external` allows no network at all; `loopback` allows outbound +#: connections to the local host only, so a confined frontend can reach the +#: controller's provider relay and nothing else. +NETWORK_POLICIES = ("deny-external", "loopback") +#: the preferred Unix-socket probe; an admitted Python interpreter is the fallback +_NETCAT = "/usr/bin/nc" +_PYTHON_SOCKET_CLIENT = ( + "import socket,sys\n" + "s=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM)\n" + "s.settimeout(2)\n" + "s.connect(sys.argv[1])\n" + "sys.stdout.buffer.write(s.recv(4096))\n" +) +#: TEST-NET-1 (RFC 5737): never routable, so a permitted connect would hang +#: while a sandbox denial fails at once with "Operation not permitted". +_EXTERNAL_PROBE = "192.0.2.1" +_SENTINEL_DIR = ".aisle-capability" +_BASE_CASE_IDS = frozenset( + { + "absolute_hidden_read", + "alternate_worktree_hidden_read", + "declared_output_write", + "git_object_hidden_read", + "hidden_write", + "parent_traversal_hidden_read", + "subprocess_hidden_read", + "subprocess_visible_read", + "symlink_hidden_read", + "unrestricted_alternate_worktree_baseline", + "unrestricted_git_object_baseline", + "unrestricted_hidden_baseline", + "visible_git_object_read", + "visible_read", + "unrestricted_tcp_baseline", + "unrestricted_unix_socket_baseline", + "unix_socket_network_control", + "unix_socket_read", + "unrestricted_exec_baseline", + "unlisted_executable", + } +) + + +def required_case_ids(network_policy: str) -> frozenset[str]: + """The case matrix an attestation must carry for a compiled network policy.""" + if network_policy == "deny-external": + return _BASE_CASE_IDS | {"tcp_read"} + if network_policy == "loopback": + return _BASE_CASE_IDS | { + "loopback_tcp_read", + "foreign_loopback_tcp_read", + "external_tcp_read", + } + raise ConfinementError(f"unknown network policy: {network_policy}") + + +def _loopback_rule(port: int) -> str: + """Outbound to one local port only: the arm's own provider relay. SBPL + accepts only `localhost` or `*` as the host, and `localhost` means every + address this host owns, so the port pin is what makes the grant exclusive: + the relay holds that port on 127.0.0.1 and the audit proves any other local + port is refused.""" + return f'(allow network-outbound (remote ip "localhost:{port}"))' + + +def verify_relay_port(policy: Any, provider: Any) -> None: + """A `loopback` policy and its provider binding must name the same relay + port (MON-8): the sandbox admits exactly that port and the relay binds it.""" + if type(policy) is not dict or policy.get("network_policy") != "loopback": + return + pinned = policy.get("loopback_port") + declared = provider.get("relay_port") if type(provider) is dict else None + if type(declared) is not int or isinstance(declared, bool) or declared != pinned: + raise ValueError("provider relay_port must equal the confinement policy's loopback_port") class ConfinementError(RuntimeError): @@ -68,10 +122,14 @@ class MacOSPolicy: allowed_executables: tuple[Path, ...] hidden_roots: tuple[Path, ...] network_policy: str + #: the one local TCP port a `loopback` policy may reach (its provider relay) + loopback_port: int | None = None def as_dict(self) -> dict[str, Any]: - """Return constructor-compatible fields for tests and policy transforms.""" - return { + """Return constructor-compatible fields for tests and policy transforms. + The port appears only when pinned, so every existing consumer of the + root/network fields sees exactly the shape it always did.""" + fields = { "visible_roots": self.visible_roots, "output_roots": self.output_roots, "runtime_read_roots": self.runtime_read_roots, @@ -79,9 +137,14 @@ def as_dict(self) -> dict[str, Any]: "hidden_roots": self.hidden_roots, "network_policy": self.network_policy, } + if self.loopback_port is not None: + fields["loopback_port"] = self.loopback_port + return fields def canonical_dict(self) -> dict[str, Any]: - return { + """The identity form; the port appears only when a policy pins one, so a + deny-external policy keeps the identity it always had.""" + canonical = { "allowed_executables": [str(path) for path in self.allowed_executables], "hidden_roots": [str(path) for path in self.hidden_roots], "network_policy": self.network_policy, @@ -89,6 +152,44 @@ def canonical_dict(self) -> dict[str, Any]: "runtime_read_roots": [str(path) for path in self.runtime_read_roots], "visible_roots": [str(path) for path in self.visible_roots], } + if self.loopback_port is not None: + canonical["loopback_port"] = self.loopback_port + return canonical + + @classmethod + def from_canonical(cls, declared: Any) -> MacOSPolicy: + """Rebuild a policy from its canonical form; anything mis-shaped refuses.""" + fields = set(_CANONICAL_ROOT_FIELDS) | {"network_policy"} + if ( + type(declared) is not dict + or not fields <= set(declared) + or not set(declared) <= fields | {"loopback_port"} + ): + raise ConfinementError("policy must carry exactly the MacOSPolicy fields") + port = declared.get("loopback_port") + if ( + type(declared["network_policy"]) is not str + or (port is not None and (type(port) is not int or isinstance(port, bool))) + or any( + not isinstance(declared[k], list) or any(type(p) is not str for p in declared[k]) + for k in _CANONICAL_ROOT_FIELDS + ) + ): + raise ConfinementError("policy fields must be lists of paths and a network policy") + return cls( + **{key: tuple(Path(p) for p in declared[key]) for key in _CANONICAL_ROOT_FIELDS}, + network_policy=declared["network_policy"], + loopback_port=port, + ) + + +_CANONICAL_ROOT_FIELDS = ( + "visible_roots", + "output_roots", + "runtime_read_roots", + "allowed_executables", + "hidden_roots", +) @dataclass(frozen=True) @@ -96,6 +197,7 @@ class CompiledProfile: text: str sha256: str policy_id: str + network_policy: str def _canonical_bytes(value: Any) -> bytes: @@ -122,6 +224,10 @@ def _validate_roots(name: str, roots: tuple[Path, ...]) -> None: for path in roots: if not path.is_absolute(): raise ConfinementError(f"{name} contains a relative path: {path}") + if not str(path).isascii() or not str(path).isprintable(): + # SBPL literals are JSON-escaped; a non-ASCII name would compile + # to a different string than the directory it must guard. + raise ConfinementError(f"{name} contains a non-ASCII or unprintable path: {path}") if path == Path("/"): raise ConfinementError(f"{name} cannot authorize the filesystem root") if not path.exists(): @@ -139,8 +245,16 @@ def _contains(parent: Path, child: Path) -> bool: def _validate_policy(policy: MacOSPolicy) -> None: - if policy.network_policy != "deny-external": - raise ConfinementError("macOS capability audit requires deny-external network policy") + if policy.network_policy not in NETWORK_POLICIES: + raise ConfinementError( + "macOS capability audit requires a deny-external or loopback network policy" + ) + port = policy.loopback_port + if policy.network_policy == "loopback": + if type(port) is not int or isinstance(port, bool) or not 0 < port < 65536: + raise ConfinementError("loopback network policy must pin one relay port") + elif port is not None: + raise ConfinementError("only a loopback network policy may carry a relay port") for name in ( "visible_roots", "output_roots", @@ -189,12 +303,16 @@ def compile_macos_profile(policy: MacOSPolicy) -> CompiledProfile: ) lines.extend([")", "(allow file-write*"]) lines.extend(f" (subpath {_sbpl_path(path)})" for path in policy.output_roots) - lines.extend([")", "(allow signal (target self))", ""]) + lines.extend([")", "(allow signal (target self))"]) + if policy.network_policy == "loopback": + lines.append(_loopback_rule(policy.loopback_port)) + lines.append("") text = "\n".join(lines) return CompiledProfile( text=text, sha256=_sha256_bytes(text.encode()), policy_id=_sha256_bytes(_canonical_bytes(policy.canonical_dict())), + network_policy=policy.network_policy, ) @@ -248,6 +366,26 @@ def _run( ) from exc +#: the C locale keeps the probes' strerror text stable ("Operation not permitted") +_PROBE_ENV = {"LC_ALL": "C", "LANG": "C"} + + +def _run_denial_probe(command: list[str], *, cwd: Path) -> subprocess.CompletedProcess[bytes]: + """A probe that must be refused outright: a hang (the sandbox let the + connect through to an address that never answers) is recorded as a failed + case instead of aborting the whole audit.""" + try: + return subprocess.run( + command, cwd=cwd, check=False, capture_output=True, env=_PROBE_ENV, timeout=15 + ) + except subprocess.TimeoutExpired as exc: + return subprocess.CompletedProcess(command, -1, b"", f"probe hung: {exc}".encode()) + except OSError as exc: + raise ConfinementError( + f"capability command failed to execute: {command[0]}: {exc}" + ) from exc + + def _wrapped(profile_path: Path, command: list[str]) -> list[str]: return [str(SANDBOX_EXEC), "-f", str(profile_path), *command] @@ -336,72 +474,147 @@ def _platform_record() -> dict[str, str]: } -def _socket_capability_cases(profile_path: Path, cwd: Path, sentinel: bytes) -> list[dict]: - """Exercise the same loopback read outside and inside the external profile.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: - listener.bind(("127.0.0.1", 0)) - listener.listen() - listener.settimeout(0.1) - stopped = threading.Event() +def _tcp_read_command(port: int, host: str = "127.0.0.1") -> list[str]: + return ["/bin/bash", "-c", f"exec 3<>/dev/tcp/{host}/{port} && /bin/cat <&3"] - def serve(): - while not stopped.is_set(): - try: - connection, _ = listener.accept() - except TimeoutError: - continue - with connection: - connection.settimeout(1) - connection.sendall(sentinel) - - server = threading.Thread(target=serve, daemon=True) - server.start() - command = [ - "/bin/bash", - "-c", - f"exec 3<>/dev/tcp/127.0.0.1/{listener.getsockname()[1]} && /bin/cat <&3", - ] + +def _serve_once(listener: socket.socket, payload: bytes, stopped: threading.Event) -> None: + while not stopped.is_set(): + try: + connection, _ = listener.accept() + except TimeoutError: + continue + with connection: + connection.settimeout(1) + connection.sendall(payload) + + +def _socket_capability_cases( + profile_path: Path, cwd: Path, sentinel: bytes, policy: MacOSPolicy +) -> list[dict]: + """Exercise the same loopback read outside and inside the external profile. + + Under `deny-external` the confined loopback read must be denied. Under + `loopback` it must succeed (the relay is reachable) while a connect to a + never-routable external address must be refused by the sandbox itself: a + permitted connect would hang, so the denial is proven by the immediate + "Operation not permitted" and the absence of any sentinel, never by a + timeout. That external probe has no unconfined baseline for the same + reason (it would hang until the TCP timeout). Under `loopback` the + listener serves its own canary, not the hidden sentinel, so the permitted + read is a declared allow rather than a hidden-byte exposure. + """ + network_policy = policy.network_policy + served = sentinel if network_policy == "deny-external" else b"LOOPBACK-SYNTHETIC-CANARY\n" + # Under loopback the permitted listener sits on the pinned relay port (it + # must be free now, exactly as the relay will need it) and a second, + # foreign loopback listener proves the pin: any other local port is refused. + pinned = policy.loopback_port if network_policy == "loopback" else 0 + with contextlib.ExitStack() as sockets: + listener = sockets.enter_context(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) + try: + listener.bind(("127.0.0.1", pinned)) + except OSError as exc: + raise ConfinementError( + f"pinned relay port {pinned} is not free for the audit: {exc}" + ) from exc + listeners = [listener] + if network_policy == "loopback": + foreign = sockets.enter_context(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) + foreign.bind(("127.0.0.1", 0)) + listeners.append(foreign) + stopped = threading.Event() + servers = [] + for sock in listeners: + sock.listen() + sock.settimeout(0.1) + servers.append( + threading.Thread(target=_serve_once, args=(sock, served, stopped), daemon=True) + ) + for server in servers: + server.start() + command = _tcp_read_command(listener.getsockname()[1]) try: baseline = _run(command, cwd=cwd) confined = _run(_wrapped(profile_path, command), cwd=cwd) + if network_policy == "loopback": + foreign_read = _run_denial_probe( + _wrapped(profile_path, _tcp_read_command(foreign.getsockname()[1])), cwd=cwd + ) + external = _run_denial_probe( + _wrapped(profile_path, _tcp_read_command(9, host=_EXTERNAL_PROBE)), cwd=cwd + ) finally: stopped.set() - server.join(timeout=2) - return [ + for server in servers: + server.join(timeout=2) + cases = [ _case_result( - "unrestricted_tcp_baseline", baseline, sentinel, expected="baseline-exposure" - ), - _case_result("tcp_read", confined, sentinel, expected="deny"), + "unrestricted_tcp_baseline", baseline, served, expected="baseline-exposure" + ) ] + if network_policy == "deny-external": + cases.append(_case_result("tcp_read", confined, served, expected="deny")) + return cases + cases.append( + _case_result( + "loopback_tcp_read", + confined, + sentinel, + expected="allow", + extra_pass=confined.stdout == served, + ) + ) + cases.append( + _case_result( + "foreign_loopback_tcp_read", + foreign_read, + served, + expected="deny", + extra_pass=b"Operation not permitted" in foreign_read.stderr, + ) + ) + cases.append( + _case_result( + "external_tcp_read", + external, + served, + expected="deny", + extra_pass=b"Operation not permitted" in external.stderr, + ) + ) + return cases + +def _unix_socket_capability_cases( + profile_path: Path, cwd: Path, sentinel: bytes, client: list[str] +) -> list[dict]: + """Distinguish local socket denial from executable or filesystem failure. -def _unix_socket_capability_cases(profile_path: Path, cwd: Path, sentinel: bytes) -> list[dict]: - """Distinguish local socket denial from executable or filesystem failure.""" - endpoint = cwd / "fixture.sock" + `client` is the admitted argv prefix that connects to a Unix socket path + and prints what it reads (nc, or an admitted Python interpreter).""" + # A session root may exceed macOS sockaddr_un.sun_path; the endpoint lives + # in a short controller-private directory outside every agent-readable root, + # which changes nothing about the deny (network) or control (allow) cases. control_path = profile_path.with_name("unix-network-control.sb") control_text = profile_path.read_text() + ( "\n(allow network* (local unix-socket) (remote unix-socket))\n" ) control_path.write_text(control_text) - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: + with ( + tempfile.TemporaryDirectory(prefix="aisle-sock-", dir="/private/tmp") as socket_root, + socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener, + ): + endpoint = Path(socket_root) / "fixture.sock" listener.bind(str(endpoint)) listener.listen() listener.settimeout(0.1) stopped = threading.Event() - - def serve(): - while not stopped.is_set(): - try: - connection, _ = listener.accept() - except TimeoutError: - continue - with connection: - connection.settimeout(1) - connection.sendall(sentinel) - - server = threading.Thread(target=serve, daemon=True) + server = threading.Thread( + target=_serve_once, args=(listener, sentinel, stopped), daemon=True + ) server.start() - command = ["/usr/bin/nc", "-w", "2", "-U", str(endpoint)] + command = [*client, str(endpoint)] try: baseline = _run(command, cwd=cwd) confined = _run(_wrapped(profile_path, command), cwd=cwd) @@ -425,8 +638,234 @@ def serve(): ] -def run_macos_capability_audit() -> dict[str, Any]: - """Run the synthetic deny/allow matrix through the external adapter.""" +def _synthetic_policy( + visible: Path, output: Path, hidden: Path, git: Path, developer_root: Path +) -> MacOSPolicy: + """The fixed policy of the synthetic (session-less) audit.""" + return MacOSPolicy( + visible_roots=(visible,), + output_roots=(output,), + runtime_read_roots=( + Path("/Library/Apple").resolve(), + Path("/System").resolve(), + Path("/bin").resolve(), + Path("/usr/bin").resolve(), + Path("/usr/lib").resolve(), + Path("/usr/share").resolve(), + developer_root, + ), + allowed_executables=( + Path("/bin/bash").resolve(), + Path("/bin/cat").resolve(), + Path("/usr/bin/nc").resolve(), + git, + ), + hidden_roots=(hidden,), + network_policy="deny-external", + ) + + +def _socket_client(policy: MacOSPolicy) -> list[str]: + """The Unix-socket probe the policy already admits: `nc` when present, + otherwise an admitted Python interpreter. The audit never widens the arm's + executable grants for its own probes.""" + netcat = Path(_NETCAT).resolve() + if netcat in policy.allowed_executables: + return [str(netcat), "-w", "2", "-U"] + for path in policy.allowed_executables: + if path.name.startswith("python"): + return [str(path), "-I", "-c", _PYTHON_SOCKET_CLIENT] + raise ConfinementError( + "session policy must admit a Unix-socket probe (/usr/bin/nc or a python interpreter)" + ) + + +def _check_session_policy(policy: MacOSPolicy, git: Path) -> list[str]: + """A session policy can only be attested if the audit's own probes are + admitted executables and its unlisted-executable control is not; returns + the admitted Unix-socket probe argv.""" + _validate_policy(policy) + probes = {Path(p).resolve() for p in ("/bin/bash", "/bin/cat")} | {git} + missing = sorted(str(p) for p in probes - set(policy.allowed_executables)) + if missing: + raise ConfinementError("session policy must admit the audit probes: " + ", ".join(missing)) + control = Path("/usr/bin/printf").resolve() + if control in policy.allowed_executables: + raise ConfinementError(f"session policy admits the unlisted-executable control {control}") + return _socket_client(policy) + + +def _last_directory(name: str, roots: tuple[Path, ...]) -> Path: + for root in reversed(roots): + if root.is_dir() and not root.is_symlink(): + return root + raise ConfinementError(f"{name} holds no directory for a sentinel") + + +def _sentinel_homes( + policy: MacOSPolicy, requested: tuple[Path, Path, Path] | None +) -> tuple[Path, Path, Path]: + """Where the three sentinels go: the caller's directories (each inside a + root of the matching kind) or, by default, the last directory of each root + list. A live session's caller names controller-private homes so the audit + never writes into another arm's readable view.""" + kinds = ( + ("visible_roots", policy.visible_roots), + ("output_roots", policy.output_roots), + ("hidden_roots", policy.hidden_roots), + ) + if requested is None: + return tuple(_last_directory(name, roots) for name, roots in kinds) + homes = [] + for (name, roots), home in zip(kinds, requested, strict=True): + home = Path(home) + if ( + not home.is_absolute() + or home.resolve() != home + or not home.is_dir() + or not any(_contains(root, home) for root in roots if root.is_dir()) + ): + raise ConfinementError(f"sentinel home must be a canonical directory inside {name}") + homes.append(home) + return tuple(homes) + + +def _place_sentinels( + policy: MacOSPolicy, + cleanup: contextlib.ExitStack, + homes: tuple[Path, Path, Path] | None = None, +) -> tuple[Path, ...]: + """Fresh sentinel directories inside the session's own visible, output and + hidden roots. Each directory is registered for removal the moment it + exists, so a later failure never leaves one behind; anything already at a + sentinel path (file, directory or symlink) refuses.""" + homes = _sentinel_homes(policy, homes) + sentinels = [] + for home in homes: + path = home / _SENTINEL_DIR + if path.is_dir() and not path.is_symlink() and _reclaimable(path): + # residue of an audit that died uncleanly (SIGKILL): reclaim it + _remove_sentinel(path) + try: + path.mkdir() + except FileExistsError as exc: + raise ConfinementError(f"sentinel path already exists: {path}") from exc + except OSError as exc: + raise ConfinementError(f"sentinel path could not be created: {path}: {exc}") from exc + cleanup.callback(_remove_sentinel, path) + sentinels.append(path) + _owner_marker(path).write_text( + json.dumps({"schema_version": "aisle.sentinel-owner.v1", "pid": os.getpid()}) + ) + return tuple(sentinels) + + +def _redacted_policy(policy: MacOSPolicy) -> dict[str, Any]: + """The attested policy with hidden roots replaced by their digests: the + report is retained beside session evidence and must not name the private + roots it protects. `policy_id` still binds the full canonical form.""" + canonical = policy.canonical_dict() + canonical["hidden_roots"] = [ + "sha256:" + _sha256_bytes(path.encode()) for path in canonical["hidden_roots"] + ] + canonical["hidden_roots_redacted"] = True + return canonical + + +def _owner_marker(path: Path) -> Path: + return path / ".owner.json" + + +def _reclaimable(path: Path) -> bool: + """A leftover sentinel from an audit process that no longer exists (a + SIGKILL mid-run) may be reclaimed; anything else refuses.""" + marker_path = _owner_marker(path) + if marker_path.is_symlink() or not marker_path.is_file(): + return False + try: + marker = json.loads(marker_path.read_bytes()) + pid = marker["pid"] + except (OSError, ValueError, KeyError, TypeError): + return False # a corrupt or unreadable marker never authorizes removal + if ( + type(pid) is not int + or isinstance(pid, bool) + or marker.get("schema_version") != "aisle.sentinel-owner.v1" + ): + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return True # the owner is gone + except OSError: + return False # alive but not ours (EPERM) or undecidable + return False + + +def _remove_sentinel(path: Path) -> None: + failures: list[str] = [] + + def record(_function, failed_path, exc): + failures.append(f"{failed_path}: {exc}") + + shutil.rmtree(path, onexc=record) + if path.exists() or path.is_symlink(): + detail = f" ({failures[0]})" if failures else "" + raise ConfinementError(f"sentinel path could not be removed: {path}{detail}") + + +def _fixture_roots( + policy: MacOSPolicy | None, + sentinel_homes: tuple[Path, Path, Path] | None, + root: Path, + git: Path, + developer_root: Path, + cleanup: contextlib.ExitStack, +) -> tuple[MacOSPolicy, str, list[str], Path, Path, Path]: + """The audited policy and the three fixture directories: a synthetic policy + over fresh temporary roots, or the session policy with sentinels placed in + its own roots and registered for removal.""" + if policy is None: + visible, output, hidden = root / "visible", root / "output", root / "hidden" + for path in (visible, output, hidden): + path.mkdir() + policy = _synthetic_policy(visible, output, hidden, git, developer_root) + return policy, "synthetic", _socket_client(policy), visible, output, hidden + client = _check_session_policy(policy, git) + visible, output, hidden = _place_sentinels(policy, cleanup, sentinel_homes) + return policy, "session", client, visible, output, hidden + + +def _place_hidden_link( + visible: Path, hidden: Path, root: Path, source: str, hidden_sentinel: bytes +) -> None: + """The symlink case proves following a link out of the visible root is + refused. Under a session policy the link targets a hidden directory of the + controller's private temporary tree, never the session's own hidden root: a + link is readable text, and its target would otherwise disclose that root's + path to the arm (the arm's real hidden root is still covered by the + absolute and parent-traversal cases).""" + link_target = hidden if source == "synthetic" else root / "hidden-target" + if source == "session": + link_target.mkdir() + (link_target / "secret.txt").write_bytes(hidden_sentinel) + (visible / "hidden-link").symlink_to(link_target, target_is_directory=True) + + +def run_macos_capability_audit( + policy: MacOSPolicy | None = None, + *, + sentinel_homes: tuple[Path, Path, Path] | None = None, +) -> dict[str, Any]: + """Run the synthetic deny/allow matrix through the external adapter. + + Without a policy the audit runs its fixed synthetic policy in a private + temporary tree. With a session policy it runs the same matrix under that + exact policy, placing its sentinels inside the session's own roots (the + caller's `sentinel_homes`, or the last directory of each root list) and + removing them afterwards, so the retained attestation carries the profile + hash and policy id the launch wrapper will demand (TRT-5/TRT-7). + """ if sys.platform != "darwin": raise ConfinementError( "macOS confinement capability requires macOS; no simulation accepted" @@ -436,25 +875,32 @@ def run_macos_capability_audit() -> dict[str, Any]: if not SYSTEM_PROFILE.is_file(): raise ConfinementError(f"required imported profile is unavailable: {SYSTEM_PROFILE}") + # Cheap shape checks first: no subprocess is spawned for a malformed call. + if policy is None and sentinel_homes is not None: + raise ConfinementError("sentinel homes require a session policy") + if policy is not None: + _validate_policy(policy) + # Keep the Unix socket pathname below macOS sockaddr_un.sun_path limits. - with tempfile.TemporaryDirectory( - prefix="aisle-confinement-capability-", dir="/private/tmp" - ) as temporary: + with ( + tempfile.TemporaryDirectory( + prefix="aisle-confinement-capability-", dir="/private/tmp" + ) as temporary, + contextlib.ExitStack() as cleanup, + ): root = Path(temporary).resolve() - visible = root / "visible" - output = root / "output" - hidden = root / "hidden" - for path in (visible, output, hidden): - path.mkdir() + git, developer_root = _apple_git_runtime(cwd=root) + policy, source, client, visible, output, hidden = _fixture_roots( + policy, sentinel_homes, root, git, developer_root, cleanup + ) visible_sentinel = b"VISIBLE-SYNTHETIC-CAPABILITY\n" hidden_sentinel = b"HIDDEN-SYNTHETIC-CAPABILITY-7E4E\n" visible_file = visible / "allowed.txt" hidden_file = hidden / "secret.txt" visible_file.write_bytes(visible_sentinel) hidden_file.write_bytes(hidden_sentinel) - (visible / "hidden-link").symlink_to(hidden, target_is_directory=True) + _place_hidden_link(visible, hidden, root, source, hidden_sentinel) - git, developer_root = _apple_git_runtime(cwd=root) isolated_home = visible / "isolated-home" isolated_home.mkdir() git_environment = { @@ -494,27 +940,6 @@ def run_macos_capability_audit() -> dict[str, Any]: env=git_environment, ) - policy = MacOSPolicy( - visible_roots=(visible,), - output_roots=(output,), - runtime_read_roots=( - Path("/Library/Apple").resolve(), - Path("/System").resolve(), - Path("/bin").resolve(), - Path("/usr/bin").resolve(), - Path("/usr/lib").resolve(), - Path("/usr/share").resolve(), - developer_root, - ), - allowed_executables=( - Path("/bin/bash").resolve(), - Path("/bin/cat").resolve(), - Path("/usr/bin/nc").resolve(), - git, - ), - hidden_roots=(hidden,), - network_policy="deny-external", - ) compiled = compile_macos_profile(policy) profile_path = root / "controller-profile.sb" profile_path.write_text(compiled.text, encoding="utf-8") @@ -613,7 +1038,7 @@ def run_macos_capability_audit() -> dict[str, Any]: denied_reads = { "absolute_hidden_read": str(hidden_file), - "parent_traversal_hidden_read": str(visible / ".." / "hidden" / "secret.txt"), + "parent_traversal_hidden_read": os.path.relpath(hidden_file, visible), "symlink_hidden_read": str(visible / "hidden-link" / "secret.txt"), } for case_id, path in denied_reads.items(): @@ -710,8 +1135,8 @@ def run_macos_capability_audit() -> dict[str, Any]: _case_result("hidden_write", forbidden_write, hidden_sentinel, expected="deny") ) - cases.extend(_socket_capability_cases(profile_path, visible, hidden_sentinel)) - cases.extend(_unix_socket_capability_cases(profile_path, visible, hidden_sentinel)) + cases.extend(_socket_capability_cases(profile_path, visible, hidden_sentinel, policy)) + cases.extend(_unix_socket_capability_cases(profile_path, visible, hidden_sentinel, client)) executable_command = ["/usr/bin/printf", "%s", hidden_sentinel.decode()] cases.append( _case_result( @@ -762,7 +1187,8 @@ def run_macos_capability_audit() -> dict[str, Any]: "Apple system.sb is a private interface and is hashed per audit", ], "platform": _platform_record(), - "policy": policy.canonical_dict(), + "policy": _redacted_policy(policy), + "policy_source": source, "ephemeral_profile_path": str(profile_path), "recorded_at": recorded_at.isoformat(), "schema_version": SCHEMA_VERSION, @@ -781,12 +1207,17 @@ def run_macos_capability_audit() -> dict[str, Any]: } -def write_macos_capability_audit(output: Path) -> dict[str, Any]: +def write_macos_capability_audit( + output: Path, + policy: MacOSPolicy | None = None, + *, + sentinel_homes: tuple[Path, Path, Path] | None = None, +) -> dict[str, Any]: """Retain one unscored audit without overwriting earlier evidence.""" output = Path(output) if output.exists(): raise ConfinementError(f"capability audit already exists: {output}") - report = run_macos_capability_audit() + report = run_macos_capability_audit(policy, sentinel_homes=sentinel_homes) output.parent.mkdir(parents=True, exist_ok=True) rendered = json.dumps(report, indent=2, sort_keys=True) + "\n" try: @@ -827,11 +1258,17 @@ def wrap_verified_command( raise ConfinementError("adapter profile hash does not match compiled profile") if adapter.get("policy_id") != compiled.policy_id: raise ConfinementError("adapter policy id does not match compiled policy") + attested_policy = attestation.get("policy") + if ( + not isinstance(attested_policy, dict) + or attested_policy.get("network_policy") != compiled.network_policy + ): + raise ConfinementError("adapter attestation network policy does not match compiled profile") cases = attestation.get("cases") if ( not isinstance(cases, list) or any(not isinstance(row, dict) for row in cases) - or {row.get("id") for row in cases} != _REQUIRED_CASE_IDS + or {row.get("id") for row in cases} != required_case_ids(compiled.network_policy) or any(not row.get("passed") for row in cases) ): raise ConfinementError("adapter attestation contains a failed case") @@ -855,14 +1292,48 @@ def _parser() -> argparse.ArgumentParser: commands = parser.add_subparsers(dest="command", required=True) audit = commands.add_parser("audit-macos") audit.add_argument("--output", type=Path, required=True) + audit.add_argument( + "--policy", + type=Path, + default=None, + help="session policy (MacOSPolicy.canonical_dict JSON) to attest under", + ) + for role in ("visible", "output", "hidden"): + audit.add_argument( + f"--sentinel-{role}", + type=Path, + default=None, + help=f"directory inside the policy's {role} roots that receives that sentinel", + ) return parser +def load_policy(path: Path) -> MacOSPolicy: + """A MacOSPolicy from its canonical JSON form (lists of absolute paths).""" + try: + declared = json.loads(Path(path).read_bytes()) + except (OSError, json.JSONDecodeError) as exc: + raise ConfinementError(f"policy file unreadable: {exc}") from exc + return MacOSPolicy.from_canonical(declared) + + +def _sentinel_homes_from_args(args) -> tuple[Path, Path, Path] | None: + homes = (args.sentinel_visible, args.sentinel_output, args.sentinel_hidden) + if all(home is None for home in homes): + return None + if any(home is None for home in homes): + raise ConfinementError("sentinel homes must be given for visible, output and hidden") + return tuple(Path(home) for home in homes) + + def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) try: - report = write_macos_capability_audit(args.output) - except ConfinementError as exc: + policy = load_policy(args.policy) if args.policy is not None else None + report = write_macos_capability_audit( + args.output, policy, sentinel_homes=_sentinel_homes_from_args(args) + ) + except (ConfinementError, OSError) as exc: print(json.dumps({"error": str(exc), "ok": False}, sort_keys=True), file=sys.stderr) return 2 if not report["capability_pass"]: diff --git a/src/aisle/harness/typed_graph_audit.py b/src/aisle/harness/typed_graph_audit.py index 01effbf3..802d190b 100644 --- a/src/aisle/harness/typed_graph_audit.py +++ b/src/aisle/harness/typed_graph_audit.py @@ -36,12 +36,7 @@ def _worker(output, config, digest): if terminal.get("classification") != "module_result" or terminal.get("stage") != "done": raise StageError("worker has no completed module result") launch = dict(config["launch"]) - launch["policy"] = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in launch["policy"].items() - } - ) + launch["policy"] = MacOSPolicy.from_canonical(launch["policy"]) compiled = verify_typed_launch( **{ key: launch[key] diff --git a/src/aisle/harness/typed_node_host.py b/src/aisle/harness/typed_node_host.py index b93604d5..069b69c8 100644 --- a/src/aisle/harness/typed_node_host.py +++ b/src/aisle/harness/typed_node_host.py @@ -137,12 +137,7 @@ def preflight_host_config(path, digest): config = load_host_config(path, digest) try: launch = dict(config["launch"]) - launch["policy"] = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in launch["policy"].items() - } - ) + launch["policy"] = MacOSPolicy.from_canonical(launch["policy"]) compiled = verify_typed_launch( **{ key: launch[key] diff --git a/src/aisle/harness/typed_validation.py b/src/aisle/harness/typed_validation.py index 4ed2dcc4..4c1fb150 100644 --- a/src/aisle/harness/typed_validation.py +++ b/src/aisle/harness/typed_validation.py @@ -449,12 +449,7 @@ def _verify_validation_binding_fields(binding, runtime_record, source_roots, par raise ValidationError("validation binding fields are unresolved") if binding["schema_version"] != "aisle.typed-validation-binding.v1": raise ValidationError("validation binding schema is unsupported") - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in binding["policy"].items() - } - ) + policy = MacOSPolicy.from_canonical(binding["policy"]) kwargs = {key: value for key, value in binding.items() if key != "schema_version"} kwargs["policy"] = policy kwargs["runtime_record"] = runtime_record diff --git a/src/aisle/harness/worker_capability.py b/src/aisle/harness/worker_capability.py index a4189d30..c6cb9070 100644 --- a/src/aisle/harness/worker_capability.py +++ b/src/aisle/harness/worker_capability.py @@ -15,13 +15,13 @@ from aisle.harness.treatment_ambient import verify_declared_environment from aisle.harness.treatment_confinement import ( - _REQUIRED_CASE_IDS, EVIDENCE_CLASS, SANDBOX_EXEC, SCHEMA_VERSION, SYSTEM_PROFILE, _apple_git_runtime, compile_macos_profile, + required_case_ids, ) from aisle.harness.typed_snapshot import _read from aisle.harness.worker_authority_probe import operation_case, operation_command @@ -278,8 +278,8 @@ def observe(name, operation, target, sentinel, *, confined, allowed, control=Fal report["capability_pass"] = ( report["error"] is None and not report["cleanup_errors"] - and len(report["cases"]) == len(_REQUIRED_CASE_IDS) - and {row["id"] for row in report["cases"]} == _REQUIRED_CASE_IDS + and len(report["cases"]) == len(required_case_ids("deny-external")) + and {row["id"] for row in report["cases"]} == required_case_ids("deny-external") and all(row["passed"] for row in (*report["cases"], *report["controls"])) ) (output / "report.json").write_text(json.dumps(report, allow_nan=False)) diff --git a/src/aisle/monolith/worker_config.py b/src/aisle/monolith/worker_config.py index 4f42e81c..044810ba 100644 --- a/src/aisle/monolith/worker_config.py +++ b/src/aisle/monolith/worker_config.py @@ -8,7 +8,7 @@ from contextlib import contextmanager from pathlib import Path -from aisle.harness.treatment_confinement import MacOSPolicy +from aisle.harness.treatment_confinement import ConfinementError, MacOSPolicy from aisle.monolith.supervisor import WorkerFailure from aisle.monolith.worker_launch import launch_worker @@ -119,13 +119,8 @@ def factory(primitives): raise WorkerFailure("worker embodiment differs from configuration") launch = dict(config["launch"]) try: - launch["policy"] = MacOSPolicy( - **{ - name: value if name == "network_policy" else tuple(Path(p) for p in value) - for name, value in launch["policy"].items() - } - ) - except (AttributeError, TypeError, ValueError) as exc: + launch["policy"] = MacOSPolicy.from_canonical(launch["policy"]) + except (AttributeError, TypeError, ValueError, ConfinementError) as exc: raise WorkerFailure("invalid worker policy fields") from exc with launch_worker( **launch, output=Path(config["output_root"]) / phase, primitives=primitives diff --git a/tests/accept/test_provider_relay_transport.py b/tests/accept/test_provider_relay_transport.py index d539b58a..9a19d76c 100644 --- a/tests/accept/test_provider_relay_transport.py +++ b/tests/accept/test_provider_relay_transport.py @@ -471,3 +471,30 @@ def do_POST(self): request = json.loads((root / "00000001-request.json").read_bytes()) assert request["authorization_present"] is True assert request["account_header_present"] is True + + +def test_relay_binds_the_pinned_port_and_refuses_a_bad_one(tmp_path): + """MON-8/TRT-7: `relay_port` is the one loopback port a confined frontend may + reach, so the relay must bind exactly it; a malformed value refuses.""" + import socket + + from aisle.harness.provider_relay import verify_provider + + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + pinned = probe.getsockname()[1] + with DispatchBudget(tmp_path / "budget", session_id="session", ceiling=1) as budget: + with ProviderRelay( + {"base_url": "http://127.0.0.1:1/v1", "relay_port": pinned}, + dispatch=budget, + output=tmp_path / "provider", + timeout_s=5, + ) as relay: + assert relay.address == ("127.0.0.1", pinned) + listener = json.loads((tmp_path / "provider" / "listener.json").read_text()) + assert listener["port"] == pinned + for bad in (-1, 65536, True, "4321"): + with pytest.raises(ValueError, match="relay port"): + verify_provider({"base_url": "http://127.0.0.1:1/v1", "relay_port": bad}) + with pytest.raises(ValueError, match="invalid provider binding"): + verify_provider({"base_url": "http://127.0.0.1:1/v1", "port": 1}) diff --git a/tests/unit/test_freeze_registry.py b/tests/unit/test_freeze_registry.py index 0d0cc051..e2bc1922 100644 --- a/tests/unit/test_freeze_registry.py +++ b/tests/unit/test_freeze_registry.py @@ -409,7 +409,7 @@ def test_committed_registrations_check_clean_with_withheld_seeds(): registration names it in `superseded`; drift with no successor is the refusal the registry promises (analysis/freeze/README.md).""" manifests = _committed_manifests() - assert len(manifests) == 56 + assert len(manifests) == 58 superseded_ids: set[str] = set() for path in manifests: declaration = json.loads(path.with_name("declaration.json").read_text()) diff --git a/tests/unit/test_treatment_confinement.py b/tests/unit/test_treatment_confinement.py index 81c2d570..7a5cecfb 100644 --- a/tests/unit/test_treatment_confinement.py +++ b/tests/unit/test_treatment_confinement.py @@ -4,6 +4,8 @@ import hashlib import json +import os +import shutil import subprocess import sys from pathlib import Path @@ -119,6 +121,7 @@ def _attestation(compiled, profile_path: Path, adapter_path: Path) -> dict: ).hexdigest(), }, "profile_path": str(profile_path), + "policy": {"network_policy": compiled.network_policy}, "cases": [ {"id": case_id, "passed": True} for case_id in ( @@ -260,6 +263,10 @@ def test_live_capability_denies_hidden_path_variants_and_retains_no_hidden_bytes assert report["session_id"].startswith("macos-capability-") assert report["recorded_at"].endswith("+00:00") assert report["policy"]["network_policy"] == "deny-external" + assert report["policy_source"] == "synthetic" + from aisle.harness.treatment_confinement import required_case_ids + + assert set(cases) == required_case_ids("deny-external") assert report["adapter"]["imported_system_profile_sha256"] assert "HIDDEN-SYNTHETIC" not in json.dumps(report) @@ -334,3 +341,439 @@ def test_live_capability_blocks_unix_socket_with_matching_network_control(): assert denial["passed"] and denial["denied"] assert not denial["sentinel_exposed"] assert report["confirmatory_ready"] is False + + +# --- session-bound attestation and the loopback relay policy (PR7 part 2) --- + + +def _session_policy(tmp_path: Path, *, network_policy: str = "loopback") -> MacOSPolicy: + """A policy shaped like a real arm's: the audit's probes must be admitted + executables and the Apple developer tree a runtime read root.""" + from aisle.harness.treatment_confinement import _apple_git_runtime + + visible = tmp_path / "view" + output = tmp_path / "scratch" + hidden = tmp_path / "controller" + for path in (visible, output, hidden): + path.mkdir() + git, developer_root = _apple_git_runtime(cwd=tmp_path) + runtime_roots = tuple( + dict.fromkeys( + Path(p).resolve() + for p in ("/Library/Apple", "/System", "/bin", "/usr/bin", "/usr/lib", "/usr/share") + ) + ) + (developer_root,) + executables = tuple( + dict.fromkeys(Path(p).resolve() for p in ("/bin/bash", "/bin/cat", "/usr/bin/nc")) + ) + (git,) + return MacOSPolicy( + visible_roots=(visible.resolve(),), + output_roots=(output.resolve(),), + runtime_read_roots=runtime_roots, + allowed_executables=executables, + hidden_roots=(hidden.resolve(),), + network_policy=network_policy, + loopback_port=_free_port() if network_policy == "loopback" else None, + ) + + +def _free_port() -> int: + import socket + + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def test_loopback_policy_allows_only_localhost_outbound(tmp_path: Path): + """TRT-5/TRT-7: the relay policy admits outbound loopback and nothing else; + the deny-external profile carries no network allowance at all.""" + from aisle.harness.treatment_confinement import NETWORK_POLICIES + + assert NETWORK_POLICIES == ("deny-external", "loopback") + base = _policy(tmp_path).as_dict() + pinned = MacOSPolicy(**{**base, "network_policy": "loopback", "loopback_port": 4321}) + loopback = compile_macos_profile(pinned) + denied = compile_macos_profile(MacOSPolicy(**{**base, "network_policy": "deny-external"})) + assert '(allow network-outbound (remote ip "localhost:4321"))' in loopback.text + assert "localhost:*" not in loopback.text + assert "network*" not in loopback.text and "network-inbound" not in loopback.text + assert "network" not in denied.text + assert loopback.network_policy == "loopback" and denied.network_policy == "deny-external" + assert loopback.sha256 != denied.sha256 and loopback.policy_id != denied.policy_id + other = compile_macos_profile(MacOSPolicy(**{**pinned.as_dict(), "loopback_port": 4322})) + assert other.policy_id != loopback.policy_id + assert "loopback_port" not in denied.text + assert "loopback_port" not in MacOSPolicy(**base).canonical_dict() + assert pinned.canonical_dict()["loopback_port"] == 4321 + assert MacOSPolicy.from_canonical(pinned.canonical_dict()) == pinned + for bad in ( + {**base, "network_policy": "loopback"}, + {**base, "network_policy": "loopback", "loopback_port": 0}, + {**base, "network_policy": "loopback", "loopback_port": True}, + {**base, "network_policy": "deny-external", "loopback_port": 4321}, + ): + with pytest.raises(ConfinementError, match="port"): + compile_macos_profile(MacOSPolicy(**bad)) + + +def test_relay_port_must_match_the_pinned_loopback_port(): + """MON-8: a loopback policy and its provider binding name the same relay port.""" + from aisle.harness.treatment_confinement import verify_relay_port + + policy = {"network_policy": "loopback", "loopback_port": 4321} + verify_relay_port(policy, {"base_url": "https://x", "relay_port": 4321}) + verify_relay_port({"network_policy": "deny-external"}, {"base_url": "https://x"}) + verify_relay_port(None, {"base_url": "https://x"}) + for provider in ({"base_url": "https://x"}, {"base_url": "https://x", "relay_port": 4322}): + with pytest.raises(ValueError, match="relay_port"): + verify_relay_port(policy, provider) + + +def test_roots_must_be_printable_ascii(tmp_path: Path): + """TRT-5: a root the SBPL literal could not name exactly is refused.""" + odd = tmp_path / "visé" + odd.mkdir() + with pytest.raises(ConfinementError, match="non-ASCII"): + compile_macos_profile( + MacOSPolicy(**{**_policy(tmp_path).as_dict(), "visible_roots": (odd,)}) + ) + + +def test_required_case_ids_follow_the_network_policy(): + """TRT-6: under loopback the TCP denial is replaced by a loopback allow + control plus an external denial; every other case is unchanged.""" + from aisle.harness.treatment_confinement import required_case_ids + + denied = required_case_ids("deny-external") + loopback = required_case_ids("loopback") + assert "tcp_read" in denied and "loopback_tcp_read" not in denied + assert loopback - denied == { + "loopback_tcp_read", + "foreign_loopback_tcp_read", + "external_tcp_read", + } + assert denied - loopback == {"tcp_read"} + with pytest.raises(ConfinementError): + required_case_ids("unrestricted") + + +def test_launch_wrapper_requires_the_case_set_of_the_compiled_network_policy(tmp_path: Path): + """TRT-5: a deny-external attestation cannot authorize a loopback profile, + and the attested policy's network value must match the compiled one.""" + policy = MacOSPolicy( + **{**_policy(tmp_path).as_dict(), "network_policy": "loopback", "loopback_port": 4321} + ) + compiled = compile_macos_profile(policy) + profile_path = tmp_path / "profile.sb" + profile_path.write_text(compiled.text) + adapter_path = _fake_adapter(tmp_path) + report = _attestation(compiled, profile_path, adapter_path) + report["policy"] = policy.canonical_dict() + with pytest.raises(ConfinementError, match="failed case"): + wrap_verified_command(["/bin/cat", "x"], compiled, profile_path, report) + report["cases"] = [ + {"id": case_id, "passed": True} + for case_id in {row["id"] for row in report["cases"]} - {"tcp_read"} + | {"loopback_tcp_read", "foreign_loopback_tcp_read", "external_tcp_read"} + ] + assert wrap_verified_command(["/bin/cat", "x"], compiled, profile_path, report)[1:3] == [ + "-f", + str(profile_path), + ] + for broken in ({"network_policy": "deny-external"}, [], None): + report["policy"] = broken + with pytest.raises(ConfinementError, match="network policy"): + wrap_verified_command(["/bin/cat", "x"], compiled, profile_path, report) + + +def test_sentinel_placement_is_transactional_and_refuses_occupied_paths(tmp_path: Path): + """TRT-6: a sentinel that cannot be created removes the ones already placed, + an occupied sentinel path refuses, and an unremovable sentinel is an error.""" + import contextlib + + from aisle.harness import treatment_confinement as tc + + policy = _policy(tmp_path) + visible, output, hidden = ( + policy.visible_roots[0], + policy.output_roots[0], + policy.hidden_roots[0], + ) + hidden.chmod(0o500) + try: + with pytest.raises(ConfinementError, match="could not be created"): + with contextlib.ExitStack() as cleanup: + tc._place_sentinels(policy, cleanup) + finally: + hidden.chmod(0o700) + assert not (visible / ".aisle-capability").exists() + assert not (output / ".aisle-capability").exists() + (output / ".aisle-capability").write_text("occupied") + with pytest.raises(ConfinementError, match="already exists"): + with contextlib.ExitStack() as cleanup: + tc._place_sentinels(policy, cleanup) + assert not (visible / ".aisle-capability").exists() + (output / ".aisle-capability").unlink() + with contextlib.ExitStack() as cleanup: + placed = tc._place_sentinels(policy, cleanup) + assert placed == tuple(root / ".aisle-capability" for root in (visible, output, hidden)) + assert all(path.is_dir() for path in placed) + assert not any(path.exists() for path in placed) + # residue of a dead audit process is reclaimed; a live owner's is not + dead = visible / ".aisle-capability" + dead.mkdir() + (dead / "leftover.txt").write_text("x") + (dead / ".owner.json").write_text( + json.dumps({"schema_version": "aisle.sentinel-owner.v1", "pid": 2**22 + 12345}) + ) + with contextlib.ExitStack() as cleanup: + placed = tc._place_sentinels(policy, cleanup) + assert not (dead / "leftover.txt").exists() + assert json.loads((dead / ".owner.json").read_text())["pid"] == os.getpid() + assert not dead.exists() + dead.mkdir() + (dead / ".owner.json").write_text( + json.dumps({"schema_version": "aisle.sentinel-owner.v1", "pid": os.getpid()}) + ) + with pytest.raises(ConfinementError, match="already exists"): + with contextlib.ExitStack() as cleanup: + tc._place_sentinels(policy, cleanup) + shutil.rmtree(dead) + stuck = tmp_path / ".aisle-capability" + stuck.mkdir() + with pytest.MonkeyPatch.context() as patch: + patch.setattr(tc.shutil, "rmtree", lambda *args, **kwargs: None) + with pytest.raises(ConfinementError, match="could not be removed"): + tc._remove_sentinel(stuck) + + +def test_last_directory_prefers_the_last_directory_and_refuses_file_only_roots(tmp_path: Path): + """TRT-6: sentinels go into the last directory root; file-only roots refuse.""" + from aisle.harness.treatment_confinement import _last_directory + + first, second = tmp_path / "a", tmp_path / "b" + first.mkdir() + second.mkdir() + editable = tmp_path / "graph.yaml" + editable.write_text("nodes: []\n") + assert _last_directory("output_roots", (first, second, editable)) == second + with pytest.raises(ConfinementError, match="holds no directory"): + _last_directory("output_roots", (editable,)) + + +def _policy_json(tmp_path: Path) -> dict: + return _policy(tmp_path).canonical_dict() + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda body: "{not json", "unreadable"), + (lambda body: "[]", "exactly the MacOSPolicy fields"), + (lambda body: json.dumps({"visible_roots": []}), "exactly the MacOSPolicy fields"), + ( + lambda body: json.dumps({**body, "extra": 1}), + "exactly the MacOSPolicy fields", + ), + (lambda body: json.dumps({**body, "network_policy": ["loopback"]}), "lists of paths"), + (lambda body: json.dumps({**body, "hidden_roots": "/x"}), "lists of paths"), + (lambda body: json.dumps({**body, "visible_roots": [1]}), "lists of paths"), + ], +) +def test_load_policy_refuses_malformed_files(tmp_path: Path, mutate, message: str): + """TRT-5/CON-8: a malformed `--policy` file refuses instead of attesting.""" + from aisle.harness.treatment_confinement import load_policy + + policy_file = tmp_path / "policy.json" + policy_file.write_text(mutate(_policy_json(tmp_path))) + with pytest.raises(ConfinementError, match=message): + load_policy(policy_file) + with pytest.raises(ConfinementError, match="unreadable"): + load_policy(tmp_path / "absent.json") + + +def test_load_policy_round_trips_the_canonical_form_and_the_cli_refuses_with_json( + tmp_path: Path, capsys +): + """TRT-5/CON-8: canonical JSON loads back to the same policy; a bad policy file + makes `audit-macos` exit 2 with JSON on stderr and nothing on stdout.""" + from aisle.harness.treatment_confinement import load_policy, main + + policy = _policy(tmp_path) + policy_file = tmp_path / "policy.json" + policy_file.write_text(json.dumps(policy.canonical_dict())) + assert load_policy(policy_file) == policy + policy_file.write_text("{") + assert main(["audit-macos", "--policy", str(policy_file), "--output", str(tmp_path / "o")]) == 2 + out, err = capsys.readouterr() + assert out == "" and json.loads(err)["ok"] is False + + +@pytest.mark.skipif(sys.platform != "darwin", reason="sandbox-exec capability is macOS-only") +def test_session_policy_audit_binds_the_launch_profile_and_leaves_no_sentinels(tmp_path: Path): + """TRT-5/TRT-6/TRT-7: the audit run under a session's exact policy yields an + attestation the launch wrapper accepts, proves loopback reaches and external + connects are refused, and removes every sentinel it placed in the roots.""" + from aisle.harness.treatment_confinement import SANDBOX_EXEC, required_case_ids + + before = set(Path("/private/tmp").glob("aisle-sock-*")) + policy = _session_policy(tmp_path) + compiled = compile_macos_profile(policy) + report = run_macos_capability_audit(policy=policy) + cases = {row["id"]: row for row in report["cases"]} + assert report["capability_pass"] is True, [c for c in report["cases"] if not c["passed"]] + assert report["policy_source"] == "session" + expected_policy = policy.canonical_dict() + expected_policy["hidden_roots"] = [ + "sha256:" + hashlib.sha256(str(root).encode()).hexdigest() for root in policy.hidden_roots + ] + expected_policy["hidden_roots_redacted"] = True + assert report["policy"] == expected_policy + assert str(policy.hidden_roots[0]) not in json.dumps(report) + assert report["adapter"]["compiled_profile_sha256"] == compiled.sha256 + assert report["adapter"]["policy_id"] == compiled.policy_id + assert set(cases) == required_case_ids("loopback") + assert report["summary"] == { + "baseline_tests": 7, + "capability_pass": True, + "declared_allow_tests": 5, + "denial_detection_rate": 1.0, + "denial_tests": 11, + "false_alarm_rate": 0.0, + } + assert cases["loopback_tcp_read"]["passed"] and not cases["loopback_tcp_read"]["denied"] + for denial in ("foreign_loopback_tcp_read", "external_tcp_read"): + assert cases[denial]["passed"] and cases[denial]["denied"], cases[denial] + assert not cases[denial]["sentinel_exposed"] + assert set(Path("/private/tmp").glob("aisle-sock-*")) == before + assert cases["parent_traversal_hidden_read"]["denied"] + assert "HIDDEN-SYNTHETIC" not in json.dumps(report) + for root in (*policy.visible_roots, *policy.output_roots, *policy.hidden_roots): + assert not (root / ".aisle-capability").exists() + assert list(root.iterdir()) == [] + profile_path = tmp_path / "profile.sb" + profile_path.write_text(compiled.text) + wrapped = wrap_verified_command(["/bin/cat", "x"], compiled, profile_path, report) + assert wrapped[:3] == [str(SANDBOX_EXEC), "-f", str(profile_path)] + + +@pytest.mark.skipif(sys.platform != "darwin", reason="sandbox-exec capability is macOS-only") +@pytest.mark.parametrize( + ("mutate", "message"), + [ + ( + lambda policy, root: MacOSPolicy( + **{ + **policy.as_dict(), + "allowed_executables": ( + *policy.allowed_executables, + Path("/usr/bin/printf").resolve(), + ), + } + ), + "unlisted-executable control", + ), + ( + lambda policy, root: MacOSPolicy( + **{ + **policy.as_dict(), + "allowed_executables": tuple( + p for p in policy.allowed_executables if p.name != "nc" + ), + } + ), + "Unix-socket probe", + ), + ( + lambda policy, root: MacOSPolicy( + **{ + **policy.as_dict(), + "allowed_executables": tuple( + p for p in policy.allowed_executables if p.name != "cat" + ), + } + ), + "audit probes", + ), + ( + lambda policy, root: (root / "view" / ".aisle-capability").mkdir() or policy, + "sentinel", + ), + ], +) +def test_session_policy_audit_refuses_policies_it_cannot_prove(tmp_path: Path, mutate, message): + """TRT-5: a session policy that lacks the probes, admits the unlisted-executable + control, or already carries sentinel paths refuses instead of attesting.""" + policy = mutate(_session_policy(tmp_path), tmp_path) + with pytest.raises(ConfinementError, match=message): + run_macos_capability_audit(policy=policy) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="sandbox-exec capability is macOS-only") +def test_capability_cli_accepts_a_session_policy_file(tmp_path: Path): + """TRT-6/CON-8: `audit-macos --policy` retains an attestation bound to that policy.""" + policy = _session_policy(tmp_path) + policy_file = tmp_path / "policy.json" + policy_file.write_text(json.dumps(policy.canonical_dict())) + output = tmp_path / "audit.json" + result = subprocess.run( + [ + sys.executable, + "-m", + "aisle.harness.treatment_confinement", + "audit-macos", + "--policy", + str(policy_file), + "--output", + str(output), + ], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["ok"] is True + report = json.loads(output.read_text()) + assert report["policy_source"] == "session" + assert report["adapter"]["policy_id"] == compile_macos_profile(policy).policy_id + + +def test_socket_probe_uses_an_admitted_client_and_never_widens_the_grants(tmp_path: Path): + """TRT-5: the Unix-socket probe is nc when admitted, else an admitted Python + interpreter, else the policy refuses; the audit adds no executable of its own.""" + from aisle.harness.treatment_confinement import _socket_client + + base = _policy(tmp_path).as_dict() + python = Path(sys.executable).resolve() + # admitted executables are canonical paths (on Linux /usr/bin/nc is an + # alternatives link), and the audit resolves its probe the same way + netcat = Path("/usr/bin/nc").resolve() + with_nc = MacOSPolicy(**{**base, "allowed_executables": (*base["allowed_executables"], netcat)}) + assert _socket_client(with_nc) == [str(netcat), "-w", "2", "-U"] + with_python = MacOSPolicy( + **{**base, "allowed_executables": (*base["allowed_executables"], python)} + ) + argv = _socket_client(with_python) + assert argv[:3] == [str(python), "-I", "-c"] and "AF_UNIX" in argv[3] + with pytest.raises(ConfinementError, match="Unix-socket probe"): + _socket_client(MacOSPolicy(**base)) + + +def test_corrupt_or_live_owner_markers_are_never_reclaimed(tmp_path: Path): + """TRT-6: only a well-formed marker naming a dead process authorizes reclaiming + a leftover sentinel; corrupt, missing or live markers refuse.""" + from aisle.harness.treatment_confinement import _reclaimable + + sentinel = tmp_path / ".aisle-capability" + sentinel.mkdir() + assert not _reclaimable(sentinel) + marker = sentinel / ".owner.json" + marker.write_text("{corrupt") + assert not _reclaimable(sentinel) + marker.write_text(json.dumps({"schema_version": "other", "pid": 2**22 + 1})) + assert not _reclaimable(sentinel) + marker.write_text(json.dumps({"schema_version": "aisle.sentinel-owner.v1", "pid": os.getpid()})) + assert not _reclaimable(sentinel) + marker.write_text(json.dumps({"schema_version": "aisle.sentinel-owner.v1", "pid": 2**22 + 1})) + assert _reclaimable(sentinel) diff --git a/tests/unit/test_typed_graph_preflight.py b/tests/unit/test_typed_graph_preflight.py index cd927a6b..e4e9b43d 100644 --- a/tests/unit/test_typed_graph_preflight.py +++ b/tests/unit/test_typed_graph_preflight.py @@ -1,7 +1,7 @@ """MON-6/MON-13: every staged worker must pass admission before graph transport starts.""" import json -from dataclasses import asdict, replace +from dataclasses import replace from pathlib import Path import pytest @@ -47,7 +47,7 @@ def _stage(tmp_path, *, share_home=False, direct_python=True): compiled = compile_macos_profile(policy) profile.write_text(compiled.text) launch.update( - policy=asdict(policy), + policy=policy.as_dict(), environment=environment, environment_record=environment_record, profile_path=str(profile), diff --git a/tests/unit/test_typed_graph_stage.py b/tests/unit/test_typed_graph_stage.py index 013c6255..af24d5fc 100644 --- a/tests/unit/test_typed_graph_stage.py +++ b/tests/unit/test_typed_graph_stage.py @@ -2,7 +2,6 @@ import copy import json -from dataclasses import asdict from pathlib import Path import pytest @@ -45,7 +44,7 @@ def _declarations(inputs): "timeout_s", ) } - launch["policy"] = asdict(launch["policy"]) + launch["policy"] = launch["policy"].as_dict() launch.update(bundle=bundle, bundle_manifest=manifest, source_roots=[ROOT, inputs["snapshot"]]) launch = json.loads(json.dumps(launch, default=str)) return {n["id"]: copy.deepcopy(launch) for n in graph["nodes"] if n["id"] in names} diff --git a/tests/unit/test_typed_node_host.py b/tests/unit/test_typed_node_host.py index 90ba7699..ce0f9e51 100644 --- a/tests/unit/test_typed_node_host.py +++ b/tests/unit/test_typed_node_host.py @@ -2,7 +2,6 @@ import hashlib import json -from dataclasses import asdict from pathlib import Path import pytest @@ -24,7 +23,7 @@ def _config(tmp_path): for name in ("node", "module", "outputs"): inputs.pop(name) output = inputs.pop("output") - inputs["policy"] = asdict(inputs["policy"]) + inputs["policy"] = inputs["policy"].as_dict() config = { "schema_version": "aisle.typed-node-host.v1", "purpose": "expert_parity", diff --git a/tests/unit/test_worker_capability.py b/tests/unit/test_worker_capability.py index 9161db23..e44944f4 100644 --- a/tests/unit/test_worker_capability.py +++ b/tests/unit/test_worker_capability.py @@ -13,10 +13,12 @@ def test_actual_worker_capability_report_and_cleanup(tmp_path): """TRT-6: all required observations bind to the actual policy and preserve worker assets.""" from aisle.harness.treatment_confinement import ( - _REQUIRED_CASE_IDS, compile_macos_profile, + required_case_ids, wrap_verified_command, ) + + _REQUIRED_CASE_IDS = required_case_ids("deny-external") from aisle.harness.worker_capability import audit_worker_capability inputs = _launch_inputs(tmp_path, direct_python=True) diff --git a/tools/matched_campaign.py b/tools/matched_campaign.py index 30c451f6..99f83cb8 100644 --- a/tools/matched_campaign.py +++ b/tools/matched_campaign.py @@ -57,12 +57,7 @@ def launch(destination: Path) -> dict: raise AdmissionError( "process requires confinement, ambient and launch bindings" ) from exc - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in declared.items() - } - ) + policy = MacOSPolicy.from_canonical(declared) compiled = compile_macos_profile(policy) if ( attestation.get("adapter", {}).get("sha256") From 777f2077155d0dfd339d2423b114c58754eaa822 Mon Sep 17 00:00:00 2001 From: Yong Date: Thu, 17 Sep 2026 16:57:26 -0500 Subject: [PATCH 2/2] fix: bind freeze manifests to corrected source commit --- analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json | 2 +- analysis/freeze/cse-causal-study-v24/freeze-manifest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json b/analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json index c7532aea..9450562c 100644 --- a/analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json +++ b/analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json @@ -563,7 +563,7 @@ "declaration_sha256": "sha256:b70d4f89fe0a3d935f4e45c58211ab630199368ea075450c8045eba2d9501c8f", "frozen": false, "gate_record_hashes": {}, - "git_head": "8e0dd9e068178808c5b4e5e3a77c9f8bafea5ede", + "git_head": "cf834d629865b99de30308c73513060ebc8f48f6", "issue": 347, "pending_gates": [ "BND-12 task-band freeze (#346)", diff --git a/analysis/freeze/cse-causal-study-v24/freeze-manifest.json b/analysis/freeze/cse-causal-study-v24/freeze-manifest.json index 00fece30..32d86484 100644 --- a/analysis/freeze/cse-causal-study-v24/freeze-manifest.json +++ b/analysis/freeze/cse-causal-study-v24/freeze-manifest.json @@ -534,7 +534,7 @@ "declaration_sha256": "sha256:148b1fadc570e572d5301f021b671f0b2ad1ef8970efa4c6406e9f528496a892", "frozen": false, "gate_record_hashes": {}, - "git_head": "8e0dd9e068178808c5b4e5e3a77c9f8bafea5ede", + "git_head": "cf834d629865b99de30308c73513060ebc8f48f6", "issue": 347, "pending_gates": [ "BND-12 task-band freeze (#346)",