diff --git a/analysis/freeze/README.md b/analysis/freeze/README.md index 746e291c..80d90ca8 100644 --- a/analysis/freeze/README.md +++ b/analysis/freeze/README.md @@ -212,3 +212,10 @@ including the failed BND audit. Pilot v8 remains the historical fresh-seed registration; pilot v9 inherits its commitment unverified and does not authorize collection. A pilot run needs a fresh-seed run registration and execution-context admission for the revised graphs and cache. + +CSE v24 and pilot v10 supersede v23 and pilot v9 after #589 adds +session-bound capability audits and refuses the unsupported loopback endpoint +grant. They inherit #578's public-observation and cache source coverage, +commitments and pending gates. Historical registrations remain unchanged. +The inherited seed commitments remain unverified without private sources; +these registrations do not authorize collection or relay-backed confinement. diff --git a/analysis/freeze/cse-causal-study-pilot-v10/declaration.json b/analysis/freeze/cse-causal-study-pilot-v10/declaration.json new file mode 100644 index 00000000..d7e332c1 --- /dev/null +++ b/analysis/freeze/cse-causal-study-pilot-v10/declaration.json @@ -0,0 +1,393 @@ +{ + "analysis": { + "scripts": [ + "src/aisle/harness/benchmark_statistics.py", + "src/aisle/harness/causal_study.py", + "src/aisle/harness/pilot_records.py" + ], + "seed": 500001 + }, + "artifacts": { + "agent_adapters": "tools/agent_adapters.py", + "analyzer": "src/aisle/harness/benchmark_statistics.py", + "bnd_calibration_manifest": "analysis/freeze/bnd-task-band-calibration-v11/freeze-manifest.json", + "bnd_perception_audit_report": "analysis/perception-audit/records/bnd-perception-corpus-03/report.json", + "budget_ceiling": "harness/budget.toml", + "dependency_declaration": "pyproject.toml", + "dependency_lock": "uv.lock", + "dora_runtime_selection": "dora-runtime.json", + "frontend_request_authorization_documentation": "docs/benchmark/v1/frontend-request-authorization.md", + "matched_session_decision": "docs/decisions/ADR-62.md", + "matched_session_documentation": "docs/monolithic/matched-session.md", + "matched_source:assets/so101/so101.urdf": "assets/so101/so101.urdf", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "docs/monolithic/allowlist-t1-l2.json", + "matched_source:docs/monolithic/candidates.json": "docs/monolithic/candidates.json", + "matched_source:docs/monolithic/experts-t1-l2.json": "docs/monolithic/experts-t1-l2.json", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "docs/monolithic/interface-map-t1-l2.json", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "docs/monolithic/parity-protocol-t1-l2.json", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "docs/monolithic/treatment-table-t1-l2.json", + "matched_source:env/limits.toml": "env/limits.toml", + "matched_source:experts/monolithic/expert_t1_l2.py": "experts/monolithic/expert_t1_l2.py", + "matched_source:graphs/pilot_t1_l2_monolithic.yaml": "graphs/pilot_t1_l2_monolithic.yaml", + "matched_source:src/aisle/__init__.py": "src/aisle/__init__.py", + "matched_source:src/aisle/embodiment.py": "src/aisle/embodiment.py", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "src/aisle/harness/_code_mode_protocol/__init__.py", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "src/aisle/harness/_code_mode_protocol/descriptor.pb", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "src/aisle/harness/_code_mode_protocol/protocol.proto", + "matched_source:src/aisle/harness/candidate_launch.py": "src/aisle/harness/candidate_launch.py", + "matched_source:src/aisle/harness/candidates.py": "src/aisle/harness/candidates.py", + "matched_source:src/aisle/harness/cli.py": "src/aisle/harness/cli.py", + "matched_source:src/aisle/harness/code_mode_audit.py": "src/aisle/harness/code_mode_audit.py", + "matched_source:src/aisle/harness/code_mode_authority.py": "src/aisle/harness/code_mode_authority.py", + "matched_source:src/aisle/harness/code_mode_proxy.py": "src/aisle/harness/code_mode_proxy.py", + "matched_source:src/aisle/harness/code_mode_runner.py": "src/aisle/harness/code_mode_runner.py", + "matched_source:src/aisle/harness/common.py": "src/aisle/harness/common.py", + "matched_source:src/aisle/harness/frontend_app_server.py": "src/aisle/harness/frontend_app_server.py", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "src/aisle/harness/frontend_app_server_audit.py", + "matched_source:src/aisle/harness/frontend_dispatch.py": "src/aisle/harness/frontend_dispatch.py", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "src/aisle/harness/frontend_dispatch_audit.py", + "matched_source:src/aisle/harness/frontend_request_audit.py": "src/aisle/harness/frontend_request_audit.py", + "matched_source:src/aisle/harness/frontend_request_authority.py": "src/aisle/harness/frontend_request_authority.py", + "matched_source:src/aisle/harness/guard_divergence.py": "src/aisle/harness/guard_divergence.py", + "matched_source:src/aisle/harness/heldout_eval.py": "src/aisle/harness/heldout_eval.py", + "matched_source:src/aisle/harness/matched_app_server.py": "src/aisle/harness/matched_app_server.py", + "matched_source:src/aisle/harness/matched_collection.py": "src/aisle/harness/matched_collection.py", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "src/aisle/harness/matched_dynamic_run.py", + "matched_source:src/aisle/harness/matched_evidence.py": "src/aisle/harness/matched_evidence.py", + "matched_source:src/aisle/harness/matched_frontend.py": "src/aisle/harness/matched_frontend.py", + "matched_source:src/aisle/harness/matched_run.py": "src/aisle/harness/matched_run.py", + "matched_source:src/aisle/harness/matched_run_launch.py": "src/aisle/harness/matched_run_launch.py", + "matched_source:src/aisle/harness/matched_runtime.py": "src/aisle/harness/matched_runtime.py", + "matched_source:src/aisle/harness/matched_session.py": "src/aisle/harness/matched_session.py", + "matched_source:src/aisle/harness/matched_tool_service.py": "src/aisle/harness/matched_tool_service.py", + "matched_source:src/aisle/harness/matched_tools.py": "src/aisle/harness/matched_tools.py", + "matched_source:src/aisle/harness/monolith.py": "src/aisle/harness/monolith.py", + "matched_source:src/aisle/harness/monolithic.py": "src/aisle/harness/monolithic.py", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "src/aisle/harness/monolithic_run_evidence.py", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "src/aisle/harness/monolithic_run_prepare.py", + "matched_source:src/aisle/harness/pilot_records.py": "src/aisle/harness/pilot_records.py", + "matched_source:src/aisle/harness/provider_relay.py": "src/aisle/harness/provider_relay.py", + "matched_source:src/aisle/harness/provider_response_authority.py": "src/aisle/harness/provider_response_authority.py", + "matched_source:src/aisle/harness/provider_runner.py": "src/aisle/harness/provider_runner.py", + "matched_source:src/aisle/harness/provider_source_audit.py": "src/aisle/harness/provider_source_audit.py", + "matched_source:src/aisle/harness/registry.py": "src/aisle/harness/registry.py", + "matched_source:src/aisle/harness/rollout.py": "src/aisle/harness/rollout.py", + "matched_source:src/aisle/harness/rollout_client.py": "src/aisle/harness/rollout_client.py", + "matched_source:src/aisle/harness/simulator_work.py": "src/aisle/harness/simulator_work.py", + "matched_source:src/aisle/harness/traces.py": "src/aisle/harness/traces.py", + "matched_source:src/aisle/harness/treatment_ambient.py": "src/aisle/harness/treatment_ambient.py", + "matched_source:src/aisle/harness/treatment_confinement.py": "src/aisle/harness/treatment_confinement.py", + "matched_source:src/aisle/harness/treatment_integrity.py": "src/aisle/harness/treatment_integrity.py", + "matched_source:src/aisle/harness/treatment_postflight.py": "src/aisle/harness/treatment_postflight.py", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "src/aisle/harness/typed_execution_bundle.py", + "matched_source:src/aisle/harness/typed_graph_audit.py": "src/aisle/harness/typed_graph_audit.py", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "src/aisle/harness/typed_graph_hosts.py", + "matched_source:src/aisle/harness/typed_graph_stage.py": "src/aisle/harness/typed_graph_stage.py", + "matched_source:src/aisle/harness/typed_node_host.py": "src/aisle/harness/typed_node_host.py", + "matched_source:src/aisle/harness/typed_node_requests.py": "src/aisle/harness/typed_node_requests.py", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "src/aisle/harness/typed_node_supervisor.py", + "matched_source:src/aisle/harness/typed_node_worker.py": "src/aisle/harness/typed_node_worker.py", + "matched_source:src/aisle/harness/typed_run_prepare.py": "src/aisle/harness/typed_run_prepare.py", + "matched_source:src/aisle/harness/typed_snapshot.py": "src/aisle/harness/typed_snapshot.py", + "matched_source:src/aisle/harness/typed_stage_provider.py": "src/aisle/harness/typed_stage_provider.py", + "matched_source:src/aisle/harness/typed_validation.py": "src/aisle/harness/typed_validation.py", + "matched_source:src/aisle/harness/typed_worker_launch.py": "src/aisle/harness/typed_worker_launch.py", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "src/aisle/harness/typed_worker_provisioning.py", + "matched_source:src/aisle/harness/validate.py": "src/aisle/harness/validate.py", + "matched_source:src/aisle/harness/worker_authority_probe.py": "src/aisle/harness/worker_authority_probe.py", + "matched_source:src/aisle/harness/worker_capability.py": "src/aisle/harness/worker_capability.py", + "matched_source:src/aisle/harness/worker_declaration.py": "src/aisle/harness/worker_declaration.py", + "matched_source:src/aisle/harness/worker_network_probe.py": "src/aisle/harness/worker_network_probe.py", + "matched_source:src/aisle/kinematics.py": "src/aisle/kinematics.py", + "matched_source:src/aisle/mobility/guard.py": "src/aisle/mobility/guard.py", + "matched_source:src/aisle/monolith/confinement.py": "src/aisle/monolith/confinement.py", + "matched_source:src/aisle/monolith/primitive_api.py": "src/aisle/monolith/primitive_api.py", + "matched_source:src/aisle/monolith/primitives.py": "src/aisle/monolith/primitives.py", + "matched_source:src/aisle/monolith/proxy.py": "src/aisle/monolith/proxy.py", + "matched_source:src/aisle/monolith/requests.py": "src/aisle/monolith/requests.py", + "matched_source:src/aisle/monolith/supervisor.py": "src/aisle/monolith/supervisor.py", + "matched_source:src/aisle/monolith/wire.py": "src/aisle/monolith/wire.py", + "matched_source:src/aisle/monolith/worker.py": "src/aisle/monolith/worker.py", + "matched_source:src/aisle/monolith/worker_config.py": "src/aisle/monolith/worker_config.py", + "matched_source:src/aisle/monolith/worker_launch.py": "src/aisle/monolith/worker_launch.py", + "matched_source:src/aisle/nodes/budget_guard.py": "src/aisle/nodes/budget_guard.py", + "matched_source:src/aisle/nodes/h6_fault.py": "src/aisle/nodes/h6_fault.py", + "matched_source:src/aisle/nodes/l2_pose.py": "src/aisle/nodes/l2_pose.py", + "matched_source:src/aisle/nodes/label_reader.py": "src/aisle/nodes/label_reader.py", + "matched_source:src/aisle/nodes/monolith_broker.py": "src/aisle/nodes/monolith_broker.py", + "matched_source:src/aisle/nodes/perception_session.py": "src/aisle/nodes/perception_session.py", + "matched_source:src/aisle/scenes/locations.toml": "src/aisle/scenes/locations.toml", + "matched_source:src/aisle/scenes/meds.toml": "src/aisle/scenes/meds.toml", + "matched_source:src/aisle/scenes/pharmacy.py": "src/aisle/scenes/pharmacy.py", + "matched_source:src/aisle/scenes/physics.toml": "src/aisle/scenes/physics.toml", + "matched_source:src/aisle/scenes/planogram.toml": "src/aisle/scenes/planogram.toml", + "matched_source:src/aisle/scenes/store.py": "src/aisle/scenes/store.py", + "matched_source:src/aisle/topics.py": "src/aisle/topics.py", + "matched_source:src/aisle/turn_node.py": "src/aisle/turn_node.py", + "matched_source:src/aisle/turns.py": "src/aisle/turns.py", + "matched_source:src/aisle/verifier/calibration.py": "src/aisle/verifier/calibration.py", + "matched_source:src/aisle/verifier/models.lock": "src/aisle/verifier/models.lock", + "matched_source:src/aisle/verifier/models.py": "src/aisle/verifier/models.py", + "matched_source:src/aisle/verifier/placement.toml": "src/aisle/verifier/placement.toml", + "matched_source:src/aisle/verifier/realistic.py": "src/aisle/verifier/realistic.py", + "matched_source:src/aisle/verifier/stages.py": "src/aisle/verifier/stages.py", + "matched_source:src/aisle/verifier/thresholds.toml": "src/aisle/verifier/thresholds.toml", + "matched_source:tools/campaign.py": "tools/campaign.py", + "matched_source:tools/matched_campaign.py": "tools/matched_campaign.py", + "monolithic_validators": "src/aisle/harness/monolithic.py", + "nested_host_documentation": "docs/monolithic/code-mode-admission.md", + "pilot_assignments_cli": "tools/pilot_assignments.py", + "pilot_decision": "docs/decisions/ADR-66.md", + "pilot_records_cli": "tools/pilot_records.py", + "protocol": "analysis/freeze/cse-causal-study-pilot-v8/protocol.json", + "protocol_template": "analysis/statistics/protocol-template.json", + "provider_admission_documentation": "docs/monolithic/provider-admission.md", + "public_observation_source:analysis/monolithic-control/experts-t1-l2-v1.json": "analysis/monolithic-control/experts-t1-l2-v1.json", + "public_observation_source:analysis/monolithic-control/treatment-table-t1-l2-v1.json": "analysis/monolithic-control/treatment-table-t1-l2-v1.json", + "public_observation_source:docs/decisions/ADR-pilot-public-observations.md": "docs/decisions/ADR-pilot-public-observations.md", + "public_observation_source:docs/monolithic/public-observations.md": "docs/monolithic/public-observations.md", + "public_observation_source:graphs/pilot_t1_l2_monolithic.yaml": "graphs/pilot_t1_l2_monolithic.yaml", + "public_observation_source:graphs/pilot_t1_l2_typed.yaml": "graphs/pilot_t1_l2_typed.yaml", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_monolithic.json": "graphs/turn_plans/pilot_t1_l2_monolithic.json", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_typed.json": "graphs/turn_plans/pilot_t1_l2_typed.json", + "public_observation_source:registry/manifests/pilot-policy-surface.yaml": "registry/manifests/pilot-policy-surface.yaml", + "public_observation_source:src/aisle/harness/pilot_model_cache.py": "src/aisle/harness/pilot_model_cache.py", + "public_observation_source:src/aisle/harness/pilot_policy_surface.py": "src/aisle/harness/pilot_policy_surface.py", + "research_prompt": "harness/CLAUDE.research.md", + "seed_commitment_predecessor": "analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json", + "session_runner": "tools/campaign.py", + "session_validators": "src/aisle/harness/causal_study.py", + "simulator_bridge": "src/aisle/nodes/dora_genesis.py", + "spec": "specs/500-typed-monolithic-causal-study.md", + "treatment_integrity": "src/aisle/harness/treatment_integrity.py", + "treatment_postflight": "src/aisle/harness/treatment_postflight.py", + "treatment_table": "analysis/monolithic-control/treatment-table-v1.json" + }, + "budgets": { + "campaign_ceiling": "harness/budget.toml", + "rollouts_per_session": 20, + "session_tokens": 200000, + "session_wall_s": 3600, + "sim_episodes_per_session": 160 + }, + "campaign_id": "cse-causal-study-pilot-v10", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-pilot-v8/protocol.json --purpose power", + "uv run harness stats power --protocol analysis/freeze/cse-causal-study-pilot-v8/protocol.json --output analysis/freeze/cse-causal-study-pilot-v8/power.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json --allow-withheld-seeds" + ], + "decision_rules": { + "alpha": 0.05, + "equivalence_margin": { + "cost_at_equal_quality": { + "acceptance_probability": 0.1, + "heldout_quality": 0.1, + "rule": "CSE-5: both must pass or cost claims stay unconditional", + "sidedness": "two-sided 0.90" + } + }, + "multiplicity": "single primary endpoint", + "power": 0.8, + "primary_rule": "one-sided superiority at alpha 0.05 on the pooled Newcombe interval lower bound > 0 with no reversed stratum; otherwise weakened/rejected/undecidable by the #358 rule (CSE-3)", + "smallest_effect": { + "measure": "risk difference in session success", + "value": 0.25 + }, + "stopping_rule": "fixed n from the frozen power analysis using pilot-only control success; absolute floor 10 randomized assignments per arm per agent-system x task stratum (CSE-8); no optional stopping or extension" + }, + "disposition": "registered 2026-09-13; collection pending. Supersedes the zero-session pilot-v1 lineage with the instrument set the pilot can actually run on (T1-L2 realistic pair, #581 to #586). Named limitations: BND-7 not eligible (recorded failed), external-review and parity gates pending, and the session-bound capability audit not merged (an unattested environment stops every session).", + "endpoints": { + "primary": [ + { + "aggregation": "risk difference typed_dataflow - monolithic; exact arm intervals; Newcombe interval; per agent-system x task stratum before pooling (CSE-1, CSE-17)", + "direction": "higher", + "id": "E-CSE-1", + "outcome": "session_success", + "status": "inferential", + "unit": "agent_session" + } + ], + "secondary": [ + { + "aggregation": "counts by arm", + "direction": "descriptive", + "id": "E-CSE-2", + "outcome": "validation_and_launch_failure_class", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "Kaplan-Meier with censoring", + "direction": "lower", + "id": "E-CSE-3", + "outcome": "time_to_first_accepted_system", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "unconditional session estimands; no at-equal-quality wording unless CSE-5 equivalence passes", + "direction": "lower", + "id": "E-CSE-4", + "outcome": "tokens_tool_calls_wall_active_time_sim_work_rollouts", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "counts", + "direction": "descriptive", + "id": "E-CSE-5", + "outcome": "safety_interventions", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "dropped unless a blinded rubric is ratified before freeze (CSE-18)", + "direction": "n/a", + "id": "E-CSE-6", + "outcome": "artifact_legibility", + "status": "exploratory", + "unit": "agent_session" + } + ] + }, + "exclusions": { + "deviation_policy": "any behaviour-affecting change after registration is a documented deviation record and a new campaign version id; confirmatory data never revises the registered protocol", + "infrastructure": [ + "independent-controller-determined, arm-neutral facts only (CSE-14): host outage, provider interruption before first tool call, launcher crash before the session started" + ], + "rerun_policy": "a permitted rerun receives a new randomized id; the original row is retained with its exclusion reason and never replaced in place", + "treatment_integrity": [ + "contamination sentinel tripped (CSE-12)", + "model alias or CLI drift outside the frozen identity (CSE-6)", + "integrity verdict authored session-side (CSE-11)" + ] + }, + "hypotheses": [ + { + "direction": "typed_dataflow - monolithic > 0", + "id": "H-CSE-1", + "statement": "The bundled typed-dataflow surface raises the probability that a fresh coding-agent session produces a launching system passing the hidden held-out acceptance within budget, versus the equal-capability monolithic surface." + } + ], + "instrument_set": { + "agent_systems": [ + { + "binary": "codex", + "binary_sha256": "b973d440acac501fd2594a43e7ca9ce41e0a65b9dfb28d0d7a7837c99e1261e3", + "frozen": true, + "id": "codex-cli", + "install": "private prefix ~/aisle-private/codex-0.153.4 (the qualified darwin-arm64 binary, not the JS shim); ChatGPT-login credentials copied byte-identically into both arms' private CODEX_HOME, hash recorded only", + "version_observed": "0.153.4" + } + ], + "candidate": { + "id": "t1-l2-realistic", + "monolithic_module": "experts/monolithic/expert_t1_l2.py", + "monolithic_template": "graphs/monolithic_t1_l2.yaml", + "perception": "L2", + "reset": "teleport", + "typed_graph": "graphs/expert_t1_l2.yaml", + "verifier": "realistic (oracle demoted to the held-out scoring tap, BND-3)" + }, + "candidate_table_sha256": "08a1dc56502abc4673cb0b54f01664382214bb473c3ef293b1df7b6c74f64778", + "items": [ + "short_composition:t1-l2-realistic" + ], + "kind": "selected_task" + }, + "integrity_checks": [ + { + "gate": "CON-14 spec approval", + "kind": "external_review", + "owner_role": "project owner", + "pilot_record_2026_09_13": "pending; external review, not a pilot prerequisite (ADR-66 section 2); named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "STA-12 independent statistical review", + "kind": "external_review", + "owner_role": "independent statistician", + "pilot_record_2026_09_13": "pending; external review, not a pilot prerequisite (ADR-66 section 2); named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "MON-1 treatment table and expert parity (#344)", + "kind": "machine_check", + "owner_role": "parity harness; control surface exists, independent authorship and operator evidence remain pending", + "pilot_record_2026_09_13": "pending; the T1-L2 pair (#583) passes the interface, table and module checks but has no operator parity record at an immutable hash; named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "BND-12 task-band freeze (#346)", + "kind": "machine_check", + "owner_role": "task-band auditor", + "pilot_record_2026_09_13": "recorded failed: bnd-task-band-calibration-v11 regenerated the BND-5 perception audit on a 32-seed T1-L2 corpus; 22 of 22 strata below the 0.90 floor (BND-7), led by target_class=ibuprofen refusals; role open (BND-10). ADR-66 section 2: a failing machine gate is a named limitation of this pilot, not a blocker", + "record": "analysis/perception-audit/records/bnd-perception-corpus-03/report.json", + "status": "failed" + }, + { + "gate": "TRT confinement/mutation/agent-path (#353)", + "kind": "machine_check", + "owner_role": "integrity controller", + "pilot_record_2026_09_13": "pending; the session-bound capability audit (loopback relay policy) is not merged; an unattested environment is a stop condition for every session", + "record": null, + "status": "pending" + }, + { + "gate": "seed commitment verification", + "kind": "machine_check", + "owner_role": "holder of the original private seed and salt sources", + "pilot_record_2026_09_13": "not applicable to this registration: it commits a fresh private seed source directly (no inheritance); kept for shape parity and left pending", + "public_observation_successor_note": "Inherited from pilot-v8 without access to private seed sources; verification remains pending. This byte-binding successor does not authorize a pilot run.", + "record": null, + "status": "pending" + }, + { + "gate": "MON-8/MON-12 matched-session acceptance (#519)", + "kind": "machine_check", + "owner_role": "matched-session controller; complete acceptance evidence still required", + "pilot_record_2026_09_13": "pending; frontend conformance qualified on the T1 oracle development surface only (#568); the T1-L2 realistic surface runs under the same executor with the same admission checks; named limitation", + "record": null, + "status": "pending" + } + ], + "issue": 347, + "pilot_design": { + "agent_system": "codex-cli 0.153.4 only (the owned provider relay, the ChatGPT-login account header and the pinned conformance profile are Codex-bound)", + "arm_order": "sealed balanced plan (revealed one assignment at a time); no longer a fixed alternation", + "arms": [ + "typed", + "monolithic" + ], + "assignment": "tools/pilot_assignments.py create --protocol analysis/freeze/cse-causal-study-pilot-v8/protocol.json (sealed balanced plan, one typed and one monolithic assignment per temporal block, TRT-8/CSE-7); reveal exactly one assignment before each session; the ledger retains every assignment, started or not", + "below_cse8_floor": "6 per arm is below the CSE-8 floor of 10 by design; this registration can never become a confirmatory estimate (STA-3)", + "decision": "docs/decisions/ADR-66.md (pilot first; agent-drafted under the ADR-65 label, renumbered by #568)", + "executor": "tools/matched_campaign.py admit / run --purpose engineering / collect-run, then the held-out evaluation; retained and unscored, output root ~/aisle-private/raw/cse-causal-study-pilot-v8/", + "records": "tools/pilot_records.py derives one aisle.stats.records.v1 row per ledger assignment; harness stats validate --purpose analysis; harness stats analyze; harness stats power from observed control success (CSE-8)", + "reporting": "analysis/reports/cse_pilot_v8.md; results enter docs/claim-evidence.yaml only as CSE-8 power inputs under the unrun confirmatory claim", + "rerun_policy": "no reruns; a failed attempt is retained under a new session id, never overwritten", + "session_success": "controller-run held-out evaluation of the retained final snapshot on the 32 private seeds after the agent process ends, oracle as the held-out scorer (BND-3); success iff at least 28 of 32 oracle successes; a launch failure counts as a failed seed (CSE-2, CSE-9)", + "sessions_per_arm": 6, + "stop_conditions": [ + "wrong_object outcome", + "guard bypass", + "unattested environment", + "relay bypass (provider traffic outside the owned relay)", + "any oracle field on a policy input" + ], + "task_stratum": "short_composition on candidate t1-l2-realistic (T1 franka, L2 perception, realistic verifier); bnd-task-band-calibration-v11 records it as not perception-eligible (BND-7), carried as a machine-gate limitation per ADR-66" + }, + "purpose": "pilot", + "schema_version": "aisle.freeze.declaration.v1", + "seed_commitment": { + "inherited_from": "analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json", + "rule": "32 distinct integers in [1000,9999) drawn once from the OS CSPRNG on 2026-09-13, retained only under ~/aisle-private/freeze/cse-causal-study-pilot-v8/, disjoint by value from every private seed list committed so far (including cse-causal-study-pilot-v1 and the cse-causal-study-v1 confirmatory commitment) and from the committed development seed lists; never inherited by a confirmatory declaration (ADR-66); revealed after the pilot closes. The same directory holds the controller-private 256-bit randomization seed for the sealed assignment plan (TRT-8); its commitment is recorded in the assignment ledger, never here.", + "salt_source": "~/aisle-private/freeze/cse-causal-study-pilot-v8/salt.bin", + "seeds_source": "~/aisle-private/freeze/cse-causal-study-pilot-v8/seeds.json" + }, + "spec": "500", + "superseded": "cse-causal-study-pilot-v9 (session-bound capability audits and fail-closed rejection of unsupported loopback endpoint authority; public-observation and cache source coverage, study rules, inherited seed commitment and pending gates preserved; no collection authorization)" +} diff --git a/analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json b/analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json new file mode 100644 index 00000000..9450562c --- /dev/null +++ b/analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json @@ -0,0 +1,584 @@ +{ + "analysis_script_hashes": { + "src/aisle/harness/benchmark_statistics.py": "sha256:22cde3e9d7c5cc80874c7eec30aa2898e30476d6c939c94ab3009e8b12af641e", + "src/aisle/harness/causal_study.py": "sha256:1b1c53d6389881c146726d865a741a681f3d5039c9bf99558a53ee368250971e", + "src/aisle/harness/pilot_records.py": "sha256:68ac7ef9afd449d8527e2ffe99a9ef926e06253c466f3421bc1848e9aa094b70" + }, + "analysis_seed": 500001, + "artifact_hashes": { + "agent_adapters": "sha256:848f8f7da5ad978e53712991a688eb74f56c7a77deabee8afae717ad712666ca", + "analyzer": "sha256:22cde3e9d7c5cc80874c7eec30aa2898e30476d6c939c94ab3009e8b12af641e", + "bnd_calibration_manifest": "sha256:8615de2078337065bf6045c3e4bb0f92924987606768fdb699a8ce58ce8f8b57", + "bnd_perception_audit_report": "sha256:27134904b4b739423085a9b3c9b23678658d39f7a48e00cd2fbeaedea8c7a4d8", + "budget_ceiling": "sha256:bcbb79aa146afed1d877da18093175ae144b4607db50119022ef4f8173122cda", + "dependency_declaration": "sha256:add4e718001da7678b404bdf37ea8d9889f563324ffba9b03fe65c46b172ee4c", + "dependency_lock": "sha256:9b8a82a17ba68c192f045405feed3ae7046af9568bddcb398292c20f0ed6f5a8", + "dora_runtime_selection": "sha256:f9eb7ffe285a0f0ed26d50303e31cf1e94d2f93ed62b599071dff5a183c15f53", + "frontend_request_authorization_documentation": "sha256:a1d4e65c0e467af4bd77922b95b12f26ee6478932eb5dd460de34b81249ca86b", + "matched_session_decision": "sha256:d88e0ef18c92cedff7b74dd3ffafed14d92affaec65beb9ddfec2801b365dc61", + "matched_session_documentation": "sha256:735e3e5d2032e7d2c3901d9b631181de4a345027a9afe29ef4dd9f9ae17aa1f4", + "matched_source:assets/so101/so101.urdf": "sha256:3a65d2d35e68a8d2f0c2cc176d19b884506543c93ba72980145b80abe276022c", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "sha256:6aac85db63a7061ab09950741eba8ac0e77410755d85c427731c45864adc1dc5", + "matched_source:docs/monolithic/candidates.json": "sha256:2fc51e6ff0e4b73a9b203d6ed1debf1cfa40ece9b21a35245d50b9a403f80abb", + "matched_source:docs/monolithic/experts-t1-l2.json": "sha256:68f01e1081afb36964f591fe0e58a630c8e734c8809833279ac608b1c554fe1a", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "sha256:31fede313d1573e29a83c42acc92883ae8ffeb36beee0a62a8d4481ad68cd591", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "sha256:2bfc9854fb5cffbe70be341799f208813eee8c02c9d54993b83ce86f6b80f054", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "sha256:544bd956688b8d85b54ff496dc9d98bdc42676c1c0c5fe5583e276614d4ed0d5", + "matched_source:env/limits.toml": "sha256:1ab1f4fd5d52e7949b81d21933daaa2c66e80ef07fdedf8217280468e0142edc", + "matched_source:experts/monolithic/expert_t1_l2.py": "sha256:93ade82db523fe808312f9d83c88ab3464eee303be1bfc3f824880bf1de7924c", + "matched_source:graphs/pilot_t1_l2_monolithic.yaml": "sha256:ebf12538d0b9def23ce4ea70a9feb375c98535f774bbf16d4d839aaa4e99d27c", + "matched_source:src/aisle/__init__.py": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "matched_source:src/aisle/embodiment.py": "sha256:d17c204492d680175cbaf365aaf164e2a6049da23ce125a9b9f21606b79f4cca", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "sha256:7e7fb09527b647c2bb13787011de8f9a4412e8b6f304e89efa107162d8150e82", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "sha256:459465e666c6a76a70e902c10c58905dfd8417b6c2ca3e038be99624e3542540", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "sha256:406d8f47d8d0b95a9d4ae104890e5e52764c9597e9271f75ff434a4ce24748d0", + "matched_source:src/aisle/harness/candidate_launch.py": "sha256:654eb90264703e3ce3ab599bb8e143047eaee288c8852c5b09c74e3147039270", + "matched_source:src/aisle/harness/candidates.py": "sha256:d1b441a30288a6c0500aaa765a24d4aeecca4f8d52e96f633befc50c84ecf5f5", + "matched_source:src/aisle/harness/cli.py": "sha256:c855f059d28409ffaa14be99ad3fc0bdc16fc33cf1a3c9584a75d968bd59d72d", + "matched_source:src/aisle/harness/code_mode_audit.py": "sha256:24638163f205408da1c6cf20a1b77c80a0d740a3f8ed44696eb90baf7de61f6a", + "matched_source:src/aisle/harness/code_mode_authority.py": "sha256:217ea025f10a4dc02b4b0ff2ff733adf4a695d1098b3c8ce0d2b27b8a140853d", + "matched_source:src/aisle/harness/code_mode_proxy.py": "sha256:333eee561d610489d2f48f9359ad94a3d9d8ff729fbae0fd3dec4acf64be2cb2", + "matched_source:src/aisle/harness/code_mode_runner.py": "sha256:4f91577dcfc6c239bf4111bd6735a64e97d6e22b7ca6287389abae09023cc25d", + "matched_source:src/aisle/harness/common.py": "sha256:ef62bf518a99c8c4154908bc9cf99e68402e6ae6c8e3fb5ec30c3f5fce9b5fac", + "matched_source:src/aisle/harness/frontend_app_server.py": "sha256:d21b9bf218efc3066399b8b22bbe3892e772433da315a9ecd40e2773369e9283", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "sha256:fb3cf5b8a115eb93e3f292dae694a4318cbe43e9552347fbc25ea8e43102ddbb", + "matched_source:src/aisle/harness/frontend_dispatch.py": "sha256:6bff2abbacca6c332abdb5c5cb4cb3d688d8b00535e3f428633619bf91cd30c8", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "sha256:dc75024bd52c21c89a6112da847875a1d0ebf44f53a3704705e0bb8c81b9bc4a", + "matched_source:src/aisle/harness/frontend_request_audit.py": "sha256:a4bc60b2ce1968ada1776324b72ad87025b93d366046029445fdf9b2e92f8830", + "matched_source:src/aisle/harness/frontend_request_authority.py": "sha256:0a061f1528b8bb2071dd1ba7e1d0bbee178b2c982bd6b688b916fb612522e163", + "matched_source:src/aisle/harness/guard_divergence.py": "sha256:af2c4755cabe46fabba913aee4819e05c10caaae79105615d50f8cf4567dad14", + "matched_source:src/aisle/harness/heldout_eval.py": "sha256:56c83164b04ae1b4623e8fd7a0579dd0a9caa7d5dbb891448643464a07893962", + "matched_source:src/aisle/harness/matched_app_server.py": "sha256:b7177caca8f1e4d3664b6cef76539b572c2ea6604fe0799c629bc50a52951b52", + "matched_source:src/aisle/harness/matched_collection.py": "sha256:b347127344a4c15fbb73fa9f3ac7c711a3d55b0fa368a11f76683bb1b5bbea14", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "sha256:b7f2b81397bcd232710408b9c6c9f463b3928e37f117d7310d519728f9b4b2c9", + "matched_source:src/aisle/harness/matched_evidence.py": "sha256:6d65476a9a923fd445ea9dad0334e521378353454b873ef59e2b0ff848c650f7", + "matched_source:src/aisle/harness/matched_frontend.py": "sha256:a4b9827af7dbf36f4af5228ca8c87ea36d2d32f8cffc7fcb8fea1923bcc09401", + "matched_source:src/aisle/harness/matched_run.py": "sha256:825fec659b8bdbb104d6fa07fa2adf29f4b23be74c1f3ec894a8ffcab6b68b64", + "matched_source:src/aisle/harness/matched_run_launch.py": "sha256:8701f034d0bd9873aa7981e3db47849d50bd4465f085e617e6e3193aac070539", + "matched_source:src/aisle/harness/matched_runtime.py": "sha256:c4d81e8309fcf5212315013dbf8e83347a65080e1221f37a0491757834ac2797", + "matched_source:src/aisle/harness/matched_session.py": "sha256:344ea0bc92705ef23f550045f1d4b26d55c7e2cf894133354c22068804887b21", + "matched_source:src/aisle/harness/matched_tool_service.py": "sha256:1bf6685addbeee541961e1053b2eea68cba66ce110667d8e7720398d49191a17", + "matched_source:src/aisle/harness/matched_tools.py": "sha256:eff206d64477d6a91f88c4889e2934a17bba254612c7dc72d542e799e11ef948", + "matched_source:src/aisle/harness/monolith.py": "sha256:a12428eab867b869d902733e43626669b58f87218f0c40d1713712c094ef7414", + "matched_source:src/aisle/harness/monolithic.py": "sha256:1983ed8c26b2371c993c571f33f5c7e4ee81ea2597f9468b43d53c9970465ccf", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "sha256:4c0cb658f0148eb5d06ad69dceafd39148a2d4787e085cb624047c365c87bb24", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "sha256:23f2646571d1ed67c5b8e09e7b0b10bcc6cfa996b05b3161fe39f859d88c15ba", + "matched_source:src/aisle/harness/pilot_records.py": "sha256:68ac7ef9afd449d8527e2ffe99a9ef926e06253c466f3421bc1848e9aa094b70", + "matched_source:src/aisle/harness/provider_relay.py": "sha256:0b1698ac82da51458da5d4cdee6ca9f79334703d8c4a9dbc297af173ec0ff9a2", + "matched_source:src/aisle/harness/provider_response_authority.py": "sha256:9562a8de28967e2eb83910ea5062dc491c8588a6bb4a2a04407f44d59f2251fe", + "matched_source:src/aisle/harness/provider_runner.py": "sha256:2c39d11d67cc4da2f9a2c732e86ba4e9a4b8da88570bdb1ecd1364d4668831cb", + "matched_source:src/aisle/harness/provider_source_audit.py": "sha256:e3be1f9d70288b4f596427a39ab4274399e9fdc74e87043e3c1249596fe1872b", + "matched_source:src/aisle/harness/registry.py": "sha256:214da7bb8db26c80c79262fa79c40a624d8a9981463d38a9c4aa30bc989bc1fb", + "matched_source:src/aisle/harness/rollout.py": "sha256:371789bd5093ef5c9e5ebb92177de6470caf5ec93ceeb59627411852fa62684a", + "matched_source:src/aisle/harness/rollout_client.py": "sha256:18808db0aa2df356d1a7cc9a538585aa9dcc653abd120e761649f1dff2c27c31", + "matched_source:src/aisle/harness/simulator_work.py": "sha256:2ade87416ab8c5d8902e9734af850b7613ee9ee42f9ca672c9482625e15d3d91", + "matched_source:src/aisle/harness/traces.py": "sha256:c2e6a717a56ce47e296bdc0115b15de9361b72084638b4f3bc66f7e7d032546b", + "matched_source:src/aisle/harness/treatment_ambient.py": "sha256:13dce400a649bc0d01749661517138f8ad1f8cde106caceaedb9f33af6f6d856", + "matched_source:src/aisle/harness/treatment_confinement.py": "sha256:c46e448a63041119f1d0dbdb136ae982459d64a305867dae114e480410ed4f6c", + "matched_source:src/aisle/harness/treatment_integrity.py": "sha256:b37162c6f295c5686da9b3089eb3bd96b69f68f01f41da0a784c4017658b10a6", + "matched_source:src/aisle/harness/treatment_postflight.py": "sha256:0bf01138ef0c277c6fcd728fcf25f4f13795de679fb41ab58b868b81f6c2f0ca", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "sha256:607945d5a51e3a139ecd35a8dceaf77a2027b0f842a577587616e0490b35c663", + "matched_source:src/aisle/harness/typed_graph_audit.py": "sha256:ca315a2717668f6e1029c0940471df6320eb23af859a0bf41220180bbcae656c", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "sha256:a5cfd9b3b603f901370579f65320f995d4bc0c71852cf2ed64784d429a942abd", + "matched_source:src/aisle/harness/typed_graph_stage.py": "sha256:0ea37ad396fb3321978c5f8f63d869a8ba0a652f7377cad60f13060b5a5ccbfb", + "matched_source:src/aisle/harness/typed_node_host.py": "sha256:56a5f0ab8159071897aad22d49aae2ab1785e6de9a186412b7b5566db334d056", + "matched_source:src/aisle/harness/typed_node_requests.py": "sha256:c3712381bc7383beef1e3304ce05f6a4fe27fb16d9b7e97e7f51f8d7cb6de78b", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "sha256:d964517b767bdd087771bf9233c05de61604cf5f705124d6b83be05feb0cb577", + "matched_source:src/aisle/harness/typed_node_worker.py": "sha256:890cb125e12b05afa3609186ceae6d1d45d1d765f653e2c0b7345b00b2c24a89", + "matched_source:src/aisle/harness/typed_run_prepare.py": "sha256:787ba2edc04efa6df752fdba2112ff50e34d2061acb74471c52c78e813382063", + "matched_source:src/aisle/harness/typed_snapshot.py": "sha256:96003b648f09d1348375ca72709537dd80aa4ddecd60e87465011d221d3200ec", + "matched_source:src/aisle/harness/typed_stage_provider.py": "sha256:afe7e37ef08020f5bf19630ef9117af2bdf9d8fe68ceba7f4a352f7c61c8c9ff", + "matched_source:src/aisle/harness/typed_validation.py": "sha256:bf51adce9f8fe107a2586bb4ffe2aec0cff684a235b6fde5837a95223f37ffa2", + "matched_source:src/aisle/harness/typed_worker_launch.py": "sha256:0759d03e0d26bfa2b38e11771a2ba79a2d7b0f8ca816cd10f084fdfbaa7c93d0", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "sha256:0e594beec5a07829044900d4a93fcde8e2e0e9d50e1e0f7e19809f9d2845b3ba", + "matched_source:src/aisle/harness/validate.py": "sha256:ed82967e800af6ec4d1f5336374343df38a5ba437771dd94e24b06c7dc64f481", + "matched_source:src/aisle/harness/worker_authority_probe.py": "sha256:9ee0638062e062c500c7103664eecd541e5d68056bb83368223c09f145dad6ee", + "matched_source:src/aisle/harness/worker_capability.py": "sha256:272c82452cd6f81912593b4584729cb7bb0e8f528d662cb3021e8dfc6ce70e83", + "matched_source:src/aisle/harness/worker_declaration.py": "sha256:15f42f83cae62a8e917f010b3e1571dda81a3cc7e7e3d630ed2b697fb0e8c0c2", + "matched_source:src/aisle/harness/worker_network_probe.py": "sha256:eb0d75fcbd4e8242eb61701f07e02e83ee6e7d9053247df5572f699969bcceb2", + "matched_source:src/aisle/kinematics.py": "sha256:0656cb4c9c4769365c42fc175deefb69d7a7a3dda77991329b1cedf33f7d5aaa", + "matched_source:src/aisle/mobility/guard.py": "sha256:3d0d4ddd757a8ae1840807511f8816f85feff0f2103fc490a8d29db6fa3663d0", + "matched_source:src/aisle/monolith/confinement.py": "sha256:f207d08bac411ae0cb9a5986b3f5f4d55e55d1407e0c0e6889ec5d489b232969", + "matched_source:src/aisle/monolith/primitive_api.py": "sha256:d5a344125ca3f434e489325b6bc7e20422082d18abee8ba7cd4deccda901327a", + "matched_source:src/aisle/monolith/primitives.py": "sha256:e4b3ef279708ef5fbd6e114f524a6c727f3c9d39a41d56dbfbe73d26e3ac515b", + "matched_source:src/aisle/monolith/proxy.py": "sha256:191b4ce164ea431ccbe728ad9ca5b482a09894093ee389643e099c3774df4ca4", + "matched_source:src/aisle/monolith/requests.py": "sha256:04151bf8838eda75be2a9d0c91240607283fb826712b90a8f99eab5d83d30185", + "matched_source:src/aisle/monolith/supervisor.py": "sha256:7dbe215dc5bddd12d414d1b88e896000fc64a9c71746230518eb2f47f1ef1051", + "matched_source:src/aisle/monolith/wire.py": "sha256:3418587b67e3ce2481e68c2809aaafc8e17f244246d33b055ee05f06ac6177a6", + "matched_source:src/aisle/monolith/worker.py": "sha256:931e42442b9f69157eab17cb73f1fa235eb053da7d843915ef889df5f84fb2d7", + "matched_source:src/aisle/monolith/worker_config.py": "sha256:55a26fdfa8b701470e50d54ffa9384d0ed89a0dc157867ac7e0005671e623312", + "matched_source:src/aisle/monolith/worker_launch.py": "sha256:0e0d053037e1dc9aef27c0248b8adce21c29546ca38ca7ebff6fd0c41c14475b", + "matched_source:src/aisle/nodes/budget_guard.py": "sha256:e15bc5f70b295e7dfd32f72e4a5cf72dffca9da50e9e8f6bb18360fb38069abf", + "matched_source:src/aisle/nodes/h6_fault.py": "sha256:c93b8e1607b7627adef9d971753727b9c602be44ae539def9188344332a63485", + "matched_source:src/aisle/nodes/l2_pose.py": "sha256:d2889dac35af94cdf89a280cf734f672098ce95a4a347e1b9797545c2269fda2", + "matched_source:src/aisle/nodes/label_reader.py": "sha256:aeb51080744779f68fb56d6ff408993e92e3e0b929b7d353af7521537f252517", + "matched_source:src/aisle/nodes/monolith_broker.py": "sha256:6fb86d2fdcae8d9ac0314e4cab3100cd925c35a96f74249006392e042847c048", + "matched_source:src/aisle/nodes/perception_session.py": "sha256:cf0fd80a43a338711456d7db546167d082fb6ecc461a828f3cb0e2d42701fc9e", + "matched_source:src/aisle/scenes/locations.toml": "sha256:baea316611ce2f05751ab3058b4b46acbdd9801d8734faca0272c06b2565bcca", + "matched_source:src/aisle/scenes/meds.toml": "sha256:3b25c5eb6586b311fbe8aa3ffc9fdab76cb1b60c9781c3a7a31b12ab6b61dc6b", + "matched_source:src/aisle/scenes/pharmacy.py": "sha256:642e4a57c9225804b44d4e6909fad6514475598cf002dff300b9ce4455225981", + "matched_source:src/aisle/scenes/physics.toml": "sha256:7a400f5cb01a4a93e2a1daa07fa3b2fafd6598643e15965d779c924480f7c36c", + "matched_source:src/aisle/scenes/planogram.toml": "sha256:c579e207de210ed102ff81e15bf0877a91ce0d4ac31ebce2c31e484322f0e576", + "matched_source:src/aisle/scenes/store.py": "sha256:ea79ce1c91d2085ac920fa1ffe87c0c662c875e0e36da5866c1bc60ae5093a66", + "matched_source:src/aisle/topics.py": "sha256:431a0e0bee4c4c3d83f09f9452eb481f238ec0b8cab11b7370bebc117beeb26f", + "matched_source:src/aisle/turn_node.py": "sha256:0c6cd11cd802ebf8934a671caeb4b8a590e5f43b23f03c52515f51eac4288705", + "matched_source:src/aisle/turns.py": "sha256:f0014e3d8ea6b07d4effd10c5bf0b7de08b89ecec1e164580509ad25694b36a4", + "matched_source:src/aisle/verifier/calibration.py": "sha256:a00b91406d27a3b6123701a7a7270258633ad9906c28dbb6f1110fbb07b59568", + "matched_source:src/aisle/verifier/models.lock": "sha256:3fa7271740f3ecd361213f45de140452f41216b80600e343e2daa9f2a413eab5", + "matched_source:src/aisle/verifier/models.py": "sha256:1b2905181bc757f8390666c2a4c7303f674a6d8a31232e8feae582b5cc24909f", + "matched_source:src/aisle/verifier/placement.toml": "sha256:8d8f37165cf4bb4214486597689d40d5df8fb36e2bb8f15e65f6be8801d3717f", + "matched_source:src/aisle/verifier/realistic.py": "sha256:769f6295d3c15a5be02da8ec815a401728059608f2ae92c904d5bec0dd18d156", + "matched_source:src/aisle/verifier/stages.py": "sha256:63848f9e4c0e59020461e3350bf8ca2a8d0ae93cc45cd43c5786c7b6f95a443c", + "matched_source:src/aisle/verifier/thresholds.toml": "sha256:567ac0544a4c382d3ce6547d8df868e6ca37c056b146cf0d1ee11934b1be16d3", + "matched_source:tools/campaign.py": "sha256:cb33ab83f9125ea9708a73f639ca38ad539fbc77e6c5b03e0b49dc2e66abd8f0", + "matched_source:tools/matched_campaign.py": "sha256:71e648035529877ee68b26eb2a4ccf5c0126e7872e3e9f6134785a1b63554609", + "monolithic_validators": "sha256:1983ed8c26b2371c993c571f33f5c7e4ee81ea2597f9468b43d53c9970465ccf", + "nested_host_documentation": "sha256:1c4777e4d8a29882ae5332f96df6b175423e7e5d1f279357fdb1286bace12959", + "pilot_assignments_cli": "sha256:bafcfdf4f79d5d0abd7442355454ea7ca476928afd57ed7c83f38da0c3726cc8", + "pilot_decision": "sha256:8a2f932b7ff6e87c833baa0fd85f2e5d892bed4bd7287e05488eaa8652a85966", + "pilot_records_cli": "sha256:51438c6e14fca6368e1b91ebce77fd13c6bcc821edb61666553750a91d3ab75f", + "protocol": "sha256:e51ad66dea4b96db7a9af17744db5971c6ac1e4bcb165a03c939d0d299619150", + "protocol_template": "sha256:9d636f3dc462a2e051ee0b3ded565aff1c5be86331e5b1b6692988509ccf088e", + "provider_admission_documentation": "sha256:856851935d15d8cb0c5653548ae9312441263cce9a3300e36d0863f5bdb96cf4", + "public_observation_source:analysis/monolithic-control/experts-t1-l2-v1.json": "sha256:eca1034c63620f5a901a0b2da4c157391643bc4e5aaf48457ac172ff8a656288", + "public_observation_source:analysis/monolithic-control/treatment-table-t1-l2-v1.json": "sha256:58f7164a95500f514a815b44ef1a681462c13127c3fbff9f887ae3873f21eac2", + "public_observation_source:docs/decisions/ADR-pilot-public-observations.md": "sha256:0bf46aac693fa9b9e78997d6843cb87112f7716d006a0f4a63e987f3eca5ea5e", + "public_observation_source:docs/monolithic/public-observations.md": "sha256:1b395b766ef5345169e2047a3a86f80648f5d265f3d4445315f314214c7068b7", + "public_observation_source:graphs/pilot_t1_l2_monolithic.yaml": "sha256:ebf12538d0b9def23ce4ea70a9feb375c98535f774bbf16d4d839aaa4e99d27c", + "public_observation_source:graphs/pilot_t1_l2_typed.yaml": "sha256:9dd2a3f4a71ed954987861e80a163c644e3a0aec4f3556b5fa353fa8f83aa8cb", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_monolithic.json": "sha256:089558615e62539fb8121c6521d1bcc2af1403d4e896be9ef731b353e9b55d8c", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_typed.json": "sha256:605b00af15cc3e8ba0331468532277c2e8ca4d3bcc196e84978d957fde59f23e", + "public_observation_source:registry/manifests/pilot-policy-surface.yaml": "sha256:5e189856d64aa49c90b573879d5b71abe230a2790afad5b9214a061d521de655", + "public_observation_source:src/aisle/harness/pilot_model_cache.py": "sha256:17a90497a33015388f5426dad4d5c3e29700a8b48ccb47a6e02380c3274339f5", + "public_observation_source:src/aisle/harness/pilot_policy_surface.py": "sha256:bb08dc54cf6a86907d68bab554baab1d562ec754b2736812e68798b96395cf80", + "research_prompt": "sha256:e736bbd6caa7236dbfafe92ae42c0c7f13088764af1b753f6685d398c111b7c0", + "seed_commitment_predecessor": "sha256:4d6c2c6bee3032c427f63600a761095faa2cd0ba58ad145be7720bf52fd6eb6d", + "session_runner": "sha256:cb33ab83f9125ea9708a73f639ca38ad539fbc77e6c5b03e0b49dc2e66abd8f0", + "session_validators": "sha256:1b1c53d6389881c146726d865a741a681f3d5039c9bf99558a53ee368250971e", + "simulator_bridge": "sha256:196837073251bedefd0365d220aa94407612f7670ed0b1b39c7319d63a1c18f8", + "spec": "sha256:f5096c587cd9687859f2dc80ec807d1bbf959cb4e05c152154e71e47bdc1f593", + "treatment_integrity": "sha256:b37162c6f295c5686da9b3089eb3bd96b69f68f01f41da0a784c4017658b10a6", + "treatment_postflight": "sha256:0bf01138ef0c277c6fcd728fcf25f4f13795de679fb41ab58b868b81f6c2f0ca", + "treatment_table": "sha256:0f860aefee6272ae5ca92f0a6d704d9f2dd924d931eed97d0b666f09361217d8" + }, + "campaign_id": "cse-causal-study-pilot-v10", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-pilot-v8/protocol.json --purpose power", + "uv run harness stats power --protocol analysis/freeze/cse-causal-study-pilot-v8/protocol.json --output analysis/freeze/cse-causal-study-pilot-v8/power.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json --allow-withheld-seeds" + ], + "declaration": { + "analysis": { + "scripts": [ + "src/aisle/harness/benchmark_statistics.py", + "src/aisle/harness/causal_study.py", + "src/aisle/harness/pilot_records.py" + ], + "seed": 500001 + }, + "artifacts": { + "agent_adapters": "tools/agent_adapters.py", + "analyzer": "src/aisle/harness/benchmark_statistics.py", + "bnd_calibration_manifest": "analysis/freeze/bnd-task-band-calibration-v11/freeze-manifest.json", + "bnd_perception_audit_report": "analysis/perception-audit/records/bnd-perception-corpus-03/report.json", + "budget_ceiling": "harness/budget.toml", + "dependency_declaration": "pyproject.toml", + "dependency_lock": "uv.lock", + "dora_runtime_selection": "dora-runtime.json", + "frontend_request_authorization_documentation": "docs/benchmark/v1/frontend-request-authorization.md", + "matched_session_decision": "docs/decisions/ADR-62.md", + "matched_session_documentation": "docs/monolithic/matched-session.md", + "matched_source:assets/so101/so101.urdf": "assets/so101/so101.urdf", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "docs/monolithic/allowlist-t1-l2.json", + "matched_source:docs/monolithic/candidates.json": "docs/monolithic/candidates.json", + "matched_source:docs/monolithic/experts-t1-l2.json": "docs/monolithic/experts-t1-l2.json", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "docs/monolithic/interface-map-t1-l2.json", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "docs/monolithic/parity-protocol-t1-l2.json", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "docs/monolithic/treatment-table-t1-l2.json", + "matched_source:env/limits.toml": "env/limits.toml", + "matched_source:experts/monolithic/expert_t1_l2.py": "experts/monolithic/expert_t1_l2.py", + "matched_source:graphs/pilot_t1_l2_monolithic.yaml": "graphs/pilot_t1_l2_monolithic.yaml", + "matched_source:src/aisle/__init__.py": "src/aisle/__init__.py", + "matched_source:src/aisle/embodiment.py": "src/aisle/embodiment.py", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "src/aisle/harness/_code_mode_protocol/__init__.py", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "src/aisle/harness/_code_mode_protocol/descriptor.pb", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "src/aisle/harness/_code_mode_protocol/protocol.proto", + "matched_source:src/aisle/harness/candidate_launch.py": "src/aisle/harness/candidate_launch.py", + "matched_source:src/aisle/harness/candidates.py": "src/aisle/harness/candidates.py", + "matched_source:src/aisle/harness/cli.py": "src/aisle/harness/cli.py", + "matched_source:src/aisle/harness/code_mode_audit.py": "src/aisle/harness/code_mode_audit.py", + "matched_source:src/aisle/harness/code_mode_authority.py": "src/aisle/harness/code_mode_authority.py", + "matched_source:src/aisle/harness/code_mode_proxy.py": "src/aisle/harness/code_mode_proxy.py", + "matched_source:src/aisle/harness/code_mode_runner.py": "src/aisle/harness/code_mode_runner.py", + "matched_source:src/aisle/harness/common.py": "src/aisle/harness/common.py", + "matched_source:src/aisle/harness/frontend_app_server.py": "src/aisle/harness/frontend_app_server.py", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "src/aisle/harness/frontend_app_server_audit.py", + "matched_source:src/aisle/harness/frontend_dispatch.py": "src/aisle/harness/frontend_dispatch.py", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "src/aisle/harness/frontend_dispatch_audit.py", + "matched_source:src/aisle/harness/frontend_request_audit.py": "src/aisle/harness/frontend_request_audit.py", + "matched_source:src/aisle/harness/frontend_request_authority.py": "src/aisle/harness/frontend_request_authority.py", + "matched_source:src/aisle/harness/guard_divergence.py": "src/aisle/harness/guard_divergence.py", + "matched_source:src/aisle/harness/heldout_eval.py": "src/aisle/harness/heldout_eval.py", + "matched_source:src/aisle/harness/matched_app_server.py": "src/aisle/harness/matched_app_server.py", + "matched_source:src/aisle/harness/matched_collection.py": "src/aisle/harness/matched_collection.py", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "src/aisle/harness/matched_dynamic_run.py", + "matched_source:src/aisle/harness/matched_evidence.py": "src/aisle/harness/matched_evidence.py", + "matched_source:src/aisle/harness/matched_frontend.py": "src/aisle/harness/matched_frontend.py", + "matched_source:src/aisle/harness/matched_run.py": "src/aisle/harness/matched_run.py", + "matched_source:src/aisle/harness/matched_run_launch.py": "src/aisle/harness/matched_run_launch.py", + "matched_source:src/aisle/harness/matched_runtime.py": "src/aisle/harness/matched_runtime.py", + "matched_source:src/aisle/harness/matched_session.py": "src/aisle/harness/matched_session.py", + "matched_source:src/aisle/harness/matched_tool_service.py": "src/aisle/harness/matched_tool_service.py", + "matched_source:src/aisle/harness/matched_tools.py": "src/aisle/harness/matched_tools.py", + "matched_source:src/aisle/harness/monolith.py": "src/aisle/harness/monolith.py", + "matched_source:src/aisle/harness/monolithic.py": "src/aisle/harness/monolithic.py", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "src/aisle/harness/monolithic_run_evidence.py", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "src/aisle/harness/monolithic_run_prepare.py", + "matched_source:src/aisle/harness/pilot_records.py": "src/aisle/harness/pilot_records.py", + "matched_source:src/aisle/harness/provider_relay.py": "src/aisle/harness/provider_relay.py", + "matched_source:src/aisle/harness/provider_response_authority.py": "src/aisle/harness/provider_response_authority.py", + "matched_source:src/aisle/harness/provider_runner.py": "src/aisle/harness/provider_runner.py", + "matched_source:src/aisle/harness/provider_source_audit.py": "src/aisle/harness/provider_source_audit.py", + "matched_source:src/aisle/harness/registry.py": "src/aisle/harness/registry.py", + "matched_source:src/aisle/harness/rollout.py": "src/aisle/harness/rollout.py", + "matched_source:src/aisle/harness/rollout_client.py": "src/aisle/harness/rollout_client.py", + "matched_source:src/aisle/harness/simulator_work.py": "src/aisle/harness/simulator_work.py", + "matched_source:src/aisle/harness/traces.py": "src/aisle/harness/traces.py", + "matched_source:src/aisle/harness/treatment_ambient.py": "src/aisle/harness/treatment_ambient.py", + "matched_source:src/aisle/harness/treatment_confinement.py": "src/aisle/harness/treatment_confinement.py", + "matched_source:src/aisle/harness/treatment_integrity.py": "src/aisle/harness/treatment_integrity.py", + "matched_source:src/aisle/harness/treatment_postflight.py": "src/aisle/harness/treatment_postflight.py", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "src/aisle/harness/typed_execution_bundle.py", + "matched_source:src/aisle/harness/typed_graph_audit.py": "src/aisle/harness/typed_graph_audit.py", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "src/aisle/harness/typed_graph_hosts.py", + "matched_source:src/aisle/harness/typed_graph_stage.py": "src/aisle/harness/typed_graph_stage.py", + "matched_source:src/aisle/harness/typed_node_host.py": "src/aisle/harness/typed_node_host.py", + "matched_source:src/aisle/harness/typed_node_requests.py": "src/aisle/harness/typed_node_requests.py", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "src/aisle/harness/typed_node_supervisor.py", + "matched_source:src/aisle/harness/typed_node_worker.py": "src/aisle/harness/typed_node_worker.py", + "matched_source:src/aisle/harness/typed_run_prepare.py": "src/aisle/harness/typed_run_prepare.py", + "matched_source:src/aisle/harness/typed_snapshot.py": "src/aisle/harness/typed_snapshot.py", + "matched_source:src/aisle/harness/typed_stage_provider.py": "src/aisle/harness/typed_stage_provider.py", + "matched_source:src/aisle/harness/typed_validation.py": "src/aisle/harness/typed_validation.py", + "matched_source:src/aisle/harness/typed_worker_launch.py": "src/aisle/harness/typed_worker_launch.py", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "src/aisle/harness/typed_worker_provisioning.py", + "matched_source:src/aisle/harness/validate.py": "src/aisle/harness/validate.py", + "matched_source:src/aisle/harness/worker_authority_probe.py": "src/aisle/harness/worker_authority_probe.py", + "matched_source:src/aisle/harness/worker_capability.py": "src/aisle/harness/worker_capability.py", + "matched_source:src/aisle/harness/worker_declaration.py": "src/aisle/harness/worker_declaration.py", + "matched_source:src/aisle/harness/worker_network_probe.py": "src/aisle/harness/worker_network_probe.py", + "matched_source:src/aisle/kinematics.py": "src/aisle/kinematics.py", + "matched_source:src/aisle/mobility/guard.py": "src/aisle/mobility/guard.py", + "matched_source:src/aisle/monolith/confinement.py": "src/aisle/monolith/confinement.py", + "matched_source:src/aisle/monolith/primitive_api.py": "src/aisle/monolith/primitive_api.py", + "matched_source:src/aisle/monolith/primitives.py": "src/aisle/monolith/primitives.py", + "matched_source:src/aisle/monolith/proxy.py": "src/aisle/monolith/proxy.py", + "matched_source:src/aisle/monolith/requests.py": "src/aisle/monolith/requests.py", + "matched_source:src/aisle/monolith/supervisor.py": "src/aisle/monolith/supervisor.py", + "matched_source:src/aisle/monolith/wire.py": "src/aisle/monolith/wire.py", + "matched_source:src/aisle/monolith/worker.py": "src/aisle/monolith/worker.py", + "matched_source:src/aisle/monolith/worker_config.py": "src/aisle/monolith/worker_config.py", + "matched_source:src/aisle/monolith/worker_launch.py": "src/aisle/monolith/worker_launch.py", + "matched_source:src/aisle/nodes/budget_guard.py": "src/aisle/nodes/budget_guard.py", + "matched_source:src/aisle/nodes/h6_fault.py": "src/aisle/nodes/h6_fault.py", + "matched_source:src/aisle/nodes/l2_pose.py": "src/aisle/nodes/l2_pose.py", + "matched_source:src/aisle/nodes/label_reader.py": "src/aisle/nodes/label_reader.py", + "matched_source:src/aisle/nodes/monolith_broker.py": "src/aisle/nodes/monolith_broker.py", + "matched_source:src/aisle/nodes/perception_session.py": "src/aisle/nodes/perception_session.py", + "matched_source:src/aisle/scenes/locations.toml": "src/aisle/scenes/locations.toml", + "matched_source:src/aisle/scenes/meds.toml": "src/aisle/scenes/meds.toml", + "matched_source:src/aisle/scenes/pharmacy.py": "src/aisle/scenes/pharmacy.py", + "matched_source:src/aisle/scenes/physics.toml": "src/aisle/scenes/physics.toml", + "matched_source:src/aisle/scenes/planogram.toml": "src/aisle/scenes/planogram.toml", + "matched_source:src/aisle/scenes/store.py": "src/aisle/scenes/store.py", + "matched_source:src/aisle/topics.py": "src/aisle/topics.py", + "matched_source:src/aisle/turn_node.py": "src/aisle/turn_node.py", + "matched_source:src/aisle/turns.py": "src/aisle/turns.py", + "matched_source:src/aisle/verifier/calibration.py": "src/aisle/verifier/calibration.py", + "matched_source:src/aisle/verifier/models.lock": "src/aisle/verifier/models.lock", + "matched_source:src/aisle/verifier/models.py": "src/aisle/verifier/models.py", + "matched_source:src/aisle/verifier/placement.toml": "src/aisle/verifier/placement.toml", + "matched_source:src/aisle/verifier/realistic.py": "src/aisle/verifier/realistic.py", + "matched_source:src/aisle/verifier/stages.py": "src/aisle/verifier/stages.py", + "matched_source:src/aisle/verifier/thresholds.toml": "src/aisle/verifier/thresholds.toml", + "matched_source:tools/campaign.py": "tools/campaign.py", + "matched_source:tools/matched_campaign.py": "tools/matched_campaign.py", + "monolithic_validators": "src/aisle/harness/monolithic.py", + "nested_host_documentation": "docs/monolithic/code-mode-admission.md", + "pilot_assignments_cli": "tools/pilot_assignments.py", + "pilot_decision": "docs/decisions/ADR-66.md", + "pilot_records_cli": "tools/pilot_records.py", + "protocol": "analysis/freeze/cse-causal-study-pilot-v8/protocol.json", + "protocol_template": "analysis/statistics/protocol-template.json", + "provider_admission_documentation": "docs/monolithic/provider-admission.md", + "public_observation_source:analysis/monolithic-control/experts-t1-l2-v1.json": "analysis/monolithic-control/experts-t1-l2-v1.json", + "public_observation_source:analysis/monolithic-control/treatment-table-t1-l2-v1.json": "analysis/monolithic-control/treatment-table-t1-l2-v1.json", + "public_observation_source:docs/decisions/ADR-pilot-public-observations.md": "docs/decisions/ADR-pilot-public-observations.md", + "public_observation_source:docs/monolithic/public-observations.md": "docs/monolithic/public-observations.md", + "public_observation_source:graphs/pilot_t1_l2_monolithic.yaml": "graphs/pilot_t1_l2_monolithic.yaml", + "public_observation_source:graphs/pilot_t1_l2_typed.yaml": "graphs/pilot_t1_l2_typed.yaml", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_monolithic.json": "graphs/turn_plans/pilot_t1_l2_monolithic.json", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_typed.json": "graphs/turn_plans/pilot_t1_l2_typed.json", + "public_observation_source:registry/manifests/pilot-policy-surface.yaml": "registry/manifests/pilot-policy-surface.yaml", + "public_observation_source:src/aisle/harness/pilot_model_cache.py": "src/aisle/harness/pilot_model_cache.py", + "public_observation_source:src/aisle/harness/pilot_policy_surface.py": "src/aisle/harness/pilot_policy_surface.py", + "research_prompt": "harness/CLAUDE.research.md", + "seed_commitment_predecessor": "analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json", + "session_runner": "tools/campaign.py", + "session_validators": "src/aisle/harness/causal_study.py", + "simulator_bridge": "src/aisle/nodes/dora_genesis.py", + "spec": "specs/500-typed-monolithic-causal-study.md", + "treatment_integrity": "src/aisle/harness/treatment_integrity.py", + "treatment_postflight": "src/aisle/harness/treatment_postflight.py", + "treatment_table": "analysis/monolithic-control/treatment-table-v1.json" + }, + "budgets": { + "campaign_ceiling": "harness/budget.toml", + "rollouts_per_session": 20, + "session_tokens": 200000, + "session_wall_s": 3600, + "sim_episodes_per_session": 160 + }, + "campaign_id": "cse-causal-study-pilot-v10", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-pilot-v8/protocol.json --purpose power", + "uv run harness stats power --protocol analysis/freeze/cse-causal-study-pilot-v8/protocol.json --output analysis/freeze/cse-causal-study-pilot-v8/power.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-pilot-v10/freeze-manifest.json --allow-withheld-seeds" + ], + "decision_rules": { + "alpha": 0.05, + "equivalence_margin": { + "cost_at_equal_quality": { + "acceptance_probability": 0.1, + "heldout_quality": 0.1, + "rule": "CSE-5: both must pass or cost claims stay unconditional", + "sidedness": "two-sided 0.90" + } + }, + "multiplicity": "single primary endpoint", + "power": 0.8, + "primary_rule": "one-sided superiority at alpha 0.05 on the pooled Newcombe interval lower bound > 0 with no reversed stratum; otherwise weakened/rejected/undecidable by the #358 rule (CSE-3)", + "smallest_effect": { + "measure": "risk difference in session success", + "value": 0.25 + }, + "stopping_rule": "fixed n from the frozen power analysis using pilot-only control success; absolute floor 10 randomized assignments per arm per agent-system x task stratum (CSE-8); no optional stopping or extension" + }, + "disposition": "registered 2026-09-13; collection pending. Supersedes the zero-session pilot-v1 lineage with the instrument set the pilot can actually run on (T1-L2 realistic pair, #581 to #586). Named limitations: BND-7 not eligible (recorded failed), external-review and parity gates pending, and the session-bound capability audit not merged (an unattested environment stops every session).", + "endpoints": { + "primary": [ + { + "aggregation": "risk difference typed_dataflow - monolithic; exact arm intervals; Newcombe interval; per agent-system x task stratum before pooling (CSE-1, CSE-17)", + "direction": "higher", + "id": "E-CSE-1", + "outcome": "session_success", + "status": "inferential", + "unit": "agent_session" + } + ], + "secondary": [ + { + "aggregation": "counts by arm", + "direction": "descriptive", + "id": "E-CSE-2", + "outcome": "validation_and_launch_failure_class", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "Kaplan-Meier with censoring", + "direction": "lower", + "id": "E-CSE-3", + "outcome": "time_to_first_accepted_system", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "unconditional session estimands; no at-equal-quality wording unless CSE-5 equivalence passes", + "direction": "lower", + "id": "E-CSE-4", + "outcome": "tokens_tool_calls_wall_active_time_sim_work_rollouts", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "counts", + "direction": "descriptive", + "id": "E-CSE-5", + "outcome": "safety_interventions", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "dropped unless a blinded rubric is ratified before freeze (CSE-18)", + "direction": "n/a", + "id": "E-CSE-6", + "outcome": "artifact_legibility", + "status": "exploratory", + "unit": "agent_session" + } + ] + }, + "exclusions": { + "deviation_policy": "any behaviour-affecting change after registration is a documented deviation record and a new campaign version id; confirmatory data never revises the registered protocol", + "infrastructure": [ + "independent-controller-determined, arm-neutral facts only (CSE-14): host outage, provider interruption before first tool call, launcher crash before the session started" + ], + "rerun_policy": "a permitted rerun receives a new randomized id; the original row is retained with its exclusion reason and never replaced in place", + "treatment_integrity": [ + "contamination sentinel tripped (CSE-12)", + "model alias or CLI drift outside the frozen identity (CSE-6)", + "integrity verdict authored session-side (CSE-11)" + ] + }, + "hypotheses": [ + { + "direction": "typed_dataflow - monolithic > 0", + "id": "H-CSE-1", + "statement": "The bundled typed-dataflow surface raises the probability that a fresh coding-agent session produces a launching system passing the hidden held-out acceptance within budget, versus the equal-capability monolithic surface." + } + ], + "instrument_set": { + "agent_systems": [ + { + "binary": "codex", + "binary_sha256": "b973d440acac501fd2594a43e7ca9ce41e0a65b9dfb28d0d7a7837c99e1261e3", + "frozen": true, + "id": "codex-cli", + "install": "private prefix ~/aisle-private/codex-0.153.4 (the qualified darwin-arm64 binary, not the JS shim); ChatGPT-login credentials copied byte-identically into both arms' private CODEX_HOME, hash recorded only", + "version_observed": "0.153.4" + } + ], + "candidate": { + "id": "t1-l2-realistic", + "monolithic_module": "experts/monolithic/expert_t1_l2.py", + "monolithic_template": "graphs/monolithic_t1_l2.yaml", + "perception": "L2", + "reset": "teleport", + "typed_graph": "graphs/expert_t1_l2.yaml", + "verifier": "realistic (oracle demoted to the held-out scoring tap, BND-3)" + }, + "candidate_table_sha256": "08a1dc56502abc4673cb0b54f01664382214bb473c3ef293b1df7b6c74f64778", + "items": [ + "short_composition:t1-l2-realistic" + ], + "kind": "selected_task" + }, + "integrity_checks": [ + { + "gate": "CON-14 spec approval", + "kind": "external_review", + "owner_role": "project owner", + "pilot_record_2026_09_13": "pending; external review, not a pilot prerequisite (ADR-66 section 2); named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "STA-12 independent statistical review", + "kind": "external_review", + "owner_role": "independent statistician", + "pilot_record_2026_09_13": "pending; external review, not a pilot prerequisite (ADR-66 section 2); named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "MON-1 treatment table and expert parity (#344)", + "kind": "machine_check", + "owner_role": "parity harness; control surface exists, independent authorship and operator evidence remain pending", + "pilot_record_2026_09_13": "pending; the T1-L2 pair (#583) passes the interface, table and module checks but has no operator parity record at an immutable hash; named limitation", + "record": null, + "status": "pending" + }, + { + "gate": "BND-12 task-band freeze (#346)", + "kind": "machine_check", + "owner_role": "task-band auditor", + "pilot_record_2026_09_13": "recorded failed: bnd-task-band-calibration-v11 regenerated the BND-5 perception audit on a 32-seed T1-L2 corpus; 22 of 22 strata below the 0.90 floor (BND-7), led by target_class=ibuprofen refusals; role open (BND-10). ADR-66 section 2: a failing machine gate is a named limitation of this pilot, not a blocker", + "record": "analysis/perception-audit/records/bnd-perception-corpus-03/report.json", + "status": "failed" + }, + { + "gate": "TRT confinement/mutation/agent-path (#353)", + "kind": "machine_check", + "owner_role": "integrity controller", + "pilot_record_2026_09_13": "pending; the session-bound capability audit (loopback relay policy) is not merged; an unattested environment is a stop condition for every session", + "record": null, + "status": "pending" + }, + { + "gate": "seed commitment verification", + "kind": "machine_check", + "owner_role": "holder of the original private seed and salt sources", + "pilot_record_2026_09_13": "not applicable to this registration: it commits a fresh private seed source directly (no inheritance); kept for shape parity and left pending", + "public_observation_successor_note": "Inherited from pilot-v8 without access to private seed sources; verification remains pending. This byte-binding successor does not authorize a pilot run.", + "record": null, + "status": "pending" + }, + { + "gate": "MON-8/MON-12 matched-session acceptance (#519)", + "kind": "machine_check", + "owner_role": "matched-session controller; complete acceptance evidence still required", + "pilot_record_2026_09_13": "pending; frontend conformance qualified on the T1 oracle development surface only (#568); the T1-L2 realistic surface runs under the same executor with the same admission checks; named limitation", + "record": null, + "status": "pending" + } + ], + "issue": 347, + "pilot_design": { + "agent_system": "codex-cli 0.153.4 only (the owned provider relay, the ChatGPT-login account header and the pinned conformance profile are Codex-bound)", + "arm_order": "sealed balanced plan (revealed one assignment at a time); no longer a fixed alternation", + "arms": [ + "typed", + "monolithic" + ], + "assignment": "tools/pilot_assignments.py create --protocol analysis/freeze/cse-causal-study-pilot-v8/protocol.json (sealed balanced plan, one typed and one monolithic assignment per temporal block, TRT-8/CSE-7); reveal exactly one assignment before each session; the ledger retains every assignment, started or not", + "below_cse8_floor": "6 per arm is below the CSE-8 floor of 10 by design; this registration can never become a confirmatory estimate (STA-3)", + "decision": "docs/decisions/ADR-66.md (pilot first; agent-drafted under the ADR-65 label, renumbered by #568)", + "executor": "tools/matched_campaign.py admit / run --purpose engineering / collect-run, then the held-out evaluation; retained and unscored, output root ~/aisle-private/raw/cse-causal-study-pilot-v8/", + "records": "tools/pilot_records.py derives one aisle.stats.records.v1 row per ledger assignment; harness stats validate --purpose analysis; harness stats analyze; harness stats power from observed control success (CSE-8)", + "reporting": "analysis/reports/cse_pilot_v8.md; results enter docs/claim-evidence.yaml only as CSE-8 power inputs under the unrun confirmatory claim", + "rerun_policy": "no reruns; a failed attempt is retained under a new session id, never overwritten", + "session_success": "controller-run held-out evaluation of the retained final snapshot on the 32 private seeds after the agent process ends, oracle as the held-out scorer (BND-3); success iff at least 28 of 32 oracle successes; a launch failure counts as a failed seed (CSE-2, CSE-9)", + "sessions_per_arm": 6, + "stop_conditions": [ + "wrong_object outcome", + "guard bypass", + "unattested environment", + "relay bypass (provider traffic outside the owned relay)", + "any oracle field on a policy input" + ], + "task_stratum": "short_composition on candidate t1-l2-realistic (T1 franka, L2 perception, realistic verifier); bnd-task-band-calibration-v11 records it as not perception-eligible (BND-7), carried as a machine-gate limitation per ADR-66" + }, + "purpose": "pilot", + "schema_version": "aisle.freeze.declaration.v1", + "seed_commitment": { + "inherited_from": "analysis/freeze/cse-causal-study-pilot-v9/freeze-manifest.json", + "rule": "32 distinct integers in [1000,9999) drawn once from the OS CSPRNG on 2026-09-13, retained only under ~/aisle-private/freeze/cse-causal-study-pilot-v8/, disjoint by value from every private seed list committed so far (including cse-causal-study-pilot-v1 and the cse-causal-study-v1 confirmatory commitment) and from the committed development seed lists; never inherited by a confirmatory declaration (ADR-66); revealed after the pilot closes. The same directory holds the controller-private 256-bit randomization seed for the sealed assignment plan (TRT-8); its commitment is recorded in the assignment ledger, never here.", + "salt_source": "~/aisle-private/freeze/cse-causal-study-pilot-v8/salt.bin", + "seeds_source": "~/aisle-private/freeze/cse-causal-study-pilot-v8/seeds.json" + }, + "spec": "500", + "superseded": "cse-causal-study-pilot-v9 (session-bound capability audits and fail-closed rejection of unsupported loopback endpoint authority; public-observation and cache source coverage, study rules, inherited seed commitment and pending gates preserved; no collection authorization)" + }, + "declaration_sha256": "sha256:b70d4f89fe0a3d935f4e45c58211ab630199368ea075450c8045eba2d9501c8f", + "frozen": false, + "gate_record_hashes": {}, + "git_head": "cf834d629865b99de30308c73513060ebc8f48f6", + "issue": 347, + "pending_gates": [ + "BND-12 task-band freeze (#346)", + "CON-14 spec approval", + "MON-1 treatment table and expert parity (#344)", + "MON-8/MON-12 matched-session acceptance (#519)", + "STA-12 independent statistical review", + "TRT confinement/mutation/agent-path (#353)", + "seed commitment verification" + ], + "purpose": "pilot", + "schema_version": "aisle.freeze.manifest.v1", + "seed_commitment": "sha256:bc4a2c141eded05f77bee114330c2f9d1c5112e26618f2feb9694cf60dd56143", + "spec": "500", + "status": "registered_pending_review", + "timestamp": null, + "timestamp_source": null +} diff --git a/analysis/freeze/cse-causal-study-v24/declaration.json b/analysis/freeze/cse-causal-study-v24/declaration.json new file mode 100644 index 00000000..ae93672c --- /dev/null +++ b/analysis/freeze/cse-causal-study-v24/declaration.json @@ -0,0 +1,359 @@ +{ + "analysis": { + "scripts": [ + "src/aisle/harness/benchmark_statistics.py", + "src/aisle/harness/causal_study.py", + "src/aisle/harness/pilot_records.py" + ], + "seed": 500001 + }, + "artifacts": { + "agent_adapters": "tools/agent_adapters.py", + "analyzer": "src/aisle/harness/benchmark_statistics.py", + "budget_ceiling": "harness/budget.toml", + "dependency_declaration": "pyproject.toml", + "dependency_lock": "uv.lock", + "dora_runtime_selection": "dora-runtime.json", + "frontend_conformance_decision": "docs/decisions/ADR-65.md", + "frontend_request_authorization_documentation": "docs/benchmark/v1/frontend-request-authorization.md", + "matched_session_decision": "docs/decisions/ADR-62.md", + "matched_session_documentation": "docs/monolithic/matched-session.md", + "matched_source:assets/so101/so101.urdf": "assets/so101/so101.urdf", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "docs/monolithic/allowlist-t1-l2.json", + "matched_source:docs/monolithic/candidates.json": "docs/monolithic/candidates.json", + "matched_source:docs/monolithic/experts-t1-l2.json": "docs/monolithic/experts-t1-l2.json", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "docs/monolithic/interface-map-t1-l2.json", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "docs/monolithic/parity-protocol-t1-l2.json", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "docs/monolithic/treatment-table-t1-l2.json", + "matched_source:env/limits.toml": "env/limits.toml", + "matched_source:experts/monolithic/expert_t1_l2.py": "experts/monolithic/expert_t1_l2.py", + "matched_source:graphs/pilot_t1_l2_monolithic.yaml": "graphs/pilot_t1_l2_monolithic.yaml", + "matched_source:src/aisle/__init__.py": "src/aisle/__init__.py", + "matched_source:src/aisle/embodiment.py": "src/aisle/embodiment.py", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "src/aisle/harness/_code_mode_protocol/__init__.py", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "src/aisle/harness/_code_mode_protocol/descriptor.pb", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "src/aisle/harness/_code_mode_protocol/protocol.proto", + "matched_source:src/aisle/harness/candidate_launch.py": "src/aisle/harness/candidate_launch.py", + "matched_source:src/aisle/harness/candidates.py": "src/aisle/harness/candidates.py", + "matched_source:src/aisle/harness/cli.py": "src/aisle/harness/cli.py", + "matched_source:src/aisle/harness/code_mode_audit.py": "src/aisle/harness/code_mode_audit.py", + "matched_source:src/aisle/harness/code_mode_authority.py": "src/aisle/harness/code_mode_authority.py", + "matched_source:src/aisle/harness/code_mode_proxy.py": "src/aisle/harness/code_mode_proxy.py", + "matched_source:src/aisle/harness/code_mode_runner.py": "src/aisle/harness/code_mode_runner.py", + "matched_source:src/aisle/harness/common.py": "src/aisle/harness/common.py", + "matched_source:src/aisle/harness/frontend_app_server.py": "src/aisle/harness/frontend_app_server.py", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "src/aisle/harness/frontend_app_server_audit.py", + "matched_source:src/aisle/harness/frontend_conformance.py": "src/aisle/harness/frontend_conformance.py", + "matched_source:src/aisle/harness/frontend_dispatch.py": "src/aisle/harness/frontend_dispatch.py", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "src/aisle/harness/frontend_dispatch_audit.py", + "matched_source:src/aisle/harness/frontend_effects.py": "src/aisle/harness/frontend_effects.py", + "matched_source:src/aisle/harness/frontend_qualification.py": "src/aisle/harness/frontend_qualification.py", + "matched_source:src/aisle/harness/frontend_request_audit.py": "src/aisle/harness/frontend_request_audit.py", + "matched_source:src/aisle/harness/frontend_request_authority.py": "src/aisle/harness/frontend_request_authority.py", + "matched_source:src/aisle/harness/guard_divergence.py": "src/aisle/harness/guard_divergence.py", + "matched_source:src/aisle/harness/heldout_eval.py": "src/aisle/harness/heldout_eval.py", + "matched_source:src/aisle/harness/matched_app_server.py": "src/aisle/harness/matched_app_server.py", + "matched_source:src/aisle/harness/matched_collection.py": "src/aisle/harness/matched_collection.py", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "src/aisle/harness/matched_dynamic_run.py", + "matched_source:src/aisle/harness/matched_evidence.py": "src/aisle/harness/matched_evidence.py", + "matched_source:src/aisle/harness/matched_frontend.py": "src/aisle/harness/matched_frontend.py", + "matched_source:src/aisle/harness/matched_run.py": "src/aisle/harness/matched_run.py", + "matched_source:src/aisle/harness/matched_run_launch.py": "src/aisle/harness/matched_run_launch.py", + "matched_source:src/aisle/harness/matched_runtime.py": "src/aisle/harness/matched_runtime.py", + "matched_source:src/aisle/harness/matched_session.py": "src/aisle/harness/matched_session.py", + "matched_source:src/aisle/harness/matched_tool_service.py": "src/aisle/harness/matched_tool_service.py", + "matched_source:src/aisle/harness/matched_tools.py": "src/aisle/harness/matched_tools.py", + "matched_source:src/aisle/harness/mcp_harness_authority.py": "src/aisle/harness/mcp_harness_authority.py", + "matched_source:src/aisle/harness/mcp_harness_source.py": "src/aisle/harness/mcp_harness_source.py", + "matched_source:src/aisle/harness/mcp_harness_transport.py": "src/aisle/harness/mcp_harness_transport.py", + "matched_source:src/aisle/harness/monolith.py": "src/aisle/harness/monolith.py", + "matched_source:src/aisle/harness/monolithic.py": "src/aisle/harness/monolithic.py", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "src/aisle/harness/monolithic_run_evidence.py", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "src/aisle/harness/monolithic_run_prepare.py", + "matched_source:src/aisle/harness/pilot_records.py": "src/aisle/harness/pilot_records.py", + "matched_source:src/aisle/harness/provider_hosted.py": "src/aisle/harness/provider_hosted.py", + "matched_source:src/aisle/harness/provider_relay.py": "src/aisle/harness/provider_relay.py", + "matched_source:src/aisle/harness/provider_response_authority.py": "src/aisle/harness/provider_response_authority.py", + "matched_source:src/aisle/harness/provider_runner.py": "src/aisle/harness/provider_runner.py", + "matched_source:src/aisle/harness/provider_source_audit.py": "src/aisle/harness/provider_source_audit.py", + "matched_source:src/aisle/harness/registry.py": "src/aisle/harness/registry.py", + "matched_source:src/aisle/harness/rollout.py": "src/aisle/harness/rollout.py", + "matched_source:src/aisle/harness/rollout_client.py": "src/aisle/harness/rollout_client.py", + "matched_source:src/aisle/harness/simulator_work.py": "src/aisle/harness/simulator_work.py", + "matched_source:src/aisle/harness/traces.py": "src/aisle/harness/traces.py", + "matched_source:src/aisle/harness/treatment_ambient.py": "src/aisle/harness/treatment_ambient.py", + "matched_source:src/aisle/harness/treatment_confinement.py": "src/aisle/harness/treatment_confinement.py", + "matched_source:src/aisle/harness/treatment_integrity.py": "src/aisle/harness/treatment_integrity.py", + "matched_source:src/aisle/harness/treatment_postflight.py": "src/aisle/harness/treatment_postflight.py", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "src/aisle/harness/typed_execution_bundle.py", + "matched_source:src/aisle/harness/typed_graph_audit.py": "src/aisle/harness/typed_graph_audit.py", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "src/aisle/harness/typed_graph_hosts.py", + "matched_source:src/aisle/harness/typed_graph_stage.py": "src/aisle/harness/typed_graph_stage.py", + "matched_source:src/aisle/harness/typed_node_host.py": "src/aisle/harness/typed_node_host.py", + "matched_source:src/aisle/harness/typed_node_requests.py": "src/aisle/harness/typed_node_requests.py", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "src/aisle/harness/typed_node_supervisor.py", + "matched_source:src/aisle/harness/typed_node_worker.py": "src/aisle/harness/typed_node_worker.py", + "matched_source:src/aisle/harness/typed_run_prepare.py": "src/aisle/harness/typed_run_prepare.py", + "matched_source:src/aisle/harness/typed_snapshot.py": "src/aisle/harness/typed_snapshot.py", + "matched_source:src/aisle/harness/typed_stage_provider.py": "src/aisle/harness/typed_stage_provider.py", + "matched_source:src/aisle/harness/typed_validation.py": "src/aisle/harness/typed_validation.py", + "matched_source:src/aisle/harness/typed_worker_launch.py": "src/aisle/harness/typed_worker_launch.py", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "src/aisle/harness/typed_worker_provisioning.py", + "matched_source:src/aisle/harness/validate.py": "src/aisle/harness/validate.py", + "matched_source:src/aisle/harness/worker_authority_probe.py": "src/aisle/harness/worker_authority_probe.py", + "matched_source:src/aisle/harness/worker_capability.py": "src/aisle/harness/worker_capability.py", + "matched_source:src/aisle/harness/worker_declaration.py": "src/aisle/harness/worker_declaration.py", + "matched_source:src/aisle/harness/worker_network_probe.py": "src/aisle/harness/worker_network_probe.py", + "matched_source:src/aisle/kinematics.py": "src/aisle/kinematics.py", + "matched_source:src/aisle/mobility/guard.py": "src/aisle/mobility/guard.py", + "matched_source:src/aisle/monolith/confinement.py": "src/aisle/monolith/confinement.py", + "matched_source:src/aisle/monolith/primitive_api.py": "src/aisle/monolith/primitive_api.py", + "matched_source:src/aisle/monolith/primitives.py": "src/aisle/monolith/primitives.py", + "matched_source:src/aisle/monolith/proxy.py": "src/aisle/monolith/proxy.py", + "matched_source:src/aisle/monolith/requests.py": "src/aisle/monolith/requests.py", + "matched_source:src/aisle/monolith/supervisor.py": "src/aisle/monolith/supervisor.py", + "matched_source:src/aisle/monolith/wire.py": "src/aisle/monolith/wire.py", + "matched_source:src/aisle/monolith/worker.py": "src/aisle/monolith/worker.py", + "matched_source:src/aisle/monolith/worker_config.py": "src/aisle/monolith/worker_config.py", + "matched_source:src/aisle/monolith/worker_launch.py": "src/aisle/monolith/worker_launch.py", + "matched_source:src/aisle/nodes/budget_guard.py": "src/aisle/nodes/budget_guard.py", + "matched_source:src/aisle/nodes/h6_fault.py": "src/aisle/nodes/h6_fault.py", + "matched_source:src/aisle/nodes/l2_pose.py": "src/aisle/nodes/l2_pose.py", + "matched_source:src/aisle/nodes/label_reader.py": "src/aisle/nodes/label_reader.py", + "matched_source:src/aisle/nodes/monolith_broker.py": "src/aisle/nodes/monolith_broker.py", + "matched_source:src/aisle/nodes/perception_session.py": "src/aisle/nodes/perception_session.py", + "matched_source:src/aisle/scenes/locations.toml": "src/aisle/scenes/locations.toml", + "matched_source:src/aisle/scenes/meds.toml": "src/aisle/scenes/meds.toml", + "matched_source:src/aisle/scenes/pharmacy.py": "src/aisle/scenes/pharmacy.py", + "matched_source:src/aisle/scenes/physics.toml": "src/aisle/scenes/physics.toml", + "matched_source:src/aisle/scenes/planogram.toml": "src/aisle/scenes/planogram.toml", + "matched_source:src/aisle/scenes/store.py": "src/aisle/scenes/store.py", + "matched_source:src/aisle/topics.py": "src/aisle/topics.py", + "matched_source:src/aisle/turn_node.py": "src/aisle/turn_node.py", + "matched_source:src/aisle/turns.py": "src/aisle/turns.py", + "matched_source:src/aisle/verifier/calibration.py": "src/aisle/verifier/calibration.py", + "matched_source:src/aisle/verifier/models.lock": "src/aisle/verifier/models.lock", + "matched_source:src/aisle/verifier/models.py": "src/aisle/verifier/models.py", + "matched_source:src/aisle/verifier/placement.toml": "src/aisle/verifier/placement.toml", + "matched_source:src/aisle/verifier/realistic.py": "src/aisle/verifier/realistic.py", + "matched_source:src/aisle/verifier/stages.py": "src/aisle/verifier/stages.py", + "matched_source:src/aisle/verifier/thresholds.toml": "src/aisle/verifier/thresholds.toml", + "matched_source:tools/campaign.py": "tools/campaign.py", + "matched_source:tools/matched_campaign.py": "tools/matched_campaign.py", + "monolithic_validators": "src/aisle/harness/monolithic.py", + "nested_host_documentation": "docs/monolithic/code-mode-admission.md", + "pilot_assignments_cli": "tools/pilot_assignments.py", + "pilot_records_cli": "tools/pilot_records.py", + "protocol": "analysis/freeze/cse-causal-study-v1/protocol.json", + "protocol_template": "analysis/statistics/protocol-template.json", + "provider_admission_documentation": "docs/monolithic/provider-admission.md", + "public_observation_source:analysis/monolithic-control/experts-t1-l2-v1.json": "analysis/monolithic-control/experts-t1-l2-v1.json", + "public_observation_source:analysis/monolithic-control/treatment-table-t1-l2-v1.json": "analysis/monolithic-control/treatment-table-t1-l2-v1.json", + "public_observation_source:docs/decisions/ADR-pilot-public-observations.md": "docs/decisions/ADR-pilot-public-observations.md", + "public_observation_source:docs/monolithic/public-observations.md": "docs/monolithic/public-observations.md", + "public_observation_source:graphs/pilot_t1_l2_monolithic.yaml": "graphs/pilot_t1_l2_monolithic.yaml", + "public_observation_source:graphs/pilot_t1_l2_typed.yaml": "graphs/pilot_t1_l2_typed.yaml", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_monolithic.json": "graphs/turn_plans/pilot_t1_l2_monolithic.json", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_typed.json": "graphs/turn_plans/pilot_t1_l2_typed.json", + "public_observation_source:registry/manifests/pilot-policy-surface.yaml": "registry/manifests/pilot-policy-surface.yaml", + "public_observation_source:src/aisle/harness/pilot_model_cache.py": "src/aisle/harness/pilot_model_cache.py", + "public_observation_source:src/aisle/harness/pilot_policy_surface.py": "src/aisle/harness/pilot_policy_surface.py", + "research_prompt": "harness/CLAUDE.research.md", + "seed_commitment_predecessor": "analysis/freeze/cse-causal-study-v23/freeze-manifest.json", + "session_runner": "tools/campaign.py", + "session_validators": "src/aisle/harness/causal_study.py", + "simulator_bridge": "src/aisle/nodes/dora_genesis.py", + "spec": "specs/500-typed-monolithic-causal-study.md", + "treatment_integrity": "src/aisle/harness/treatment_integrity.py", + "treatment_postflight": "src/aisle/harness/treatment_postflight.py", + "treatment_table": "analysis/monolithic-control/treatment-table-v1.json" + }, + "budgets": { + "campaign_ceiling": "harness/budget.toml", + "rollouts_per_session": 20, + "session_tokens": 200000, + "session_wall_s": 3600, + "sim_episodes_per_session": 160 + }, + "campaign_id": "cse-causal-study-v24", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose power", + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose freeze --output /tmp/cse-causal-study-v22-freeze-refusal.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-v24/freeze-manifest.json --allow-withheld-seeds" + ], + "decision_rules": { + "alpha": 0.05, + "equivalence_margin": { + "cost_at_equal_quality": { + "acceptance_probability": 0.1, + "heldout_quality": 0.1, + "rule": "CSE-5: both must pass or cost claims stay unconditional", + "sidedness": "two-sided 0.90" + } + }, + "multiplicity": "single primary endpoint", + "power": 0.8, + "primary_rule": "one-sided superiority at alpha 0.05 on the pooled Newcombe interval lower bound > 0 with no reversed stratum; otherwise weakened/rejected/undecidable by the #358 rule (CSE-3)", + "smallest_effect": { + "measure": "risk difference in session success", + "value": 0.25 + }, + "stopping_rule": "fixed n from the frozen power analysis using pilot-only control success; absolute floor 10 randomized assignments per arm per agent-system x task stratum (CSE-8); no optional stopping or extension" + }, + "endpoints": { + "primary": [ + { + "aggregation": "risk difference typed_dataflow - monolithic; exact arm intervals; Newcombe interval; per agent-system x task stratum before pooling (CSE-1, CSE-17)", + "direction": "higher", + "id": "E-CSE-1", + "outcome": "session_success", + "status": "inferential", + "unit": "agent_session" + } + ], + "secondary": [ + { + "aggregation": "counts by arm", + "direction": "descriptive", + "id": "E-CSE-2", + "outcome": "validation_and_launch_failure_class", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "Kaplan-Meier with censoring", + "direction": "lower", + "id": "E-CSE-3", + "outcome": "time_to_first_accepted_system", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "unconditional session estimands; no at-equal-quality wording unless CSE-5 equivalence passes", + "direction": "lower", + "id": "E-CSE-4", + "outcome": "tokens_tool_calls_wall_active_time_sim_work_rollouts", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "counts", + "direction": "descriptive", + "id": "E-CSE-5", + "outcome": "safety_interventions", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "dropped unless a blinded rubric is ratified before freeze (CSE-18)", + "direction": "n/a", + "id": "E-CSE-6", + "outcome": "artifact_legibility", + "status": "exploratory", + "unit": "agent_session" + } + ] + }, + "exclusions": { + "deviation_policy": "any behaviour-affecting change after registration is a documented deviation record and a new campaign version id; confirmatory data never revises the registered protocol", + "infrastructure": [ + "independent-controller-determined, arm-neutral facts only (CSE-14): host outage, provider interruption before first tool call, launcher crash before the session started" + ], + "rerun_policy": "a permitted rerun receives a new randomized id; the original row is retained with its exclusion reason and never replaced in place", + "treatment_integrity": [ + "contamination sentinel tripped (CSE-12)", + "model alias or CLI drift outside the frozen identity (CSE-6)", + "integrity verdict authored session-side (CSE-11)" + ] + }, + "hypotheses": [ + { + "direction": "typed_dataflow - monolithic > 0", + "id": "H-CSE-1", + "statement": "The bundled typed-dataflow surface raises the probability that a fresh coding-agent session produces a launching system passing the hidden held-out acceptance within budget, versus the equal-capability monolithic surface." + } + ], + "instrument_set": { + "agent_systems": [ + { + "binary": "claude", + "frozen": false, + "id": "claude-code", + "version_observed": "2.1.261" + }, + { + "binary": "codex", + "frozen": false, + "id": "codex-cli", + "version_observed": "0.151.0" + } + ], + "items": [ + "short_composition:TBD-by-bnd-task-band-calibration-v1", + "engineering:TBD-by-bnd-task-band-calibration-v1" + ], + "kind": "selected_task" + }, + "integrity_checks": [ + { + "gate": "CON-14 spec approval", + "kind": "external_review", + "owner_role": "project owner", + "record": null, + "status": "pending" + }, + { + "gate": "STA-12 independent statistical review", + "kind": "external_review", + "owner_role": "independent statistician", + "record": null, + "status": "pending" + }, + { + "gate": "MON-1 treatment table and expert parity (#344)", + "kind": "machine_check", + "owner_role": "parity harness; control surface exists, independent authorship and operator evidence remain pending", + "record": null, + "status": "pending" + }, + { + "gate": "BND-12 task-band freeze (#346)", + "kind": "machine_check", + "owner_role": "task-band auditor", + "record": null, + "status": "pending" + }, + { + "gate": "TRT confinement/mutation/agent-path (#353)", + "kind": "machine_check", + "owner_role": "integrity controller", + "record": null, + "status": "pending" + }, + { + "gate": "seed commitment verification", + "kind": "machine_check", + "owner_role": "holder of the original private seed and salt sources", + "record": null, + "status": "pending" + }, + { + "gate": "MON-8/MON-12 matched-session acceptance (#519)", + "kind": "machine_check", + "owner_role": "matched-session controller; complete acceptance evidence still required", + "record": null, + "status": "pending" + } + ], + "issue": 347, + "purpose": "confirmatory", + "schema_version": "aisle.freeze.declaration.v1", + "seed_commitment": { + "inherited_from": "analysis/freeze/cse-causal-study-v23/freeze-manifest.json", + "rule": "32 distinct integers in [1000,9999) drawn once from the OS CSPRNG, retained only under ~/aisle-private/freeze/cse-causal-study-v1/, disjoint by value from every committed dev/pilot seed list; revealed after the campaign closes", + "salt_source": "~/aisle-private/freeze/cse-causal-study-v1/salt.bin", + "seeds_source": "~/aisle-private/freeze/cse-causal-study-v1/seeds.json" + }, + "spec": "500", + "superseded": "cse-causal-study-v23 (session-bound capability audits and fail-closed rejection of unsupported loopback endpoint authority; public-observation and cache source coverage, study rules, inherited seed commitment and pending gates preserved; no collection authorization)" +} diff --git a/analysis/freeze/cse-causal-study-v24/freeze-manifest.json b/analysis/freeze/cse-causal-study-v24/freeze-manifest.json new file mode 100644 index 00000000..32d86484 --- /dev/null +++ b/analysis/freeze/cse-causal-study-v24/freeze-manifest.json @@ -0,0 +1,555 @@ +{ + "analysis_script_hashes": { + "src/aisle/harness/benchmark_statistics.py": "sha256:22cde3e9d7c5cc80874c7eec30aa2898e30476d6c939c94ab3009e8b12af641e", + "src/aisle/harness/causal_study.py": "sha256:1b1c53d6389881c146726d865a741a681f3d5039c9bf99558a53ee368250971e", + "src/aisle/harness/pilot_records.py": "sha256:68ac7ef9afd449d8527e2ffe99a9ef926e06253c466f3421bc1848e9aa094b70" + }, + "analysis_seed": 500001, + "artifact_hashes": { + "agent_adapters": "sha256:848f8f7da5ad978e53712991a688eb74f56c7a77deabee8afae717ad712666ca", + "analyzer": "sha256:22cde3e9d7c5cc80874c7eec30aa2898e30476d6c939c94ab3009e8b12af641e", + "budget_ceiling": "sha256:bcbb79aa146afed1d877da18093175ae144b4607db50119022ef4f8173122cda", + "dependency_declaration": "sha256:add4e718001da7678b404bdf37ea8d9889f563324ffba9b03fe65c46b172ee4c", + "dependency_lock": "sha256:9b8a82a17ba68c192f045405feed3ae7046af9568bddcb398292c20f0ed6f5a8", + "dora_runtime_selection": "sha256:f9eb7ffe285a0f0ed26d50303e31cf1e94d2f93ed62b599071dff5a183c15f53", + "frontend_conformance_decision": "sha256:5059582dc735ddb85f329c647070b42bcc689b2836e198d305f381e74f89edc2", + "frontend_request_authorization_documentation": "sha256:a1d4e65c0e467af4bd77922b95b12f26ee6478932eb5dd460de34b81249ca86b", + "matched_session_decision": "sha256:d88e0ef18c92cedff7b74dd3ffafed14d92affaec65beb9ddfec2801b365dc61", + "matched_session_documentation": "sha256:735e3e5d2032e7d2c3901d9b631181de4a345027a9afe29ef4dd9f9ae17aa1f4", + "matched_source:assets/so101/so101.urdf": "sha256:3a65d2d35e68a8d2f0c2cc176d19b884506543c93ba72980145b80abe276022c", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "sha256:6aac85db63a7061ab09950741eba8ac0e77410755d85c427731c45864adc1dc5", + "matched_source:docs/monolithic/candidates.json": "sha256:2fc51e6ff0e4b73a9b203d6ed1debf1cfa40ece9b21a35245d50b9a403f80abb", + "matched_source:docs/monolithic/experts-t1-l2.json": "sha256:68f01e1081afb36964f591fe0e58a630c8e734c8809833279ac608b1c554fe1a", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "sha256:31fede313d1573e29a83c42acc92883ae8ffeb36beee0a62a8d4481ad68cd591", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "sha256:2bfc9854fb5cffbe70be341799f208813eee8c02c9d54993b83ce86f6b80f054", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "sha256:544bd956688b8d85b54ff496dc9d98bdc42676c1c0c5fe5583e276614d4ed0d5", + "matched_source:env/limits.toml": "sha256:1ab1f4fd5d52e7949b81d21933daaa2c66e80ef07fdedf8217280468e0142edc", + "matched_source:experts/monolithic/expert_t1_l2.py": "sha256:93ade82db523fe808312f9d83c88ab3464eee303be1bfc3f824880bf1de7924c", + "matched_source:graphs/pilot_t1_l2_monolithic.yaml": "sha256:ebf12538d0b9def23ce4ea70a9feb375c98535f774bbf16d4d839aaa4e99d27c", + "matched_source:src/aisle/__init__.py": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "matched_source:src/aisle/embodiment.py": "sha256:d17c204492d680175cbaf365aaf164e2a6049da23ce125a9b9f21606b79f4cca", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "sha256:7e7fb09527b647c2bb13787011de8f9a4412e8b6f304e89efa107162d8150e82", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "sha256:459465e666c6a76a70e902c10c58905dfd8417b6c2ca3e038be99624e3542540", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "sha256:406d8f47d8d0b95a9d4ae104890e5e52764c9597e9271f75ff434a4ce24748d0", + "matched_source:src/aisle/harness/candidate_launch.py": "sha256:654eb90264703e3ce3ab599bb8e143047eaee288c8852c5b09c74e3147039270", + "matched_source:src/aisle/harness/candidates.py": "sha256:d1b441a30288a6c0500aaa765a24d4aeecca4f8d52e96f633befc50c84ecf5f5", + "matched_source:src/aisle/harness/cli.py": "sha256:c855f059d28409ffaa14be99ad3fc0bdc16fc33cf1a3c9584a75d968bd59d72d", + "matched_source:src/aisle/harness/code_mode_audit.py": "sha256:24638163f205408da1c6cf20a1b77c80a0d740a3f8ed44696eb90baf7de61f6a", + "matched_source:src/aisle/harness/code_mode_authority.py": "sha256:217ea025f10a4dc02b4b0ff2ff733adf4a695d1098b3c8ce0d2b27b8a140853d", + "matched_source:src/aisle/harness/code_mode_proxy.py": "sha256:333eee561d610489d2f48f9359ad94a3d9d8ff729fbae0fd3dec4acf64be2cb2", + "matched_source:src/aisle/harness/code_mode_runner.py": "sha256:4f91577dcfc6c239bf4111bd6735a64e97d6e22b7ca6287389abae09023cc25d", + "matched_source:src/aisle/harness/common.py": "sha256:ef62bf518a99c8c4154908bc9cf99e68402e6ae6c8e3fb5ec30c3f5fce9b5fac", + "matched_source:src/aisle/harness/frontend_app_server.py": "sha256:d21b9bf218efc3066399b8b22bbe3892e772433da315a9ecd40e2773369e9283", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "sha256:fb3cf5b8a115eb93e3f292dae694a4318cbe43e9552347fbc25ea8e43102ddbb", + "matched_source:src/aisle/harness/frontend_conformance.py": "sha256:54f13041eaa2eb8679b33a7c9e441e300d93818901cc8819503425cd025aa02f", + "matched_source:src/aisle/harness/frontend_dispatch.py": "sha256:6bff2abbacca6c332abdb5c5cb4cb3d688d8b00535e3f428633619bf91cd30c8", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "sha256:dc75024bd52c21c89a6112da847875a1d0ebf44f53a3704705e0bb8c81b9bc4a", + "matched_source:src/aisle/harness/frontend_effects.py": "sha256:bc539cecad349d27f68b1b2d6e435142c1119310b2129fb5a7c5854aa899d80e", + "matched_source:src/aisle/harness/frontend_qualification.py": "sha256:6c416f992cbc1418fb6fad52c9d0ab6bc3ff2e6fefd218f1bcf62b8500131d45", + "matched_source:src/aisle/harness/frontend_request_audit.py": "sha256:a4bc60b2ce1968ada1776324b72ad87025b93d366046029445fdf9b2e92f8830", + "matched_source:src/aisle/harness/frontend_request_authority.py": "sha256:0a061f1528b8bb2071dd1ba7e1d0bbee178b2c982bd6b688b916fb612522e163", + "matched_source:src/aisle/harness/guard_divergence.py": "sha256:af2c4755cabe46fabba913aee4819e05c10caaae79105615d50f8cf4567dad14", + "matched_source:src/aisle/harness/heldout_eval.py": "sha256:56c83164b04ae1b4623e8fd7a0579dd0a9caa7d5dbb891448643464a07893962", + "matched_source:src/aisle/harness/matched_app_server.py": "sha256:b7177caca8f1e4d3664b6cef76539b572c2ea6604fe0799c629bc50a52951b52", + "matched_source:src/aisle/harness/matched_collection.py": "sha256:b347127344a4c15fbb73fa9f3ac7c711a3d55b0fa368a11f76683bb1b5bbea14", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "sha256:b7f2b81397bcd232710408b9c6c9f463b3928e37f117d7310d519728f9b4b2c9", + "matched_source:src/aisle/harness/matched_evidence.py": "sha256:6d65476a9a923fd445ea9dad0334e521378353454b873ef59e2b0ff848c650f7", + "matched_source:src/aisle/harness/matched_frontend.py": "sha256:a4b9827af7dbf36f4af5228ca8c87ea36d2d32f8cffc7fcb8fea1923bcc09401", + "matched_source:src/aisle/harness/matched_run.py": "sha256:825fec659b8bdbb104d6fa07fa2adf29f4b23be74c1f3ec894a8ffcab6b68b64", + "matched_source:src/aisle/harness/matched_run_launch.py": "sha256:8701f034d0bd9873aa7981e3db47849d50bd4465f085e617e6e3193aac070539", + "matched_source:src/aisle/harness/matched_runtime.py": "sha256:c4d81e8309fcf5212315013dbf8e83347a65080e1221f37a0491757834ac2797", + "matched_source:src/aisle/harness/matched_session.py": "sha256:344ea0bc92705ef23f550045f1d4b26d55c7e2cf894133354c22068804887b21", + "matched_source:src/aisle/harness/matched_tool_service.py": "sha256:1bf6685addbeee541961e1053b2eea68cba66ce110667d8e7720398d49191a17", + "matched_source:src/aisle/harness/matched_tools.py": "sha256:eff206d64477d6a91f88c4889e2934a17bba254612c7dc72d542e799e11ef948", + "matched_source:src/aisle/harness/mcp_harness_authority.py": "sha256:f5760ef7a4e05a1444ee6ae5da3d9c37d0859cf767e2d96862b4d21803d646eb", + "matched_source:src/aisle/harness/mcp_harness_source.py": "sha256:f5a57dd19fc1c2966a8988ff925bf744384a4bf1368760427465340c9167990e", + "matched_source:src/aisle/harness/mcp_harness_transport.py": "sha256:a0c713cd82628109cd7324ff3deeaa33a66d37819a7c517b273a32e5e82cdaf3", + "matched_source:src/aisle/harness/monolith.py": "sha256:a12428eab867b869d902733e43626669b58f87218f0c40d1713712c094ef7414", + "matched_source:src/aisle/harness/monolithic.py": "sha256:1983ed8c26b2371c993c571f33f5c7e4ee81ea2597f9468b43d53c9970465ccf", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "sha256:4c0cb658f0148eb5d06ad69dceafd39148a2d4787e085cb624047c365c87bb24", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "sha256:23f2646571d1ed67c5b8e09e7b0b10bcc6cfa996b05b3161fe39f859d88c15ba", + "matched_source:src/aisle/harness/pilot_records.py": "sha256:68ac7ef9afd449d8527e2ffe99a9ef926e06253c466f3421bc1848e9aa094b70", + "matched_source:src/aisle/harness/provider_hosted.py": "sha256:82a3c8dd10449a07970e39322f649d6680e55fb935ee0dbdc67381b136e81068", + "matched_source:src/aisle/harness/provider_relay.py": "sha256:0b1698ac82da51458da5d4cdee6ca9f79334703d8c4a9dbc297af173ec0ff9a2", + "matched_source:src/aisle/harness/provider_response_authority.py": "sha256:9562a8de28967e2eb83910ea5062dc491c8588a6bb4a2a04407f44d59f2251fe", + "matched_source:src/aisle/harness/provider_runner.py": "sha256:2c39d11d67cc4da2f9a2c732e86ba4e9a4b8da88570bdb1ecd1364d4668831cb", + "matched_source:src/aisle/harness/provider_source_audit.py": "sha256:e3be1f9d70288b4f596427a39ab4274399e9fdc74e87043e3c1249596fe1872b", + "matched_source:src/aisle/harness/registry.py": "sha256:214da7bb8db26c80c79262fa79c40a624d8a9981463d38a9c4aa30bc989bc1fb", + "matched_source:src/aisle/harness/rollout.py": "sha256:371789bd5093ef5c9e5ebb92177de6470caf5ec93ceeb59627411852fa62684a", + "matched_source:src/aisle/harness/rollout_client.py": "sha256:18808db0aa2df356d1a7cc9a538585aa9dcc653abd120e761649f1dff2c27c31", + "matched_source:src/aisle/harness/simulator_work.py": "sha256:2ade87416ab8c5d8902e9734af850b7613ee9ee42f9ca672c9482625e15d3d91", + "matched_source:src/aisle/harness/traces.py": "sha256:c2e6a717a56ce47e296bdc0115b15de9361b72084638b4f3bc66f7e7d032546b", + "matched_source:src/aisle/harness/treatment_ambient.py": "sha256:13dce400a649bc0d01749661517138f8ad1f8cde106caceaedb9f33af6f6d856", + "matched_source:src/aisle/harness/treatment_confinement.py": "sha256:c46e448a63041119f1d0dbdb136ae982459d64a305867dae114e480410ed4f6c", + "matched_source:src/aisle/harness/treatment_integrity.py": "sha256:b37162c6f295c5686da9b3089eb3bd96b69f68f01f41da0a784c4017658b10a6", + "matched_source:src/aisle/harness/treatment_postflight.py": "sha256:0bf01138ef0c277c6fcd728fcf25f4f13795de679fb41ab58b868b81f6c2f0ca", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "sha256:607945d5a51e3a139ecd35a8dceaf77a2027b0f842a577587616e0490b35c663", + "matched_source:src/aisle/harness/typed_graph_audit.py": "sha256:ca315a2717668f6e1029c0940471df6320eb23af859a0bf41220180bbcae656c", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "sha256:a5cfd9b3b603f901370579f65320f995d4bc0c71852cf2ed64784d429a942abd", + "matched_source:src/aisle/harness/typed_graph_stage.py": "sha256:0ea37ad396fb3321978c5f8f63d869a8ba0a652f7377cad60f13060b5a5ccbfb", + "matched_source:src/aisle/harness/typed_node_host.py": "sha256:56a5f0ab8159071897aad22d49aae2ab1785e6de9a186412b7b5566db334d056", + "matched_source:src/aisle/harness/typed_node_requests.py": "sha256:c3712381bc7383beef1e3304ce05f6a4fe27fb16d9b7e97e7f51f8d7cb6de78b", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "sha256:d964517b767bdd087771bf9233c05de61604cf5f705124d6b83be05feb0cb577", + "matched_source:src/aisle/harness/typed_node_worker.py": "sha256:890cb125e12b05afa3609186ceae6d1d45d1d765f653e2c0b7345b00b2c24a89", + "matched_source:src/aisle/harness/typed_run_prepare.py": "sha256:787ba2edc04efa6df752fdba2112ff50e34d2061acb74471c52c78e813382063", + "matched_source:src/aisle/harness/typed_snapshot.py": "sha256:96003b648f09d1348375ca72709537dd80aa4ddecd60e87465011d221d3200ec", + "matched_source:src/aisle/harness/typed_stage_provider.py": "sha256:afe7e37ef08020f5bf19630ef9117af2bdf9d8fe68ceba7f4a352f7c61c8c9ff", + "matched_source:src/aisle/harness/typed_validation.py": "sha256:bf51adce9f8fe107a2586bb4ffe2aec0cff684a235b6fde5837a95223f37ffa2", + "matched_source:src/aisle/harness/typed_worker_launch.py": "sha256:0759d03e0d26bfa2b38e11771a2ba79a2d7b0f8ca816cd10f084fdfbaa7c93d0", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "sha256:0e594beec5a07829044900d4a93fcde8e2e0e9d50e1e0f7e19809f9d2845b3ba", + "matched_source:src/aisle/harness/validate.py": "sha256:ed82967e800af6ec4d1f5336374343df38a5ba437771dd94e24b06c7dc64f481", + "matched_source:src/aisle/harness/worker_authority_probe.py": "sha256:9ee0638062e062c500c7103664eecd541e5d68056bb83368223c09f145dad6ee", + "matched_source:src/aisle/harness/worker_capability.py": "sha256:272c82452cd6f81912593b4584729cb7bb0e8f528d662cb3021e8dfc6ce70e83", + "matched_source:src/aisle/harness/worker_declaration.py": "sha256:15f42f83cae62a8e917f010b3e1571dda81a3cc7e7e3d630ed2b697fb0e8c0c2", + "matched_source:src/aisle/harness/worker_network_probe.py": "sha256:eb0d75fcbd4e8242eb61701f07e02e83ee6e7d9053247df5572f699969bcceb2", + "matched_source:src/aisle/kinematics.py": "sha256:0656cb4c9c4769365c42fc175deefb69d7a7a3dda77991329b1cedf33f7d5aaa", + "matched_source:src/aisle/mobility/guard.py": "sha256:3d0d4ddd757a8ae1840807511f8816f85feff0f2103fc490a8d29db6fa3663d0", + "matched_source:src/aisle/monolith/confinement.py": "sha256:f207d08bac411ae0cb9a5986b3f5f4d55e55d1407e0c0e6889ec5d489b232969", + "matched_source:src/aisle/monolith/primitive_api.py": "sha256:d5a344125ca3f434e489325b6bc7e20422082d18abee8ba7cd4deccda901327a", + "matched_source:src/aisle/monolith/primitives.py": "sha256:e4b3ef279708ef5fbd6e114f524a6c727f3c9d39a41d56dbfbe73d26e3ac515b", + "matched_source:src/aisle/monolith/proxy.py": "sha256:191b4ce164ea431ccbe728ad9ca5b482a09894093ee389643e099c3774df4ca4", + "matched_source:src/aisle/monolith/requests.py": "sha256:04151bf8838eda75be2a9d0c91240607283fb826712b90a8f99eab5d83d30185", + "matched_source:src/aisle/monolith/supervisor.py": "sha256:7dbe215dc5bddd12d414d1b88e896000fc64a9c71746230518eb2f47f1ef1051", + "matched_source:src/aisle/monolith/wire.py": "sha256:3418587b67e3ce2481e68c2809aaafc8e17f244246d33b055ee05f06ac6177a6", + "matched_source:src/aisle/monolith/worker.py": "sha256:931e42442b9f69157eab17cb73f1fa235eb053da7d843915ef889df5f84fb2d7", + "matched_source:src/aisle/monolith/worker_config.py": "sha256:55a26fdfa8b701470e50d54ffa9384d0ed89a0dc157867ac7e0005671e623312", + "matched_source:src/aisle/monolith/worker_launch.py": "sha256:0e0d053037e1dc9aef27c0248b8adce21c29546ca38ca7ebff6fd0c41c14475b", + "matched_source:src/aisle/nodes/budget_guard.py": "sha256:e15bc5f70b295e7dfd32f72e4a5cf72dffca9da50e9e8f6bb18360fb38069abf", + "matched_source:src/aisle/nodes/h6_fault.py": "sha256:c93b8e1607b7627adef9d971753727b9c602be44ae539def9188344332a63485", + "matched_source:src/aisle/nodes/l2_pose.py": "sha256:d2889dac35af94cdf89a280cf734f672098ce95a4a347e1b9797545c2269fda2", + "matched_source:src/aisle/nodes/label_reader.py": "sha256:aeb51080744779f68fb56d6ff408993e92e3e0b929b7d353af7521537f252517", + "matched_source:src/aisle/nodes/monolith_broker.py": "sha256:6fb86d2fdcae8d9ac0314e4cab3100cd925c35a96f74249006392e042847c048", + "matched_source:src/aisle/nodes/perception_session.py": "sha256:cf0fd80a43a338711456d7db546167d082fb6ecc461a828f3cb0e2d42701fc9e", + "matched_source:src/aisle/scenes/locations.toml": "sha256:baea316611ce2f05751ab3058b4b46acbdd9801d8734faca0272c06b2565bcca", + "matched_source:src/aisle/scenes/meds.toml": "sha256:3b25c5eb6586b311fbe8aa3ffc9fdab76cb1b60c9781c3a7a31b12ab6b61dc6b", + "matched_source:src/aisle/scenes/pharmacy.py": "sha256:642e4a57c9225804b44d4e6909fad6514475598cf002dff300b9ce4455225981", + "matched_source:src/aisle/scenes/physics.toml": "sha256:7a400f5cb01a4a93e2a1daa07fa3b2fafd6598643e15965d779c924480f7c36c", + "matched_source:src/aisle/scenes/planogram.toml": "sha256:c579e207de210ed102ff81e15bf0877a91ce0d4ac31ebce2c31e484322f0e576", + "matched_source:src/aisle/scenes/store.py": "sha256:ea79ce1c91d2085ac920fa1ffe87c0c662c875e0e36da5866c1bc60ae5093a66", + "matched_source:src/aisle/topics.py": "sha256:431a0e0bee4c4c3d83f09f9452eb481f238ec0b8cab11b7370bebc117beeb26f", + "matched_source:src/aisle/turn_node.py": "sha256:0c6cd11cd802ebf8934a671caeb4b8a590e5f43b23f03c52515f51eac4288705", + "matched_source:src/aisle/turns.py": "sha256:f0014e3d8ea6b07d4effd10c5bf0b7de08b89ecec1e164580509ad25694b36a4", + "matched_source:src/aisle/verifier/calibration.py": "sha256:a00b91406d27a3b6123701a7a7270258633ad9906c28dbb6f1110fbb07b59568", + "matched_source:src/aisle/verifier/models.lock": "sha256:3fa7271740f3ecd361213f45de140452f41216b80600e343e2daa9f2a413eab5", + "matched_source:src/aisle/verifier/models.py": "sha256:1b2905181bc757f8390666c2a4c7303f674a6d8a31232e8feae582b5cc24909f", + "matched_source:src/aisle/verifier/placement.toml": "sha256:8d8f37165cf4bb4214486597689d40d5df8fb36e2bb8f15e65f6be8801d3717f", + "matched_source:src/aisle/verifier/realistic.py": "sha256:769f6295d3c15a5be02da8ec815a401728059608f2ae92c904d5bec0dd18d156", + "matched_source:src/aisle/verifier/stages.py": "sha256:63848f9e4c0e59020461e3350bf8ca2a8d0ae93cc45cd43c5786c7b6f95a443c", + "matched_source:src/aisle/verifier/thresholds.toml": "sha256:567ac0544a4c382d3ce6547d8df868e6ca37c056b146cf0d1ee11934b1be16d3", + "matched_source:tools/campaign.py": "sha256:cb33ab83f9125ea9708a73f639ca38ad539fbc77e6c5b03e0b49dc2e66abd8f0", + "matched_source:tools/matched_campaign.py": "sha256:71e648035529877ee68b26eb2a4ccf5c0126e7872e3e9f6134785a1b63554609", + "monolithic_validators": "sha256:1983ed8c26b2371c993c571f33f5c7e4ee81ea2597f9468b43d53c9970465ccf", + "nested_host_documentation": "sha256:1c4777e4d8a29882ae5332f96df6b175423e7e5d1f279357fdb1286bace12959", + "pilot_assignments_cli": "sha256:bafcfdf4f79d5d0abd7442355454ea7ca476928afd57ed7c83f38da0c3726cc8", + "pilot_records_cli": "sha256:51438c6e14fca6368e1b91ebce77fd13c6bcc821edb61666553750a91d3ab75f", + "protocol": "sha256:459ce9961e3f5c7975ca38463e1807ca447f4463b0c9ad3e8eab1831b06b1d58", + "protocol_template": "sha256:9d636f3dc462a2e051ee0b3ded565aff1c5be86331e5b1b6692988509ccf088e", + "provider_admission_documentation": "sha256:856851935d15d8cb0c5653548ae9312441263cce9a3300e36d0863f5bdb96cf4", + "public_observation_source:analysis/monolithic-control/experts-t1-l2-v1.json": "sha256:eca1034c63620f5a901a0b2da4c157391643bc4e5aaf48457ac172ff8a656288", + "public_observation_source:analysis/monolithic-control/treatment-table-t1-l2-v1.json": "sha256:58f7164a95500f514a815b44ef1a681462c13127c3fbff9f887ae3873f21eac2", + "public_observation_source:docs/decisions/ADR-pilot-public-observations.md": "sha256:0bf46aac693fa9b9e78997d6843cb87112f7716d006a0f4a63e987f3eca5ea5e", + "public_observation_source:docs/monolithic/public-observations.md": "sha256:1b395b766ef5345169e2047a3a86f80648f5d265f3d4445315f314214c7068b7", + "public_observation_source:graphs/pilot_t1_l2_monolithic.yaml": "sha256:ebf12538d0b9def23ce4ea70a9feb375c98535f774bbf16d4d839aaa4e99d27c", + "public_observation_source:graphs/pilot_t1_l2_typed.yaml": "sha256:9dd2a3f4a71ed954987861e80a163c644e3a0aec4f3556b5fa353fa8f83aa8cb", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_monolithic.json": "sha256:089558615e62539fb8121c6521d1bcc2af1403d4e896be9ef731b353e9b55d8c", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_typed.json": "sha256:605b00af15cc3e8ba0331468532277c2e8ca4d3bcc196e84978d957fde59f23e", + "public_observation_source:registry/manifests/pilot-policy-surface.yaml": "sha256:5e189856d64aa49c90b573879d5b71abe230a2790afad5b9214a061d521de655", + "public_observation_source:src/aisle/harness/pilot_model_cache.py": "sha256:17a90497a33015388f5426dad4d5c3e29700a8b48ccb47a6e02380c3274339f5", + "public_observation_source:src/aisle/harness/pilot_policy_surface.py": "sha256:bb08dc54cf6a86907d68bab554baab1d562ec754b2736812e68798b96395cf80", + "research_prompt": "sha256:e736bbd6caa7236dbfafe92ae42c0c7f13088764af1b753f6685d398c111b7c0", + "seed_commitment_predecessor": "sha256:67bae9d94d48f1c421490a9597fd30b2860e12d5a073f30d29d860983a0bd6b1", + "session_runner": "sha256:cb33ab83f9125ea9708a73f639ca38ad539fbc77e6c5b03e0b49dc2e66abd8f0", + "session_validators": "sha256:1b1c53d6389881c146726d865a741a681f3d5039c9bf99558a53ee368250971e", + "simulator_bridge": "sha256:196837073251bedefd0365d220aa94407612f7670ed0b1b39c7319d63a1c18f8", + "spec": "sha256:f5096c587cd9687859f2dc80ec807d1bbf959cb4e05c152154e71e47bdc1f593", + "treatment_integrity": "sha256:b37162c6f295c5686da9b3089eb3bd96b69f68f01f41da0a784c4017658b10a6", + "treatment_postflight": "sha256:0bf01138ef0c277c6fcd728fcf25f4f13795de679fb41ab58b868b81f6c2f0ca", + "treatment_table": "sha256:0f860aefee6272ae5ca92f0a6d704d9f2dd924d931eed97d0b666f09361217d8" + }, + "campaign_id": "cse-causal-study-v24", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose power", + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose freeze --output /tmp/cse-causal-study-v22-freeze-refusal.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-v24/freeze-manifest.json --allow-withheld-seeds" + ], + "declaration": { + "analysis": { + "scripts": [ + "src/aisle/harness/benchmark_statistics.py", + "src/aisle/harness/causal_study.py", + "src/aisle/harness/pilot_records.py" + ], + "seed": 500001 + }, + "artifacts": { + "agent_adapters": "tools/agent_adapters.py", + "analyzer": "src/aisle/harness/benchmark_statistics.py", + "budget_ceiling": "harness/budget.toml", + "dependency_declaration": "pyproject.toml", + "dependency_lock": "uv.lock", + "dora_runtime_selection": "dora-runtime.json", + "frontend_conformance_decision": "docs/decisions/ADR-65.md", + "frontend_request_authorization_documentation": "docs/benchmark/v1/frontend-request-authorization.md", + "matched_session_decision": "docs/decisions/ADR-62.md", + "matched_session_documentation": "docs/monolithic/matched-session.md", + "matched_source:assets/so101/so101.urdf": "assets/so101/so101.urdf", + "matched_source:docs/monolithic/allowlist-t1-l2.json": "docs/monolithic/allowlist-t1-l2.json", + "matched_source:docs/monolithic/candidates.json": "docs/monolithic/candidates.json", + "matched_source:docs/monolithic/experts-t1-l2.json": "docs/monolithic/experts-t1-l2.json", + "matched_source:docs/monolithic/interface-map-t1-l2.json": "docs/monolithic/interface-map-t1-l2.json", + "matched_source:docs/monolithic/parity-protocol-t1-l2.json": "docs/monolithic/parity-protocol-t1-l2.json", + "matched_source:docs/monolithic/treatment-table-t1-l2.json": "docs/monolithic/treatment-table-t1-l2.json", + "matched_source:env/limits.toml": "env/limits.toml", + "matched_source:experts/monolithic/expert_t1_l2.py": "experts/monolithic/expert_t1_l2.py", + "matched_source:graphs/pilot_t1_l2_monolithic.yaml": "graphs/pilot_t1_l2_monolithic.yaml", + "matched_source:src/aisle/__init__.py": "src/aisle/__init__.py", + "matched_source:src/aisle/embodiment.py": "src/aisle/embodiment.py", + "matched_source:src/aisle/harness/_code_mode_protocol/__init__.py": "src/aisle/harness/_code_mode_protocol/__init__.py", + "matched_source:src/aisle/harness/_code_mode_protocol/descriptor.pb": "src/aisle/harness/_code_mode_protocol/descriptor.pb", + "matched_source:src/aisle/harness/_code_mode_protocol/protocol.proto": "src/aisle/harness/_code_mode_protocol/protocol.proto", + "matched_source:src/aisle/harness/candidate_launch.py": "src/aisle/harness/candidate_launch.py", + "matched_source:src/aisle/harness/candidates.py": "src/aisle/harness/candidates.py", + "matched_source:src/aisle/harness/cli.py": "src/aisle/harness/cli.py", + "matched_source:src/aisle/harness/code_mode_audit.py": "src/aisle/harness/code_mode_audit.py", + "matched_source:src/aisle/harness/code_mode_authority.py": "src/aisle/harness/code_mode_authority.py", + "matched_source:src/aisle/harness/code_mode_proxy.py": "src/aisle/harness/code_mode_proxy.py", + "matched_source:src/aisle/harness/code_mode_runner.py": "src/aisle/harness/code_mode_runner.py", + "matched_source:src/aisle/harness/common.py": "src/aisle/harness/common.py", + "matched_source:src/aisle/harness/frontend_app_server.py": "src/aisle/harness/frontend_app_server.py", + "matched_source:src/aisle/harness/frontend_app_server_audit.py": "src/aisle/harness/frontend_app_server_audit.py", + "matched_source:src/aisle/harness/frontend_conformance.py": "src/aisle/harness/frontend_conformance.py", + "matched_source:src/aisle/harness/frontend_dispatch.py": "src/aisle/harness/frontend_dispatch.py", + "matched_source:src/aisle/harness/frontend_dispatch_audit.py": "src/aisle/harness/frontend_dispatch_audit.py", + "matched_source:src/aisle/harness/frontend_effects.py": "src/aisle/harness/frontend_effects.py", + "matched_source:src/aisle/harness/frontend_qualification.py": "src/aisle/harness/frontend_qualification.py", + "matched_source:src/aisle/harness/frontend_request_audit.py": "src/aisle/harness/frontend_request_audit.py", + "matched_source:src/aisle/harness/frontend_request_authority.py": "src/aisle/harness/frontend_request_authority.py", + "matched_source:src/aisle/harness/guard_divergence.py": "src/aisle/harness/guard_divergence.py", + "matched_source:src/aisle/harness/heldout_eval.py": "src/aisle/harness/heldout_eval.py", + "matched_source:src/aisle/harness/matched_app_server.py": "src/aisle/harness/matched_app_server.py", + "matched_source:src/aisle/harness/matched_collection.py": "src/aisle/harness/matched_collection.py", + "matched_source:src/aisle/harness/matched_dynamic_run.py": "src/aisle/harness/matched_dynamic_run.py", + "matched_source:src/aisle/harness/matched_evidence.py": "src/aisle/harness/matched_evidence.py", + "matched_source:src/aisle/harness/matched_frontend.py": "src/aisle/harness/matched_frontend.py", + "matched_source:src/aisle/harness/matched_run.py": "src/aisle/harness/matched_run.py", + "matched_source:src/aisle/harness/matched_run_launch.py": "src/aisle/harness/matched_run_launch.py", + "matched_source:src/aisle/harness/matched_runtime.py": "src/aisle/harness/matched_runtime.py", + "matched_source:src/aisle/harness/matched_session.py": "src/aisle/harness/matched_session.py", + "matched_source:src/aisle/harness/matched_tool_service.py": "src/aisle/harness/matched_tool_service.py", + "matched_source:src/aisle/harness/matched_tools.py": "src/aisle/harness/matched_tools.py", + "matched_source:src/aisle/harness/mcp_harness_authority.py": "src/aisle/harness/mcp_harness_authority.py", + "matched_source:src/aisle/harness/mcp_harness_source.py": "src/aisle/harness/mcp_harness_source.py", + "matched_source:src/aisle/harness/mcp_harness_transport.py": "src/aisle/harness/mcp_harness_transport.py", + "matched_source:src/aisle/harness/monolith.py": "src/aisle/harness/monolith.py", + "matched_source:src/aisle/harness/monolithic.py": "src/aisle/harness/monolithic.py", + "matched_source:src/aisle/harness/monolithic_run_evidence.py": "src/aisle/harness/monolithic_run_evidence.py", + "matched_source:src/aisle/harness/monolithic_run_prepare.py": "src/aisle/harness/monolithic_run_prepare.py", + "matched_source:src/aisle/harness/pilot_records.py": "src/aisle/harness/pilot_records.py", + "matched_source:src/aisle/harness/provider_hosted.py": "src/aisle/harness/provider_hosted.py", + "matched_source:src/aisle/harness/provider_relay.py": "src/aisle/harness/provider_relay.py", + "matched_source:src/aisle/harness/provider_response_authority.py": "src/aisle/harness/provider_response_authority.py", + "matched_source:src/aisle/harness/provider_runner.py": "src/aisle/harness/provider_runner.py", + "matched_source:src/aisle/harness/provider_source_audit.py": "src/aisle/harness/provider_source_audit.py", + "matched_source:src/aisle/harness/registry.py": "src/aisle/harness/registry.py", + "matched_source:src/aisle/harness/rollout.py": "src/aisle/harness/rollout.py", + "matched_source:src/aisle/harness/rollout_client.py": "src/aisle/harness/rollout_client.py", + "matched_source:src/aisle/harness/simulator_work.py": "src/aisle/harness/simulator_work.py", + "matched_source:src/aisle/harness/traces.py": "src/aisle/harness/traces.py", + "matched_source:src/aisle/harness/treatment_ambient.py": "src/aisle/harness/treatment_ambient.py", + "matched_source:src/aisle/harness/treatment_confinement.py": "src/aisle/harness/treatment_confinement.py", + "matched_source:src/aisle/harness/treatment_integrity.py": "src/aisle/harness/treatment_integrity.py", + "matched_source:src/aisle/harness/treatment_postflight.py": "src/aisle/harness/treatment_postflight.py", + "matched_source:src/aisle/harness/typed_execution_bundle.py": "src/aisle/harness/typed_execution_bundle.py", + "matched_source:src/aisle/harness/typed_graph_audit.py": "src/aisle/harness/typed_graph_audit.py", + "matched_source:src/aisle/harness/typed_graph_hosts.py": "src/aisle/harness/typed_graph_hosts.py", + "matched_source:src/aisle/harness/typed_graph_stage.py": "src/aisle/harness/typed_graph_stage.py", + "matched_source:src/aisle/harness/typed_node_host.py": "src/aisle/harness/typed_node_host.py", + "matched_source:src/aisle/harness/typed_node_requests.py": "src/aisle/harness/typed_node_requests.py", + "matched_source:src/aisle/harness/typed_node_supervisor.py": "src/aisle/harness/typed_node_supervisor.py", + "matched_source:src/aisle/harness/typed_node_worker.py": "src/aisle/harness/typed_node_worker.py", + "matched_source:src/aisle/harness/typed_run_prepare.py": "src/aisle/harness/typed_run_prepare.py", + "matched_source:src/aisle/harness/typed_snapshot.py": "src/aisle/harness/typed_snapshot.py", + "matched_source:src/aisle/harness/typed_stage_provider.py": "src/aisle/harness/typed_stage_provider.py", + "matched_source:src/aisle/harness/typed_validation.py": "src/aisle/harness/typed_validation.py", + "matched_source:src/aisle/harness/typed_worker_launch.py": "src/aisle/harness/typed_worker_launch.py", + "matched_source:src/aisle/harness/typed_worker_provisioning.py": "src/aisle/harness/typed_worker_provisioning.py", + "matched_source:src/aisle/harness/validate.py": "src/aisle/harness/validate.py", + "matched_source:src/aisle/harness/worker_authority_probe.py": "src/aisle/harness/worker_authority_probe.py", + "matched_source:src/aisle/harness/worker_capability.py": "src/aisle/harness/worker_capability.py", + "matched_source:src/aisle/harness/worker_declaration.py": "src/aisle/harness/worker_declaration.py", + "matched_source:src/aisle/harness/worker_network_probe.py": "src/aisle/harness/worker_network_probe.py", + "matched_source:src/aisle/kinematics.py": "src/aisle/kinematics.py", + "matched_source:src/aisle/mobility/guard.py": "src/aisle/mobility/guard.py", + "matched_source:src/aisle/monolith/confinement.py": "src/aisle/monolith/confinement.py", + "matched_source:src/aisle/monolith/primitive_api.py": "src/aisle/monolith/primitive_api.py", + "matched_source:src/aisle/monolith/primitives.py": "src/aisle/monolith/primitives.py", + "matched_source:src/aisle/monolith/proxy.py": "src/aisle/monolith/proxy.py", + "matched_source:src/aisle/monolith/requests.py": "src/aisle/monolith/requests.py", + "matched_source:src/aisle/monolith/supervisor.py": "src/aisle/monolith/supervisor.py", + "matched_source:src/aisle/monolith/wire.py": "src/aisle/monolith/wire.py", + "matched_source:src/aisle/monolith/worker.py": "src/aisle/monolith/worker.py", + "matched_source:src/aisle/monolith/worker_config.py": "src/aisle/monolith/worker_config.py", + "matched_source:src/aisle/monolith/worker_launch.py": "src/aisle/monolith/worker_launch.py", + "matched_source:src/aisle/nodes/budget_guard.py": "src/aisle/nodes/budget_guard.py", + "matched_source:src/aisle/nodes/h6_fault.py": "src/aisle/nodes/h6_fault.py", + "matched_source:src/aisle/nodes/l2_pose.py": "src/aisle/nodes/l2_pose.py", + "matched_source:src/aisle/nodes/label_reader.py": "src/aisle/nodes/label_reader.py", + "matched_source:src/aisle/nodes/monolith_broker.py": "src/aisle/nodes/monolith_broker.py", + "matched_source:src/aisle/nodes/perception_session.py": "src/aisle/nodes/perception_session.py", + "matched_source:src/aisle/scenes/locations.toml": "src/aisle/scenes/locations.toml", + "matched_source:src/aisle/scenes/meds.toml": "src/aisle/scenes/meds.toml", + "matched_source:src/aisle/scenes/pharmacy.py": "src/aisle/scenes/pharmacy.py", + "matched_source:src/aisle/scenes/physics.toml": "src/aisle/scenes/physics.toml", + "matched_source:src/aisle/scenes/planogram.toml": "src/aisle/scenes/planogram.toml", + "matched_source:src/aisle/scenes/store.py": "src/aisle/scenes/store.py", + "matched_source:src/aisle/topics.py": "src/aisle/topics.py", + "matched_source:src/aisle/turn_node.py": "src/aisle/turn_node.py", + "matched_source:src/aisle/turns.py": "src/aisle/turns.py", + "matched_source:src/aisle/verifier/calibration.py": "src/aisle/verifier/calibration.py", + "matched_source:src/aisle/verifier/models.lock": "src/aisle/verifier/models.lock", + "matched_source:src/aisle/verifier/models.py": "src/aisle/verifier/models.py", + "matched_source:src/aisle/verifier/placement.toml": "src/aisle/verifier/placement.toml", + "matched_source:src/aisle/verifier/realistic.py": "src/aisle/verifier/realistic.py", + "matched_source:src/aisle/verifier/stages.py": "src/aisle/verifier/stages.py", + "matched_source:src/aisle/verifier/thresholds.toml": "src/aisle/verifier/thresholds.toml", + "matched_source:tools/campaign.py": "tools/campaign.py", + "matched_source:tools/matched_campaign.py": "tools/matched_campaign.py", + "monolithic_validators": "src/aisle/harness/monolithic.py", + "nested_host_documentation": "docs/monolithic/code-mode-admission.md", + "pilot_assignments_cli": "tools/pilot_assignments.py", + "pilot_records_cli": "tools/pilot_records.py", + "protocol": "analysis/freeze/cse-causal-study-v1/protocol.json", + "protocol_template": "analysis/statistics/protocol-template.json", + "provider_admission_documentation": "docs/monolithic/provider-admission.md", + "public_observation_source:analysis/monolithic-control/experts-t1-l2-v1.json": "analysis/monolithic-control/experts-t1-l2-v1.json", + "public_observation_source:analysis/monolithic-control/treatment-table-t1-l2-v1.json": "analysis/monolithic-control/treatment-table-t1-l2-v1.json", + "public_observation_source:docs/decisions/ADR-pilot-public-observations.md": "docs/decisions/ADR-pilot-public-observations.md", + "public_observation_source:docs/monolithic/public-observations.md": "docs/monolithic/public-observations.md", + "public_observation_source:graphs/pilot_t1_l2_monolithic.yaml": "graphs/pilot_t1_l2_monolithic.yaml", + "public_observation_source:graphs/pilot_t1_l2_typed.yaml": "graphs/pilot_t1_l2_typed.yaml", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_monolithic.json": "graphs/turn_plans/pilot_t1_l2_monolithic.json", + "public_observation_source:graphs/turn_plans/pilot_t1_l2_typed.json": "graphs/turn_plans/pilot_t1_l2_typed.json", + "public_observation_source:registry/manifests/pilot-policy-surface.yaml": "registry/manifests/pilot-policy-surface.yaml", + "public_observation_source:src/aisle/harness/pilot_model_cache.py": "src/aisle/harness/pilot_model_cache.py", + "public_observation_source:src/aisle/harness/pilot_policy_surface.py": "src/aisle/harness/pilot_policy_surface.py", + "research_prompt": "harness/CLAUDE.research.md", + "seed_commitment_predecessor": "analysis/freeze/cse-causal-study-v23/freeze-manifest.json", + "session_runner": "tools/campaign.py", + "session_validators": "src/aisle/harness/causal_study.py", + "simulator_bridge": "src/aisle/nodes/dora_genesis.py", + "spec": "specs/500-typed-monolithic-causal-study.md", + "treatment_integrity": "src/aisle/harness/treatment_integrity.py", + "treatment_postflight": "src/aisle/harness/treatment_postflight.py", + "treatment_table": "analysis/monolithic-control/treatment-table-v1.json" + }, + "budgets": { + "campaign_ceiling": "harness/budget.toml", + "rollouts_per_session": 20, + "session_tokens": 200000, + "session_wall_s": 3600, + "sim_episodes_per_session": 160 + }, + "campaign_id": "cse-causal-study-v24", + "commands": [ + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose power", + "uv run harness stats validate --protocol analysis/freeze/cse-causal-study-v1/protocol.json --purpose freeze --output /tmp/cse-causal-study-v22-freeze-refusal.json", + "uv run harness freeze check --manifest analysis/freeze/cse-causal-study-v24/freeze-manifest.json --allow-withheld-seeds" + ], + "decision_rules": { + "alpha": 0.05, + "equivalence_margin": { + "cost_at_equal_quality": { + "acceptance_probability": 0.1, + "heldout_quality": 0.1, + "rule": "CSE-5: both must pass or cost claims stay unconditional", + "sidedness": "two-sided 0.90" + } + }, + "multiplicity": "single primary endpoint", + "power": 0.8, + "primary_rule": "one-sided superiority at alpha 0.05 on the pooled Newcombe interval lower bound > 0 with no reversed stratum; otherwise weakened/rejected/undecidable by the #358 rule (CSE-3)", + "smallest_effect": { + "measure": "risk difference in session success", + "value": 0.25 + }, + "stopping_rule": "fixed n from the frozen power analysis using pilot-only control success; absolute floor 10 randomized assignments per arm per agent-system x task stratum (CSE-8); no optional stopping or extension" + }, + "endpoints": { + "primary": [ + { + "aggregation": "risk difference typed_dataflow - monolithic; exact arm intervals; Newcombe interval; per agent-system x task stratum before pooling (CSE-1, CSE-17)", + "direction": "higher", + "id": "E-CSE-1", + "outcome": "session_success", + "status": "inferential", + "unit": "agent_session" + } + ], + "secondary": [ + { + "aggregation": "counts by arm", + "direction": "descriptive", + "id": "E-CSE-2", + "outcome": "validation_and_launch_failure_class", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "Kaplan-Meier with censoring", + "direction": "lower", + "id": "E-CSE-3", + "outcome": "time_to_first_accepted_system", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "unconditional session estimands; no at-equal-quality wording unless CSE-5 equivalence passes", + "direction": "lower", + "id": "E-CSE-4", + "outcome": "tokens_tool_calls_wall_active_time_sim_work_rollouts", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "counts", + "direction": "descriptive", + "id": "E-CSE-5", + "outcome": "safety_interventions", + "status": "descriptive", + "unit": "agent_session" + }, + { + "aggregation": "dropped unless a blinded rubric is ratified before freeze (CSE-18)", + "direction": "n/a", + "id": "E-CSE-6", + "outcome": "artifact_legibility", + "status": "exploratory", + "unit": "agent_session" + } + ] + }, + "exclusions": { + "deviation_policy": "any behaviour-affecting change after registration is a documented deviation record and a new campaign version id; confirmatory data never revises the registered protocol", + "infrastructure": [ + "independent-controller-determined, arm-neutral facts only (CSE-14): host outage, provider interruption before first tool call, launcher crash before the session started" + ], + "rerun_policy": "a permitted rerun receives a new randomized id; the original row is retained with its exclusion reason and never replaced in place", + "treatment_integrity": [ + "contamination sentinel tripped (CSE-12)", + "model alias or CLI drift outside the frozen identity (CSE-6)", + "integrity verdict authored session-side (CSE-11)" + ] + }, + "hypotheses": [ + { + "direction": "typed_dataflow - monolithic > 0", + "id": "H-CSE-1", + "statement": "The bundled typed-dataflow surface raises the probability that a fresh coding-agent session produces a launching system passing the hidden held-out acceptance within budget, versus the equal-capability monolithic surface." + } + ], + "instrument_set": { + "agent_systems": [ + { + "binary": "claude", + "frozen": false, + "id": "claude-code", + "version_observed": "2.1.261" + }, + { + "binary": "codex", + "frozen": false, + "id": "codex-cli", + "version_observed": "0.151.0" + } + ], + "items": [ + "short_composition:TBD-by-bnd-task-band-calibration-v1", + "engineering:TBD-by-bnd-task-band-calibration-v1" + ], + "kind": "selected_task" + }, + "integrity_checks": [ + { + "gate": "CON-14 spec approval", + "kind": "external_review", + "owner_role": "project owner", + "record": null, + "status": "pending" + }, + { + "gate": "STA-12 independent statistical review", + "kind": "external_review", + "owner_role": "independent statistician", + "record": null, + "status": "pending" + }, + { + "gate": "MON-1 treatment table and expert parity (#344)", + "kind": "machine_check", + "owner_role": "parity harness; control surface exists, independent authorship and operator evidence remain pending", + "record": null, + "status": "pending" + }, + { + "gate": "BND-12 task-band freeze (#346)", + "kind": "machine_check", + "owner_role": "task-band auditor", + "record": null, + "status": "pending" + }, + { + "gate": "TRT confinement/mutation/agent-path (#353)", + "kind": "machine_check", + "owner_role": "integrity controller", + "record": null, + "status": "pending" + }, + { + "gate": "seed commitment verification", + "kind": "machine_check", + "owner_role": "holder of the original private seed and salt sources", + "record": null, + "status": "pending" + }, + { + "gate": "MON-8/MON-12 matched-session acceptance (#519)", + "kind": "machine_check", + "owner_role": "matched-session controller; complete acceptance evidence still required", + "record": null, + "status": "pending" + } + ], + "issue": 347, + "purpose": "confirmatory", + "schema_version": "aisle.freeze.declaration.v1", + "seed_commitment": { + "inherited_from": "analysis/freeze/cse-causal-study-v23/freeze-manifest.json", + "rule": "32 distinct integers in [1000,9999) drawn once from the OS CSPRNG, retained only under ~/aisle-private/freeze/cse-causal-study-v1/, disjoint by value from every committed dev/pilot seed list; revealed after the campaign closes", + "salt_source": "~/aisle-private/freeze/cse-causal-study-v1/salt.bin", + "seeds_source": "~/aisle-private/freeze/cse-causal-study-v1/seeds.json" + }, + "spec": "500", + "superseded": "cse-causal-study-v23 (session-bound capability audits and fail-closed rejection of unsupported loopback endpoint authority; public-observation and cache source coverage, study rules, inherited seed commitment and pending gates preserved; no collection authorization)" + }, + "declaration_sha256": "sha256:148b1fadc570e572d5301f021b671f0b2ad1ef8970efa4c6406e9f528496a892", + "frozen": false, + "gate_record_hashes": {}, + "git_head": "cf834d629865b99de30308c73513060ebc8f48f6", + "issue": 347, + "pending_gates": [ + "BND-12 task-band freeze (#346)", + "CON-14 spec approval", + "MON-1 treatment table and expert parity (#344)", + "MON-8/MON-12 matched-session acceptance (#519)", + "STA-12 independent statistical review", + "TRT confinement/mutation/agent-path (#353)", + "seed commitment verification" + ], + "purpose": "confirmatory", + "schema_version": "aisle.freeze.manifest.v1", + "seed_commitment": "sha256:f3f4e5efe4d573e1159d93e7b53a35a50620606e3dc72d0b5d5057611a2b7522", + "spec": "500", + "status": "registered_pending_review", + "timestamp": null, + "timestamp_source": null +} diff --git a/analysis/treatment-integrity/macos-confinement/README.md b/analysis/treatment-integrity/macos-confinement/README.md index bb34ffe5..ab23fead 100644 --- a/analysis/treatment-integrity/macos-confinement/README.md +++ b/analysis/treatment-integrity/macos-confinement/README.md @@ -23,3 +23,15 @@ adapter hash, imported `system.sb` hash, compiled-profile hash, and policy id. Temporary synthetic hidden bytes and the compiled profile are removed after the audit; only their hashes, exposure booleans, and role-scoped policy metadata are retained. + +Since the session-bound attestation (2026-09-14), `audit-macos --policy +` runs the same matrix under a session's exact policy +(`MacOSPolicy.canonical_dict` JSON) so the retained record's profile hash and +policy id are the ones `wrap_verified_command` demands; `policy_source` in the +report says whether the synthetic or a session policy was audited, and the +report's hidden roots are digests, not paths. Only `deny-external` is supported. +The proposed `loopback` grant is refused because a port pin cannot isolate +one service across the host's IPv4 and IPv6 addresses. Previously passing +loopback attestations cannot authorize a launch. This change preserves +session-bound audit support but does not enable relay-backed confined sessions. +See `docs/monolithic/matched-session.md`. diff --git a/docs/monolithic/matched-session.md b/docs/monolithic/matched-session.md index b6cb1d5c..ea738cae 100644 --- a/docs/monolithic/matched-session.md +++ b/docs/monolithic/matched-session.md @@ -71,6 +71,37 @@ process record, including on a nonzero process exit. Final snapshots and postflight failures remain in `matched-session.json` alongside the original launcher failure. +The `attestation` must have been produced under the arm's exact confinement +policy, or the launch wrapper refuses it: `python -m +aisle.harness.treatment_confinement audit-macos --policy +--output ` runs the full deny/allow matrix under that policy +(its sentinels are placed inside the policy's own visible, output and hidden +roots and removed afterwards) and retains the profile hash and policy id the +wrapper demands (TRT-5/TRT-7). The policy must admit the audit's probes +(`/bin/bash`, `/bin/cat`, the Apple developer Git, and either `/usr/bin/nc` +or a Python interpreter for the Unix-socket case) and must not admit its +unlisted-executable control (`/usr/bin/printf`). By default each sentinel goes +into the last directory of the matching root list; `--sentinel-visible`, +`--sentinel-output` and `--sentinel-hidden` name controller-private +directories instead so the audit never writes into another arm's readable +view. The retained report replaces hidden-root paths with their digests +(`policy_id` still binds the full policy). A sentinel left behind by an audit +process that died uncleanly is reclaimed on the next run; any other occupant +refuses. Only `deny-external` (no network) is supported. The proposed +`loopback` port policy is rejected during compilation, provider admission, +audit startup, and launch verification, including for previously passing +attestations. macOS SBPL's `localhost` selector covers every address owned by +the host: a relay on `127.0.0.1:` can coexist with a separate service on +`[::1]:`. A port number therefore cannot establish exclusive relay +access. Literal IP endpoint selectors are rejected by the sandbox compiler. + +Session-bound audits remain available for `deny-external`. A pinned +`provider.relay_port` configures a listener but grants no network authority. +Relay-backed confined sessions remain unavailable until an exclusive transport +is implemented and validated. This stop condition must not be bypassed with a +broader policy or a cached attestation. The pilot still requires exact-context +admission and independent confinement evidence before collection. + Deterministic integration tests run real child processes for both arms through an explicitly synthetic pass-through adapter. Those tests cover the process boundary and failure retention. Their synthetic access logs and capability diff --git a/src/aisle/harness/matched_run.py b/src/aisle/harness/matched_run.py index 83080c35..33b1b747 100644 --- a/src/aisle/harness/matched_run.py +++ b/src/aisle/harness/matched_run.py @@ -101,12 +101,7 @@ def _worker_binding(launch, config, config_path): raise ValueError("worker runtime differs from bound run runtime") if launch["python_sha256"] != hashlib.sha256(worker_interpreter().read_bytes()).hexdigest(): raise ValueError("worker and controller interpreter identities differ") - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in launch["policy"].items() - } - ) + policy = MacOSPolicy.from_canonical(launch["policy"]) readable = (*policy.visible_roots, *policy.runtime_read_roots, *policy.output_roots) if any(config_path.is_relative_to(p) for p in readable) or not any( config_path.is_relative_to(p) for p in policy.hidden_roots diff --git a/src/aisle/harness/matched_session.py b/src/aisle/harness/matched_session.py index d0397bdf..12aef5d0 100644 --- a/src/aisle/harness/matched_session.py +++ b/src/aisle/harness/matched_session.py @@ -315,13 +315,10 @@ def _permission_ids( for arm in sorted(ARMS): other = next(name for name in ARMS if name != arm) view, other_view = Path(views[arm]).resolve(), Path(views[other]).resolve() - declared = bindings[arm]["policy"] - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in declared.items() - } - ) + try: + policy = MacOSPolicy.from_canonical(bindings[arm]["policy"]) + except ConfinementError as exc: + raise AdmissionError(f"invalid confinement policy: {exc}") from exc editable = {view / p for p in manifests[arm]["repository"]["editable_allowlist"]} if set(policy.output_roots) != editable | {scratch[arm]}: raise AdmissionError("confinement write authority differs from declared edit grants") @@ -410,12 +407,18 @@ def _verify_launches( ) if "provider" in launch: from aisle.harness.provider_relay import verify_provider + from aisle.harness.treatment_confinement import verify_relay_port try: verify_provider(launch["provider"]) ceiling = candidates[arm]["budget"].get("frontend_tool_ceiling") if type(ceiling) is not int or ceiling <= 0: raise ValueError("provider requires a shared frontend dispatch ceiling") + bindings = (execution or {}).get("confinement_bindings") or {} + verify_relay_port( + bindings.get(arm, {}).get("policy") if isinstance(bindings, dict) else None, + launch["provider"], + ) except (ValueError, TypeError) as exc: raise AdmissionError("invalid provider binding: " + str(exc)) from exc if "code_mode_host" in launch: diff --git a/src/aisle/harness/matched_tools.py b/src/aisle/harness/matched_tools.py index 8d5f50c2..9e7cfbd7 100644 --- a/src/aisle/harness/matched_tools.py +++ b/src/aisle/harness/matched_tools.py @@ -616,12 +616,8 @@ def _dispatch( raise AdmissionError( "bound typed runtime requires a validation launch binding" ) - declared = current["confinement_bindings"][self.arm]["policy"] - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in declared.items() - } + policy = MacOSPolicy.from_canonical( + current["confinement_bindings"][self.arm]["policy"] ) compiled = compile_macos_profile(policy) python_hash = hashlib.sha256(self.python.read_bytes()).hexdigest() diff --git a/src/aisle/harness/monolithic_run_prepare.py b/src/aisle/harness/monolithic_run_prepare.py index c65bb539..b38fad94 100644 --- a/src/aisle/harness/monolithic_run_prepare.py +++ b/src/aisle/harness/monolithic_run_prepare.py @@ -33,12 +33,7 @@ def prepare_monolithic_run( if launch["runtime_record"] != runtime or launch["attestation"]["adapter"]["sha256"] != adapter: raise ValueError("worker declaration differs from admitted runtime or adapter") bundle, output = Path(launch["bundle"]).absolute(), Path(output).absolute() - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in launch["policy"].items() - } - ) + policy = MacOSPolicy.from_canonical(launch["policy"]) protected = [ Path(controller_root), *(Path(p) for p in views.values()), diff --git a/src/aisle/harness/provider_relay.py b/src/aisle/harness/provider_relay.py index e9eeca53..a1ad086d 100644 --- a/src/aisle/harness/provider_relay.py +++ b/src/aisle/harness/provider_relay.py @@ -33,11 +33,17 @@ def verify_provider(binding): """Require an explicit upstream; local HTTP is for operator-owned fixtures only.""" _require( type(binding) is dict - and set(binding) - {"hosted_tool_contract"} - in ({"base_url"}, {"base_url", "requires_openai_auth"}), + and {"base_url"} <= set(binding) + and set(binding) + <= {"base_url", "requires_openai_auth", "hosted_tool_contract", "relay_port"}, "invalid provider binding", ) _require(type(binding.get("requires_openai_auth", True)) is bool, "invalid provider auth mode") + relay_port = binding.get("relay_port", 0) + _require( + type(relay_port) is int and not isinstance(relay_port, bool) and 0 <= relay_port < 65536, + "invalid relay port", + ) value = binding["base_url"] _require(type(value) is str and 0 < len(value) <= 2048, "invalid provider URL") url = urlsplit(value) @@ -86,6 +92,7 @@ class ProviderRelay: def __init__(self, binding, *, dispatch, output, timeout_s, delegated_tools=()): self.url = verify_provider(binding) self.hosted_contract = binding.get("hosted_tool_contract") + self._binding_port = binding.get("relay_port", 0) _require( type(timeout_s) in (int, float) and 0 < timeout_s < 86400, "invalid provider deadline" ) @@ -380,7 +387,9 @@ def do_POST(self): relay._handle(self) try: - self._server = Server(("127.0.0.1", 0), Handler) + # An explicit relay_port fixes the listener's address; it grants + # no confinement authority. Zero keeps the ephemeral default. + self._server = Server(("127.0.0.1", self._binding_port), Handler) self.address = self._server.server_address self._retain( "listener.json", diff --git a/src/aisle/harness/treatment_confinement.py b/src/aisle/harness/treatment_confinement.py index 064f6a6f..5e3dcb8e 100644 --- a/src/aisle/harness/treatment_confinement.py +++ b/src/aisle/harness/treatment_confinement.py @@ -9,11 +9,13 @@ from __future__ import annotations import argparse +import contextlib import hashlib import json import os import platform import shlex +import shutil import socket import subprocess import sys @@ -29,29 +31,66 @@ SANDBOX_EXEC = Path("/usr/bin/sandbox-exec") SYSTEM_PROFILE = Path("/System/Library/Sandbox/Profiles/system.sb") _HASH_LENGTH = 64 -_REQUIRED_CASE_IDS = { - "absolute_hidden_read", - "alternate_worktree_hidden_read", - "declared_output_write", - "git_object_hidden_read", - "hidden_write", - "parent_traversal_hidden_read", - "subprocess_hidden_read", - "subprocess_visible_read", - "symlink_hidden_read", - "unrestricted_alternate_worktree_baseline", - "unrestricted_git_object_baseline", - "unrestricted_hidden_baseline", - "visible_git_object_read", - "visible_read", - "unrestricted_tcp_baseline", - "tcp_read", - "unrestricted_unix_socket_baseline", - "unix_socket_network_control", - "unix_socket_read", - "unrestricted_exec_baseline", - "unlisted_executable", -} +#: Only deny-external can currently enforce the declared endpoint authority. +NETWORK_POLICIES = ("deny-external",) +_LOOPBACK_REFUSAL = ( + "loopback port policy cannot isolate the provider relay: macOS localhost " + "covers multiple local addresses; use deny-external until an exclusive " + "relay transport is implemented" +) +#: the preferred Unix-socket probe; an admitted Python interpreter is the fallback +_NETCAT = "/usr/bin/nc" +_PYTHON_SOCKET_CLIENT = ( + "import socket,sys\n" + "s=socket.socket(socket.AF_UNIX,socket.SOCK_STREAM)\n" + "s.settimeout(2)\n" + "s.connect(sys.argv[1])\n" + "sys.stdout.buffer.write(s.recv(4096))\n" +) +_SENTINEL_DIR = ".aisle-capability" +_BASE_CASE_IDS = frozenset( + { + "absolute_hidden_read", + "alternate_worktree_hidden_read", + "declared_output_write", + "git_object_hidden_read", + "hidden_write", + "parent_traversal_hidden_read", + "subprocess_hidden_read", + "subprocess_visible_read", + "symlink_hidden_read", + "unrestricted_alternate_worktree_baseline", + "unrestricted_git_object_baseline", + "unrestricted_hidden_baseline", + "visible_git_object_read", + "visible_read", + "unrestricted_tcp_baseline", + "unrestricted_unix_socket_baseline", + "unix_socket_network_control", + "unix_socket_read", + "unrestricted_exec_baseline", + "unlisted_executable", + } +) + + +def _require_network_policy(network_policy: str) -> None: + if network_policy == "loopback": + raise ConfinementError(_LOOPBACK_REFUSAL) + if network_policy not in NETWORK_POLICIES: + raise ConfinementError(f"unknown network policy: {network_policy}") + + +def required_case_ids(network_policy: str) -> frozenset[str]: + """The case matrix for a supported, enforceable network policy.""" + _require_network_policy(network_policy) + return _BASE_CASE_IDS | {"tcp_read"} + + +def verify_relay_port(policy: Any, provider: Any) -> None: + """Refuse the unsupported loopback authority at provider admission (TRT-5).""" + if type(policy) is dict and policy.get("network_policy") == "loopback": + raise ValueError(_LOOPBACK_REFUSAL) class ConfinementError(RuntimeError): @@ -68,10 +107,14 @@ class MacOSPolicy: allowed_executables: tuple[Path, ...] hidden_roots: tuple[Path, ...] network_policy: str + #: Retained for parsing old declarations; loopback authority is refused. + loopback_port: int | None = None def as_dict(self) -> dict[str, Any]: - """Return constructor-compatible fields for tests and policy transforms.""" - return { + """Return constructor-compatible fields for tests and policy transforms. + The port appears only when pinned, so every existing consumer of the + root/network fields sees exactly the shape it always did.""" + fields = { "visible_roots": self.visible_roots, "output_roots": self.output_roots, "runtime_read_roots": self.runtime_read_roots, @@ -79,9 +122,14 @@ def as_dict(self) -> dict[str, Any]: "hidden_roots": self.hidden_roots, "network_policy": self.network_policy, } + if self.loopback_port is not None: + fields["loopback_port"] = self.loopback_port + return fields def canonical_dict(self) -> dict[str, Any]: - return { + """The identity form; the port appears only when a policy pins one, so a + deny-external policy keeps the identity it always had.""" + canonical = { "allowed_executables": [str(path) for path in self.allowed_executables], "hidden_roots": [str(path) for path in self.hidden_roots], "network_policy": self.network_policy, @@ -89,6 +137,44 @@ def canonical_dict(self) -> dict[str, Any]: "runtime_read_roots": [str(path) for path in self.runtime_read_roots], "visible_roots": [str(path) for path in self.visible_roots], } + if self.loopback_port is not None: + canonical["loopback_port"] = self.loopback_port + return canonical + + @classmethod + def from_canonical(cls, declared: Any) -> MacOSPolicy: + """Rebuild a policy from its canonical form; anything mis-shaped refuses.""" + fields = set(_CANONICAL_ROOT_FIELDS) | {"network_policy"} + if ( + type(declared) is not dict + or not fields <= set(declared) + or not set(declared) <= fields | {"loopback_port"} + ): + raise ConfinementError("policy must carry exactly the MacOSPolicy fields") + port = declared.get("loopback_port") + if ( + type(declared["network_policy"]) is not str + or (port is not None and (type(port) is not int or isinstance(port, bool))) + or any( + not isinstance(declared[k], list) or any(type(p) is not str for p in declared[k]) + for k in _CANONICAL_ROOT_FIELDS + ) + ): + raise ConfinementError("policy fields must be lists of paths and a network policy") + return cls( + **{key: tuple(Path(p) for p in declared[key]) for key in _CANONICAL_ROOT_FIELDS}, + network_policy=declared["network_policy"], + loopback_port=port, + ) + + +_CANONICAL_ROOT_FIELDS = ( + "visible_roots", + "output_roots", + "runtime_read_roots", + "allowed_executables", + "hidden_roots", +) @dataclass(frozen=True) @@ -96,6 +182,7 @@ class CompiledProfile: text: str sha256: str policy_id: str + network_policy: str def _canonical_bytes(value: Any) -> bytes: @@ -122,6 +209,10 @@ def _validate_roots(name: str, roots: tuple[Path, ...]) -> None: for path in roots: if not path.is_absolute(): raise ConfinementError(f"{name} contains a relative path: {path}") + if not str(path).isascii() or not str(path).isprintable(): + # SBPL literals are JSON-escaped; a non-ASCII name would compile + # to a different string than the directory it must guard. + raise ConfinementError(f"{name} contains a non-ASCII or unprintable path: {path}") if path == Path("/"): raise ConfinementError(f"{name} cannot authorize the filesystem root") if not path.exists(): @@ -139,8 +230,9 @@ def _contains(parent: Path, child: Path) -> bool: def _validate_policy(policy: MacOSPolicy) -> None: - if policy.network_policy != "deny-external": - raise ConfinementError("macOS capability audit requires deny-external network policy") + _require_network_policy(policy.network_policy) + if policy.loopback_port is not None: + raise ConfinementError("deny-external network policy cannot carry a relay port") for name in ( "visible_roots", "output_roots", @@ -189,12 +281,14 @@ def compile_macos_profile(policy: MacOSPolicy) -> CompiledProfile: ) lines.extend([")", "(allow file-write*"]) lines.extend(f" (subpath {_sbpl_path(path)})" for path in policy.output_roots) - lines.extend([")", "(allow signal (target self))", ""]) + lines.extend([")", "(allow signal (target self))"]) + lines.append("") text = "\n".join(lines) return CompiledProfile( text=text, sha256=_sha256_bytes(text.encode()), policy_id=_sha256_bytes(_canonical_bytes(policy.canonical_dict())), + network_policy=policy.network_policy, ) @@ -336,6 +430,17 @@ def _platform_record() -> dict[str, str]: } +def _serve_once(listener: socket.socket, payload: bytes, stopped: threading.Event) -> None: + while not stopped.is_set(): + try: + connection, _ = listener.accept() + except TimeoutError: + continue + with connection: + connection.settimeout(1) + connection.sendall(payload) + + def _socket_capability_cases(profile_path: Path, cwd: Path, sentinel: bytes) -> list[dict]: """Exercise the same loopback read outside and inside the external profile.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: @@ -375,33 +480,35 @@ def serve(): ] -def _unix_socket_capability_cases(profile_path: Path, cwd: Path, sentinel: bytes) -> list[dict]: - """Distinguish local socket denial from executable or filesystem failure.""" - endpoint = cwd / "fixture.sock" +def _unix_socket_capability_cases( + profile_path: Path, cwd: Path, sentinel: bytes, client: list[str] +) -> list[dict]: + """Distinguish local socket denial from executable or filesystem failure. + + `client` is the admitted argv prefix that connects to a Unix socket path + and prints what it reads (nc, or an admitted Python interpreter).""" + # A session root may exceed macOS sockaddr_un.sun_path; the endpoint lives + # in a short controller-private directory outside every agent-readable root, + # which changes nothing about the deny (network) or control (allow) cases. control_path = profile_path.with_name("unix-network-control.sb") control_text = profile_path.read_text() + ( "\n(allow network* (local unix-socket) (remote unix-socket))\n" ) control_path.write_text(control_text) - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: + with ( + tempfile.TemporaryDirectory(prefix="aisle-sock-", dir="/private/tmp") as socket_root, + socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener, + ): + endpoint = Path(socket_root) / "fixture.sock" listener.bind(str(endpoint)) listener.listen() listener.settimeout(0.1) stopped = threading.Event() - - def serve(): - while not stopped.is_set(): - try: - connection, _ = listener.accept() - except TimeoutError: - continue - with connection: - connection.settimeout(1) - connection.sendall(sentinel) - - server = threading.Thread(target=serve, daemon=True) + server = threading.Thread( + target=_serve_once, args=(listener, sentinel, stopped), daemon=True + ) server.start() - command = ["/usr/bin/nc", "-w", "2", "-U", str(endpoint)] + command = [*client, str(endpoint)] try: baseline = _run(command, cwd=cwd) confined = _run(_wrapped(profile_path, command), cwd=cwd) @@ -425,8 +532,235 @@ def serve(): ] -def run_macos_capability_audit() -> dict[str, Any]: - """Run the synthetic deny/allow matrix through the external adapter.""" +def _synthetic_policy( + visible: Path, output: Path, hidden: Path, git: Path, developer_root: Path +) -> MacOSPolicy: + """The fixed policy of the synthetic (session-less) audit.""" + return MacOSPolicy( + visible_roots=(visible,), + output_roots=(output,), + runtime_read_roots=( + Path("/Library/Apple").resolve(), + Path("/System").resolve(), + Path("/bin").resolve(), + Path("/usr/bin").resolve(), + Path("/usr/lib").resolve(), + Path("/usr/share").resolve(), + developer_root, + ), + allowed_executables=( + Path("/bin/bash").resolve(), + Path("/bin/cat").resolve(), + Path("/usr/bin/nc").resolve(), + git, + ), + hidden_roots=(hidden,), + network_policy="deny-external", + ) + + +def _socket_client(policy: MacOSPolicy) -> list[str]: + """The Unix-socket probe the policy already admits: `nc` when present, + otherwise an admitted Python interpreter. The audit never widens the arm's + executable grants for its own probes.""" + netcat = Path(_NETCAT).resolve() + if netcat in policy.allowed_executables: + return [str(netcat), "-w", "2", "-U"] + for path in policy.allowed_executables: + if path.name.startswith("python"): + return [str(path), "-I", "-c", _PYTHON_SOCKET_CLIENT] + raise ConfinementError( + "session policy must admit a Unix-socket probe (/usr/bin/nc or a python interpreter)" + ) + + +def _check_session_policy(policy: MacOSPolicy, git: Path) -> list[str]: + """A session policy can only be attested if the audit's own probes are + admitted executables and its unlisted-executable control is not; returns + the admitted Unix-socket probe argv.""" + _validate_policy(policy) + probes = {Path(p).resolve() for p in ("/bin/bash", "/bin/cat")} | {git} + missing = sorted(str(p) for p in probes - set(policy.allowed_executables)) + if missing: + raise ConfinementError("session policy must admit the audit probes: " + ", ".join(missing)) + control = Path("/usr/bin/printf").resolve() + if control in policy.allowed_executables: + raise ConfinementError(f"session policy admits the unlisted-executable control {control}") + return _socket_client(policy) + + +def _last_directory(name: str, roots: tuple[Path, ...]) -> Path: + for root in reversed(roots): + if root.is_dir() and not root.is_symlink(): + return root + raise ConfinementError(f"{name} holds no directory for a sentinel") + + +def _sentinel_homes( + policy: MacOSPolicy, requested: tuple[Path, Path, Path] | None +) -> tuple[Path, Path, Path]: + """Where the three sentinels go: the caller's directories (each inside a + root of the matching kind) or, by default, the last directory of each root + list. A live session's caller names controller-private homes so the audit + never writes into another arm's readable view.""" + kinds = ( + ("visible_roots", policy.visible_roots), + ("output_roots", policy.output_roots), + ("hidden_roots", policy.hidden_roots), + ) + if requested is None: + return tuple(_last_directory(name, roots) for name, roots in kinds) + homes = [] + for (name, roots), home in zip(kinds, requested, strict=True): + home = Path(home) + if ( + not home.is_absolute() + or home.resolve() != home + or not home.is_dir() + or not any(_contains(root, home) for root in roots if root.is_dir()) + ): + raise ConfinementError(f"sentinel home must be a canonical directory inside {name}") + homes.append(home) + return tuple(homes) + + +def _place_sentinels( + policy: MacOSPolicy, + cleanup: contextlib.ExitStack, + homes: tuple[Path, Path, Path] | None = None, +) -> tuple[Path, ...]: + """Fresh sentinel directories inside the session's own visible, output and + hidden roots. Each directory is registered for removal the moment it + exists, so a later failure never leaves one behind; anything already at a + sentinel path (file, directory or symlink) refuses.""" + homes = _sentinel_homes(policy, homes) + sentinels = [] + for home in homes: + path = home / _SENTINEL_DIR + if path.is_dir() and not path.is_symlink() and _reclaimable(path): + # residue of an audit that died uncleanly (SIGKILL): reclaim it + _remove_sentinel(path) + try: + path.mkdir() + except FileExistsError as exc: + raise ConfinementError(f"sentinel path already exists: {path}") from exc + except OSError as exc: + raise ConfinementError(f"sentinel path could not be created: {path}: {exc}") from exc + cleanup.callback(_remove_sentinel, path) + sentinels.append(path) + _owner_marker(path).write_text( + json.dumps({"schema_version": "aisle.sentinel-owner.v1", "pid": os.getpid()}) + ) + return tuple(sentinels) + + +def _redacted_policy(policy: MacOSPolicy) -> dict[str, Any]: + """The attested policy with hidden roots replaced by their digests: the + report is retained beside session evidence and must not name the private + roots it protects. `policy_id` still binds the full canonical form.""" + canonical = policy.canonical_dict() + canonical["hidden_roots"] = [ + "sha256:" + _sha256_bytes(path.encode()) for path in canonical["hidden_roots"] + ] + canonical["hidden_roots_redacted"] = True + return canonical + + +def _owner_marker(path: Path) -> Path: + return path / ".owner.json" + + +def _reclaimable(path: Path) -> bool: + """A leftover sentinel from an audit process that no longer exists (a + SIGKILL mid-run) may be reclaimed; anything else refuses.""" + marker_path = _owner_marker(path) + if marker_path.is_symlink() or not marker_path.is_file(): + return False + try: + marker = json.loads(marker_path.read_bytes()) + pid = marker["pid"] + except (OSError, ValueError, KeyError, TypeError): + return False # a corrupt or unreadable marker never authorizes removal + if ( + type(pid) is not int + or isinstance(pid, bool) + or marker.get("schema_version") != "aisle.sentinel-owner.v1" + ): + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return True # the owner is gone + except OSError: + return False # alive but not ours (EPERM) or undecidable + return False + + +def _remove_sentinel(path: Path) -> None: + failures: list[str] = [] + + def record(_function, failed_path, exc_info): + failures.append(f"{failed_path}: {exc_info[1]}") + + # onerror also works on the project's supported Python 3.11 runtime. + shutil.rmtree(path, onerror=record) + if path.exists() or path.is_symlink(): + detail = f" ({failures[0]})" if failures else "" + raise ConfinementError(f"sentinel path could not be removed: {path}{detail}") + + +def _fixture_roots( + policy: MacOSPolicy | None, + sentinel_homes: tuple[Path, Path, Path] | None, + root: Path, + git: Path, + developer_root: Path, + cleanup: contextlib.ExitStack, +) -> tuple[MacOSPolicy, str, list[str], Path, Path, Path]: + """The audited policy and the three fixture directories: a synthetic policy + over fresh temporary roots, or the session policy with sentinels placed in + its own roots and registered for removal.""" + if policy is None: + visible, output, hidden = root / "visible", root / "output", root / "hidden" + for path in (visible, output, hidden): + path.mkdir() + policy = _synthetic_policy(visible, output, hidden, git, developer_root) + return policy, "synthetic", _socket_client(policy), visible, output, hidden + client = _check_session_policy(policy, git) + visible, output, hidden = _place_sentinels(policy, cleanup, sentinel_homes) + return policy, "session", client, visible, output, hidden + + +def _place_hidden_link( + visible: Path, hidden: Path, root: Path, source: str, hidden_sentinel: bytes +) -> None: + """The symlink case proves following a link out of the visible root is + refused. Under a session policy the link targets a hidden directory of the + controller's private temporary tree, never the session's own hidden root: a + link is readable text, and its target would otherwise disclose that root's + path to the arm (the arm's real hidden root is still covered by the + absolute and parent-traversal cases).""" + link_target = hidden if source == "synthetic" else root / "hidden-target" + if source == "session": + link_target.mkdir() + (link_target / "secret.txt").write_bytes(hidden_sentinel) + (visible / "hidden-link").symlink_to(link_target, target_is_directory=True) + + +def run_macos_capability_audit( + policy: MacOSPolicy | None = None, + *, + sentinel_homes: tuple[Path, Path, Path] | None = None, +) -> dict[str, Any]: + """Run the synthetic deny/allow matrix through the external adapter. + + Without a policy the audit runs its fixed synthetic policy in a private + temporary tree. With a session policy it runs the same matrix under that + exact policy, placing its sentinels inside the session's own roots (the + caller's `sentinel_homes`, or the last directory of each root list) and + removing them afterwards, so the retained attestation carries the profile + hash and policy id the launch wrapper will demand (TRT-5/TRT-7). + """ if sys.platform != "darwin": raise ConfinementError( "macOS confinement capability requires macOS; no simulation accepted" @@ -436,25 +770,32 @@ def run_macos_capability_audit() -> dict[str, Any]: if not SYSTEM_PROFILE.is_file(): raise ConfinementError(f"required imported profile is unavailable: {SYSTEM_PROFILE}") + # Cheap shape checks first: no subprocess is spawned for a malformed call. + if policy is None and sentinel_homes is not None: + raise ConfinementError("sentinel homes require a session policy") + if policy is not None: + _validate_policy(policy) + # Keep the Unix socket pathname below macOS sockaddr_un.sun_path limits. - with tempfile.TemporaryDirectory( - prefix="aisle-confinement-capability-", dir="/private/tmp" - ) as temporary: + with ( + tempfile.TemporaryDirectory( + prefix="aisle-confinement-capability-", dir="/private/tmp" + ) as temporary, + contextlib.ExitStack() as cleanup, + ): root = Path(temporary).resolve() - visible = root / "visible" - output = root / "output" - hidden = root / "hidden" - for path in (visible, output, hidden): - path.mkdir() + git, developer_root = _apple_git_runtime(cwd=root) + policy, source, client, visible, output, hidden = _fixture_roots( + policy, sentinel_homes, root, git, developer_root, cleanup + ) visible_sentinel = b"VISIBLE-SYNTHETIC-CAPABILITY\n" hidden_sentinel = b"HIDDEN-SYNTHETIC-CAPABILITY-7E4E\n" visible_file = visible / "allowed.txt" hidden_file = hidden / "secret.txt" visible_file.write_bytes(visible_sentinel) hidden_file.write_bytes(hidden_sentinel) - (visible / "hidden-link").symlink_to(hidden, target_is_directory=True) + _place_hidden_link(visible, hidden, root, source, hidden_sentinel) - git, developer_root = _apple_git_runtime(cwd=root) isolated_home = visible / "isolated-home" isolated_home.mkdir() git_environment = { @@ -494,27 +835,6 @@ def run_macos_capability_audit() -> dict[str, Any]: env=git_environment, ) - policy = MacOSPolicy( - visible_roots=(visible,), - output_roots=(output,), - runtime_read_roots=( - Path("/Library/Apple").resolve(), - Path("/System").resolve(), - Path("/bin").resolve(), - Path("/usr/bin").resolve(), - Path("/usr/lib").resolve(), - Path("/usr/share").resolve(), - developer_root, - ), - allowed_executables=( - Path("/bin/bash").resolve(), - Path("/bin/cat").resolve(), - Path("/usr/bin/nc").resolve(), - git, - ), - hidden_roots=(hidden,), - network_policy="deny-external", - ) compiled = compile_macos_profile(policy) profile_path = root / "controller-profile.sb" profile_path.write_text(compiled.text, encoding="utf-8") @@ -613,7 +933,7 @@ def run_macos_capability_audit() -> dict[str, Any]: denied_reads = { "absolute_hidden_read": str(hidden_file), - "parent_traversal_hidden_read": str(visible / ".." / "hidden" / "secret.txt"), + "parent_traversal_hidden_read": os.path.relpath(hidden_file, visible), "symlink_hidden_read": str(visible / "hidden-link" / "secret.txt"), } for case_id, path in denied_reads.items(): @@ -711,7 +1031,7 @@ def run_macos_capability_audit() -> dict[str, Any]: ) cases.extend(_socket_capability_cases(profile_path, visible, hidden_sentinel)) - cases.extend(_unix_socket_capability_cases(profile_path, visible, hidden_sentinel)) + cases.extend(_unix_socket_capability_cases(profile_path, visible, hidden_sentinel, client)) executable_command = ["/usr/bin/printf", "%s", hidden_sentinel.decode()] cases.append( _case_result( @@ -762,7 +1082,8 @@ def run_macos_capability_audit() -> dict[str, Any]: "Apple system.sb is a private interface and is hashed per audit", ], "platform": _platform_record(), - "policy": policy.canonical_dict(), + "policy": _redacted_policy(policy), + "policy_source": source, "ephemeral_profile_path": str(profile_path), "recorded_at": recorded_at.isoformat(), "schema_version": SCHEMA_VERSION, @@ -781,12 +1102,17 @@ def run_macos_capability_audit() -> dict[str, Any]: } -def write_macos_capability_audit(output: Path) -> dict[str, Any]: +def write_macos_capability_audit( + output: Path, + policy: MacOSPolicy | None = None, + *, + sentinel_homes: tuple[Path, Path, Path] | None = None, +) -> dict[str, Any]: """Retain one unscored audit without overwriting earlier evidence.""" output = Path(output) if output.exists(): raise ConfinementError(f"capability audit already exists: {output}") - report = run_macos_capability_audit() + report = run_macos_capability_audit(policy, sentinel_homes=sentinel_homes) output.parent.mkdir(parents=True, exist_ok=True) rendered = json.dumps(report, indent=2, sort_keys=True) + "\n" try: @@ -806,6 +1132,7 @@ def wrap_verified_command( purpose: str = "unscored_capability", ) -> list[str]: """Bind an unscored command to the exact externally verified adapter.""" + _require_network_policy(compiled.network_policy) if purpose != "unscored_capability": raise ConfinementError( "this macOS capability attestation cannot authorize confirmatory work" @@ -827,11 +1154,17 @@ def wrap_verified_command( raise ConfinementError("adapter profile hash does not match compiled profile") if adapter.get("policy_id") != compiled.policy_id: raise ConfinementError("adapter policy id does not match compiled policy") + attested_policy = attestation.get("policy") + if ( + not isinstance(attested_policy, dict) + or attested_policy.get("network_policy") != compiled.network_policy + ): + raise ConfinementError("adapter attestation network policy does not match compiled profile") cases = attestation.get("cases") if ( not isinstance(cases, list) or any(not isinstance(row, dict) for row in cases) - or {row.get("id") for row in cases} != _REQUIRED_CASE_IDS + or {row.get("id") for row in cases} != required_case_ids(compiled.network_policy) or any(not row.get("passed") for row in cases) ): raise ConfinementError("adapter attestation contains a failed case") @@ -855,14 +1188,48 @@ def _parser() -> argparse.ArgumentParser: commands = parser.add_subparsers(dest="command", required=True) audit = commands.add_parser("audit-macos") audit.add_argument("--output", type=Path, required=True) + audit.add_argument( + "--policy", + type=Path, + default=None, + help="session policy (MacOSPolicy.canonical_dict JSON) to attest under", + ) + for role in ("visible", "output", "hidden"): + audit.add_argument( + f"--sentinel-{role}", + type=Path, + default=None, + help=f"directory inside the policy's {role} roots that receives that sentinel", + ) return parser +def load_policy(path: Path) -> MacOSPolicy: + """A MacOSPolicy from its canonical JSON form (lists of absolute paths).""" + try: + declared = json.loads(Path(path).read_bytes()) + except (OSError, json.JSONDecodeError) as exc: + raise ConfinementError(f"policy file unreadable: {exc}") from exc + return MacOSPolicy.from_canonical(declared) + + +def _sentinel_homes_from_args(args) -> tuple[Path, Path, Path] | None: + homes = (args.sentinel_visible, args.sentinel_output, args.sentinel_hidden) + if all(home is None for home in homes): + return None + if any(home is None for home in homes): + raise ConfinementError("sentinel homes must be given for visible, output and hidden") + return tuple(Path(home) for home in homes) + + def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) try: - report = write_macos_capability_audit(args.output) - except ConfinementError as exc: + policy = load_policy(args.policy) if args.policy is not None else None + report = write_macos_capability_audit( + args.output, policy, sentinel_homes=_sentinel_homes_from_args(args) + ) + except (ConfinementError, OSError) as exc: print(json.dumps({"error": str(exc), "ok": False}, sort_keys=True), file=sys.stderr) return 2 if not report["capability_pass"]: diff --git a/src/aisle/harness/typed_graph_audit.py b/src/aisle/harness/typed_graph_audit.py index 01effbf3..802d190b 100644 --- a/src/aisle/harness/typed_graph_audit.py +++ b/src/aisle/harness/typed_graph_audit.py @@ -36,12 +36,7 @@ def _worker(output, config, digest): if terminal.get("classification") != "module_result" or terminal.get("stage") != "done": raise StageError("worker has no completed module result") launch = dict(config["launch"]) - launch["policy"] = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in launch["policy"].items() - } - ) + launch["policy"] = MacOSPolicy.from_canonical(launch["policy"]) compiled = verify_typed_launch( **{ key: launch[key] diff --git a/src/aisle/harness/typed_node_host.py b/src/aisle/harness/typed_node_host.py index b93604d5..069b69c8 100644 --- a/src/aisle/harness/typed_node_host.py +++ b/src/aisle/harness/typed_node_host.py @@ -137,12 +137,7 @@ def preflight_host_config(path, digest): config = load_host_config(path, digest) try: launch = dict(config["launch"]) - launch["policy"] = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in launch["policy"].items() - } - ) + launch["policy"] = MacOSPolicy.from_canonical(launch["policy"]) compiled = verify_typed_launch( **{ key: launch[key] diff --git a/src/aisle/harness/typed_validation.py b/src/aisle/harness/typed_validation.py index 4ed2dcc4..4c1fb150 100644 --- a/src/aisle/harness/typed_validation.py +++ b/src/aisle/harness/typed_validation.py @@ -449,12 +449,7 @@ def _verify_validation_binding_fields(binding, runtime_record, source_roots, par raise ValidationError("validation binding fields are unresolved") if binding["schema_version"] != "aisle.typed-validation-binding.v1": raise ValidationError("validation binding schema is unsupported") - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in binding["policy"].items() - } - ) + policy = MacOSPolicy.from_canonical(binding["policy"]) kwargs = {key: value for key, value in binding.items() if key != "schema_version"} kwargs["policy"] = policy kwargs["runtime_record"] = runtime_record diff --git a/src/aisle/harness/worker_capability.py b/src/aisle/harness/worker_capability.py index a4189d30..c6cb9070 100644 --- a/src/aisle/harness/worker_capability.py +++ b/src/aisle/harness/worker_capability.py @@ -15,13 +15,13 @@ from aisle.harness.treatment_ambient import verify_declared_environment from aisle.harness.treatment_confinement import ( - _REQUIRED_CASE_IDS, EVIDENCE_CLASS, SANDBOX_EXEC, SCHEMA_VERSION, SYSTEM_PROFILE, _apple_git_runtime, compile_macos_profile, + required_case_ids, ) from aisle.harness.typed_snapshot import _read from aisle.harness.worker_authority_probe import operation_case, operation_command @@ -278,8 +278,8 @@ def observe(name, operation, target, sentinel, *, confined, allowed, control=Fal report["capability_pass"] = ( report["error"] is None and not report["cleanup_errors"] - and len(report["cases"]) == len(_REQUIRED_CASE_IDS) - and {row["id"] for row in report["cases"]} == _REQUIRED_CASE_IDS + and len(report["cases"]) == len(required_case_ids("deny-external")) + and {row["id"] for row in report["cases"]} == required_case_ids("deny-external") and all(row["passed"] for row in (*report["cases"], *report["controls"])) ) (output / "report.json").write_text(json.dumps(report, allow_nan=False)) diff --git a/src/aisle/monolith/worker_config.py b/src/aisle/monolith/worker_config.py index 4f42e81c..044810ba 100644 --- a/src/aisle/monolith/worker_config.py +++ b/src/aisle/monolith/worker_config.py @@ -8,7 +8,7 @@ from contextlib import contextmanager from pathlib import Path -from aisle.harness.treatment_confinement import MacOSPolicy +from aisle.harness.treatment_confinement import ConfinementError, MacOSPolicy from aisle.monolith.supervisor import WorkerFailure from aisle.monolith.worker_launch import launch_worker @@ -119,13 +119,8 @@ def factory(primitives): raise WorkerFailure("worker embodiment differs from configuration") launch = dict(config["launch"]) try: - launch["policy"] = MacOSPolicy( - **{ - name: value if name == "network_policy" else tuple(Path(p) for p in value) - for name, value in launch["policy"].items() - } - ) - except (AttributeError, TypeError, ValueError) as exc: + launch["policy"] = MacOSPolicy.from_canonical(launch["policy"]) + except (AttributeError, TypeError, ValueError, ConfinementError) as exc: raise WorkerFailure("invalid worker policy fields") from exc with launch_worker( **launch, output=Path(config["output_root"]) / phase, primitives=primitives diff --git a/tests/accept/test_provider_relay_transport.py b/tests/accept/test_provider_relay_transport.py index d539b58a..9a19d76c 100644 --- a/tests/accept/test_provider_relay_transport.py +++ b/tests/accept/test_provider_relay_transport.py @@ -471,3 +471,30 @@ def do_POST(self): request = json.loads((root / "00000001-request.json").read_bytes()) assert request["authorization_present"] is True assert request["account_header_present"] is True + + +def test_relay_binds_the_pinned_port_and_refuses_a_bad_one(tmp_path): + """MON-8/TRT-7: `relay_port` is the one loopback port a confined frontend may + reach, so the relay must bind exactly it; a malformed value refuses.""" + import socket + + from aisle.harness.provider_relay import verify_provider + + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + pinned = probe.getsockname()[1] + with DispatchBudget(tmp_path / "budget", session_id="session", ceiling=1) as budget: + with ProviderRelay( + {"base_url": "http://127.0.0.1:1/v1", "relay_port": pinned}, + dispatch=budget, + output=tmp_path / "provider", + timeout_s=5, + ) as relay: + assert relay.address == ("127.0.0.1", pinned) + listener = json.loads((tmp_path / "provider" / "listener.json").read_text()) + assert listener["port"] == pinned + for bad in (-1, 65536, True, "4321"): + with pytest.raises(ValueError, match="relay port"): + verify_provider({"base_url": "http://127.0.0.1:1/v1", "relay_port": bad}) + with pytest.raises(ValueError, match="invalid provider binding"): + verify_provider({"base_url": "http://127.0.0.1:1/v1", "port": 1}) diff --git a/tests/unit/test_freeze_registry.py b/tests/unit/test_freeze_registry.py index 674ab245..261aea26 100644 --- a/tests/unit/test_freeze_registry.py +++ b/tests/unit/test_freeze_registry.py @@ -409,7 +409,7 @@ def test_committed_registrations_check_clean_with_withheld_seeds(): registration names it in `superseded`; drift with no successor is the refusal the registry promises (analysis/freeze/README.md).""" manifests = _committed_manifests() - assert len(manifests) == 59 + assert len(manifests) == 61 superseded_ids: set[str] = set() for path in manifests: declaration = json.loads(path.with_name("declaration.json").read_text()) diff --git a/tests/unit/test_treatment_confinement.py b/tests/unit/test_treatment_confinement.py index 81c2d570..de02fcf7 100644 --- a/tests/unit/test_treatment_confinement.py +++ b/tests/unit/test_treatment_confinement.py @@ -4,6 +4,8 @@ import hashlib import json +import os +import shutil import subprocess import sys from pathlib import Path @@ -119,6 +121,7 @@ def _attestation(compiled, profile_path: Path, adapter_path: Path) -> dict: ).hexdigest(), }, "profile_path": str(profile_path), + "policy": {"network_policy": compiled.network_policy}, "cases": [ {"id": case_id, "passed": True} for case_id in ( @@ -260,6 +263,10 @@ def test_live_capability_denies_hidden_path_variants_and_retains_no_hidden_bytes assert report["session_id"].startswith("macos-capability-") assert report["recorded_at"].endswith("+00:00") assert report["policy"]["network_policy"] == "deny-external" + assert report["policy_source"] == "synthetic" + from aisle.harness.treatment_confinement import required_case_ids + + assert set(cases) == required_case_ids("deny-external") assert report["adapter"]["imported_system_profile_sha256"] assert "HIDDEN-SYNTHETIC" not in json.dumps(report) @@ -334,3 +341,474 @@ def test_live_capability_blocks_unix_socket_with_matching_network_control(): assert denial["passed"] and denial["denied"] assert not denial["sentinel_exposed"] assert report["confirmatory_ready"] is False + + +# --- session-bound attestation and unsupported relay authority --- + + +def _session_policy(tmp_path: Path) -> MacOSPolicy: + """A policy shaped like a real arm's: the audit's probes must be admitted + executables and the Apple developer tree a runtime read root.""" + from aisle.harness.treatment_confinement import _apple_git_runtime + + visible = tmp_path / "view" + output = tmp_path / "scratch" + hidden = tmp_path / "controller" + for path in (visible, output, hidden): + path.mkdir() + git, developer_root = _apple_git_runtime(cwd=tmp_path) + runtime_roots = tuple( + dict.fromkeys( + Path(p).resolve() + for p in ("/Library/Apple", "/System", "/bin", "/usr/bin", "/usr/lib", "/usr/share") + ) + ) + (developer_root,) + executables = tuple( + dict.fromkeys(Path(p).resolve() for p in ("/bin/bash", "/bin/cat", "/usr/bin/nc")) + ) + (git,) + return MacOSPolicy( + visible_roots=(visible.resolve(),), + output_roots=(output.resolve(),), + runtime_read_roots=runtime_roots, + allowed_executables=executables, + hidden_roots=(hidden.resolve(),), + network_policy="deny-external", + ) + + +def test_old_loopback_policy_parses_but_is_not_compilable(tmp_path: Path): + """TRT-5: old policy declarations remain readable but cannot grant broad endpoints.""" + from aisle.harness.treatment_confinement import NETWORK_POLICIES + + assert NETWORK_POLICIES == ("deny-external",) + base = _policy(tmp_path) + assert "loopback_port" not in base.canonical_dict() + assert MacOSPolicy.from_canonical(base.canonical_dict()) == base + pinned = MacOSPolicy(**{**base.as_dict(), "network_policy": "loopback", "loopback_port": 4321}) + assert MacOSPolicy.from_canonical(pinned.canonical_dict()) == pinned + with pytest.raises(ConfinementError, match="cannot isolate the provider relay"): + compile_macos_profile(pinned) + with pytest.raises(ConfinementError, match="cannot carry a relay port"): + compile_macos_profile(MacOSPolicy(**{**base.as_dict(), "loopback_port": 4321})) + + +def test_provider_admission_refuses_loopback_even_when_ports_match(): + """TRT-5/MON-8: matching port numbers cannot establish exclusive relay authority.""" + from aisle.harness.treatment_confinement import verify_relay_port + + verify_relay_port({"network_policy": "deny-external"}, {"base_url": "https://x"}) + verify_relay_port(None, {"base_url": "https://x"}) + for port in (None, 4321, 4322): + with pytest.raises(ValueError, match="cannot isolate the provider relay"): + verify_relay_port( + {"network_policy": "loopback", "loopback_port": 4321}, + {"base_url": "https://x", "relay_port": port}, + ) + + +def test_roots_must_be_printable_ascii(tmp_path: Path): + """TRT-5: a root the SBPL literal could not name exactly is refused.""" + odd = tmp_path / "visé" + odd.mkdir() + with pytest.raises(ConfinementError, match="non-ASCII"): + compile_macos_profile( + MacOSPolicy(**{**_policy(tmp_path).as_dict(), "visible_roots": (odd,)}) + ) + + +def test_required_case_ids_refuse_unsupported_network_authority(): + """TRT-6: unsupported loopback authority has no passing audit case matrix.""" + from aisle.harness.treatment_confinement import required_case_ids + + assert "tcp_read" in required_case_ids("deny-external") + for network in ("loopback", "unrestricted"): + with pytest.raises(ConfinementError): + required_case_ids(network) + + +def test_launch_wrapper_requires_matching_attested_network_policy(tmp_path: Path): + """TRT-5: an attestation must name the same supported policy as the profile.""" + compiled = compile_macos_profile(_policy(tmp_path)) + profile = tmp_path / "profile.sb" + profile.write_text(compiled.text) + report = _attestation(compiled, profile, _fake_adapter(tmp_path)) + for broken in ({"network_policy": "loopback"}, [], None): + report["policy"] = broken + with pytest.raises(ConfinementError, match="network policy"): + wrap_verified_command(["/bin/cat", "x"], compiled, profile, report) + + +def test_sentinel_placement_is_transactional_and_refuses_occupied_paths(tmp_path: Path): + """TRT-6: a sentinel that cannot be created removes the ones already placed, + an occupied sentinel path refuses, and an unremovable sentinel is an error.""" + import contextlib + + from aisle.harness import treatment_confinement as tc + + policy = _policy(tmp_path) + visible, output, hidden = ( + policy.visible_roots[0], + policy.output_roots[0], + policy.hidden_roots[0], + ) + hidden.chmod(0o500) + try: + with pytest.raises(ConfinementError, match="could not be created"): + with contextlib.ExitStack() as cleanup: + tc._place_sentinels(policy, cleanup) + finally: + hidden.chmod(0o700) + assert not (visible / ".aisle-capability").exists() + assert not (output / ".aisle-capability").exists() + (output / ".aisle-capability").write_text("occupied") + with pytest.raises(ConfinementError, match="already exists"): + with contextlib.ExitStack() as cleanup: + tc._place_sentinels(policy, cleanup) + assert not (visible / ".aisle-capability").exists() + (output / ".aisle-capability").unlink() + with contextlib.ExitStack() as cleanup: + placed = tc._place_sentinels(policy, cleanup) + assert placed == tuple(root / ".aisle-capability" for root in (visible, output, hidden)) + assert all(path.is_dir() for path in placed) + assert not any(path.exists() for path in placed) + # residue of a dead audit process is reclaimed; a live owner's is not + dead = visible / ".aisle-capability" + dead.mkdir() + (dead / "leftover.txt").write_text("x") + (dead / ".owner.json").write_text( + json.dumps({"schema_version": "aisle.sentinel-owner.v1", "pid": 2**22 + 12345}) + ) + with contextlib.ExitStack() as cleanup: + placed = tc._place_sentinels(policy, cleanup) + assert not (dead / "leftover.txt").exists() + assert json.loads((dead / ".owner.json").read_text())["pid"] == os.getpid() + assert not dead.exists() + dead.mkdir() + (dead / ".owner.json").write_text( + json.dumps({"schema_version": "aisle.sentinel-owner.v1", "pid": os.getpid()}) + ) + with pytest.raises(ConfinementError, match="already exists"): + with contextlib.ExitStack() as cleanup: + tc._place_sentinels(policy, cleanup) + shutil.rmtree(dead) + stuck = tmp_path / ".aisle-capability" + stuck.mkdir() + with pytest.MonkeyPatch.context() as patch: + patch.setattr(tc.shutil, "rmtree", lambda *args, **kwargs: None) + with pytest.raises(ConfinementError, match="could not be removed"): + tc._remove_sentinel(stuck) + + +def test_last_directory_prefers_the_last_directory_and_refuses_file_only_roots(tmp_path: Path): + """TRT-6: sentinels go into the last directory root; file-only roots refuse.""" + from aisle.harness.treatment_confinement import _last_directory + + first, second = tmp_path / "a", tmp_path / "b" + first.mkdir() + second.mkdir() + editable = tmp_path / "graph.yaml" + editable.write_text("nodes: []\n") + assert _last_directory("output_roots", (first, second, editable)) == second + with pytest.raises(ConfinementError, match="holds no directory"): + _last_directory("output_roots", (editable,)) + + +def _policy_json(tmp_path: Path) -> dict: + return _policy(tmp_path).canonical_dict() + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda body: "{not json", "unreadable"), + (lambda body: "[]", "exactly the MacOSPolicy fields"), + (lambda body: json.dumps({"visible_roots": []}), "exactly the MacOSPolicy fields"), + ( + lambda body: json.dumps({**body, "extra": 1}), + "exactly the MacOSPolicy fields", + ), + (lambda body: json.dumps({**body, "network_policy": ["loopback"]}), "lists of paths"), + (lambda body: json.dumps({**body, "hidden_roots": "/x"}), "lists of paths"), + (lambda body: json.dumps({**body, "visible_roots": [1]}), "lists of paths"), + ], +) +def test_load_policy_refuses_malformed_files(tmp_path: Path, mutate, message: str): + """TRT-5/CON-8: a malformed `--policy` file refuses instead of attesting.""" + from aisle.harness.treatment_confinement import load_policy + + policy_file = tmp_path / "policy.json" + policy_file.write_text(mutate(_policy_json(tmp_path))) + with pytest.raises(ConfinementError, match=message): + load_policy(policy_file) + with pytest.raises(ConfinementError, match="unreadable"): + load_policy(tmp_path / "absent.json") + + +def test_load_policy_round_trips_the_canonical_form_and_the_cli_refuses_with_json( + tmp_path: Path, capsys +): + """TRT-5/CON-8: canonical JSON loads back to the same policy; a bad policy file + makes `audit-macos` exit 2 with JSON on stderr and nothing on stdout.""" + from aisle.harness.treatment_confinement import load_policy, main + + policy = _policy(tmp_path) + policy_file = tmp_path / "policy.json" + policy_file.write_text(json.dumps(policy.canonical_dict())) + assert load_policy(policy_file) == policy + policy_file.write_text("{") + assert main(["audit-macos", "--policy", str(policy_file), "--output", str(tmp_path / "o")]) == 2 + out, err = capsys.readouterr() + assert out == "" and json.loads(err)["ok"] is False + + +@pytest.mark.skipif(sys.platform != "darwin", reason="sandbox-exec capability is macOS-only") +def test_session_policy_audit_binds_the_launch_profile_and_leaves_no_sentinels(tmp_path: Path): + """TRT-5/TRT-6/TRT-7: the audit run under a session's exact policy yields an + attestation the launch wrapper accepts, proves network connects are refused, + and removes every sentinel it placed in the roots.""" + from aisle.harness.treatment_confinement import SANDBOX_EXEC, required_case_ids + + before = set(Path("/private/tmp").glob("aisle-sock-*")) + policy = _session_policy(tmp_path) + compiled = compile_macos_profile(policy) + report = run_macos_capability_audit(policy=policy) + cases = {row["id"]: row for row in report["cases"]} + assert report["capability_pass"] is True, [c for c in report["cases"] if not c["passed"]] + assert report["policy_source"] == "session" + expected_policy = policy.canonical_dict() + expected_policy["hidden_roots"] = [ + "sha256:" + hashlib.sha256(str(root).encode()).hexdigest() for root in policy.hidden_roots + ] + expected_policy["hidden_roots_redacted"] = True + assert report["policy"] == expected_policy + assert str(policy.hidden_roots[0]) not in json.dumps(report) + assert report["adapter"]["compiled_profile_sha256"] == compiled.sha256 + assert report["adapter"]["policy_id"] == compiled.policy_id + assert set(cases) == required_case_ids("deny-external") + assert report["summary"] == { + "baseline_tests": 7, + "capability_pass": True, + "declared_allow_tests": 4, + "denial_detection_rate": 1.0, + "denial_tests": 10, + "false_alarm_rate": 0.0, + } + for denial in ("tcp_read",): + assert cases[denial]["passed"] and cases[denial]["denied"], cases[denial] + assert not cases[denial]["sentinel_exposed"] + assert set(Path("/private/tmp").glob("aisle-sock-*")) == before + assert cases["parent_traversal_hidden_read"]["denied"] + assert "HIDDEN-SYNTHETIC" not in json.dumps(report) + for root in (*policy.visible_roots, *policy.output_roots, *policy.hidden_roots): + assert not (root / ".aisle-capability").exists() + assert list(root.iterdir()) == [] + profile_path = tmp_path / "profile.sb" + profile_path.write_text(compiled.text) + wrapped = wrap_verified_command(["/bin/cat", "x"], compiled, profile_path, report) + assert wrapped[:3] == [str(SANDBOX_EXEC), "-f", str(profile_path)] + + +@pytest.mark.skipif(sys.platform != "darwin", reason="sandbox-exec capability is macOS-only") +@pytest.mark.parametrize( + ("mutate", "message"), + [ + ( + lambda policy, root: MacOSPolicy( + **{ + **policy.as_dict(), + "allowed_executables": ( + *policy.allowed_executables, + Path("/usr/bin/printf").resolve(), + ), + } + ), + "unlisted-executable control", + ), + ( + lambda policy, root: MacOSPolicy( + **{ + **policy.as_dict(), + "allowed_executables": tuple( + p for p in policy.allowed_executables if p.name != "nc" + ), + } + ), + "Unix-socket probe", + ), + ( + lambda policy, root: MacOSPolicy( + **{ + **policy.as_dict(), + "allowed_executables": tuple( + p for p in policy.allowed_executables if p.name != "cat" + ), + } + ), + "audit probes", + ), + ( + lambda policy, root: (root / "view" / ".aisle-capability").mkdir() or policy, + "sentinel", + ), + ], +) +def test_session_policy_audit_refuses_policies_it_cannot_prove(tmp_path: Path, mutate, message): + """TRT-5: a session policy that lacks the probes, admits the unlisted-executable + control, or already carries sentinel paths refuses instead of attesting.""" + policy = mutate(_session_policy(tmp_path), tmp_path) + with pytest.raises(ConfinementError, match=message): + run_macos_capability_audit(policy=policy) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="sandbox-exec capability is macOS-only") +def test_capability_cli_accepts_a_session_policy_file(tmp_path: Path): + """TRT-6/CON-8: `audit-macos --policy` retains an attestation bound to that policy.""" + policy = _session_policy(tmp_path) + policy_file = tmp_path / "policy.json" + policy_file.write_text(json.dumps(policy.canonical_dict())) + output = tmp_path / "audit.json" + result = subprocess.run( + [ + sys.executable, + "-m", + "aisle.harness.treatment_confinement", + "audit-macos", + "--policy", + str(policy_file), + "--output", + str(output), + ], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["ok"] is True + report = json.loads(output.read_text()) + assert report["policy_source"] == "session" + assert report["adapter"]["policy_id"] == compile_macos_profile(policy).policy_id + + +def test_socket_probe_uses_an_admitted_client_and_never_widens_the_grants(tmp_path: Path): + """TRT-5: the Unix-socket probe is nc when admitted, else an admitted Python + interpreter, else the policy refuses; the audit adds no executable of its own.""" + from aisle.harness.treatment_confinement import _socket_client + + base = _policy(tmp_path).as_dict() + python = Path(sys.executable).resolve() + # admitted executables are canonical paths (on Linux /usr/bin/nc is an + # alternatives link), and the audit resolves its probe the same way + netcat = Path("/usr/bin/nc").resolve() + with_nc = MacOSPolicy(**{**base, "allowed_executables": (*base["allowed_executables"], netcat)}) + assert _socket_client(with_nc) == [str(netcat), "-w", "2", "-U"] + with_python = MacOSPolicy( + **{**base, "allowed_executables": (*base["allowed_executables"], python)} + ) + argv = _socket_client(with_python) + assert argv[:3] == [str(python), "-I", "-c"] and "AF_UNIX" in argv[3] + with pytest.raises(ConfinementError, match="Unix-socket probe"): + _socket_client(MacOSPolicy(**base)) + + +def test_corrupt_or_live_owner_markers_are_never_reclaimed(tmp_path: Path): + """TRT-6: only a well-formed marker naming a dead process authorizes reclaiming + a leftover sentinel; corrupt, missing or live markers refuse.""" + from aisle.harness.treatment_confinement import _reclaimable + + sentinel = tmp_path / ".aisle-capability" + sentinel.mkdir() + assert not _reclaimable(sentinel) + marker = sentinel / ".owner.json" + marker.write_text("{corrupt") + assert not _reclaimable(sentinel) + marker.write_text(json.dumps({"schema_version": "other", "pid": 2**22 + 1})) + assert not _reclaimable(sentinel) + marker.write_text(json.dumps({"schema_version": "aisle.sentinel-owner.v1", "pid": os.getpid()})) + assert not _reclaimable(sentinel) + marker.write_text(json.dumps({"schema_version": "aisle.sentinel-owner.v1", "pid": 2**22 + 1})) + assert _reclaimable(sentinel) + + +def test_loopback_endpoint_authority_refuses_before_profile_compilation(tmp_path: Path): + """TRT-5/TRT-7: a port shared by local addresses cannot authorize a relay-only grant.""" + policy = MacOSPolicy( + **{**_policy(tmp_path).as_dict(), "network_policy": "loopback", "loopback_port": 4321} + ) + with pytest.raises(ConfinementError, match="cannot isolate the provider relay"): + compile_macos_profile(policy) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="sandbox-exec capability is macOS-only") +def test_same_port_ipv4_and_ipv6_services_cannot_receive_a_relay_grant(tmp_path): + """TRT-5/TRT-7: distinct live IPv4/IPv6 endpoints can share the proposed relay port.""" + import socket + import threading + + from aisle.harness import treatment_confinement as tc + + base = _session_policy(tmp_path) + with socket.socket() as relay, socket.socket(socket.AF_INET6) as foreign: + relay.bind(("127.0.0.1", 0)) + relay.listen() + port = relay.getsockname()[1] + foreign.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) + foreign.bind(("::1", port)) + foreign.listen() + foreign.settimeout(0.1) + stopped = threading.Event() + server = threading.Thread( + target=tc._serve_once, args=(foreign, b"FOREIGN-SYNTHETIC-CANARY\n", stopped) + ) + server.start() + try: + command = ["/bin/bash", "-c", f"exec 3<>/dev/tcp/::1/{port} && /bin/cat <&3"] + baseline = subprocess.run(command, capture_output=True, timeout=5) + assert baseline.returncode == 0 + assert baseline.stdout == b"FOREIGN-SYNTHETIC-CANARY\n" + policy = MacOSPolicy( + **{**base.as_dict(), "network_policy": "loopback", "loopback_port": port} + ) + with pytest.raises(ConfinementError, match="cannot isolate the provider relay"): + compile_macos_profile(policy) + # The supported policy actually denies this same reachable endpoint. + profile = tmp_path / "profile.sb" + profile.write_text(compile_macos_profile(base).text) + confined = subprocess.run(tc._wrapped(profile, command), capture_output=True, timeout=5) + assert confined.returncode != 0 + assert b"FOREIGN-SYNTHETIC-CANARY" not in confined.stdout + confined.stderr + finally: + stopped.set() + server.join(timeout=2) + + +def test_cached_loopback_attestation_cannot_launch_a_command(tmp_path: Path): + """TRT-5: even a previously passing loopback report cannot authorize a launch.""" + from dataclasses import replace + + compiled = replace(compile_macos_profile(_policy(tmp_path)), network_policy="loopback") + profile = tmp_path / "profile.sb" + profile.write_text(compiled.text) + report = _attestation(compiled, profile, _fake_adapter(tmp_path)) + report["cases"] = [row for row in report["cases"] if row["id"] != "tcp_read"] + [ + {"id": name, "passed": True} + for name in ("loopback_tcp_read", "foreign_loopback_tcp_read", "external_tcp_read") + ] + with pytest.raises(ConfinementError, match="cannot isolate the provider relay"): + wrap_verified_command(["/bin/cat", "x"], compiled, profile, report) + + +@pytest.mark.skipif(sys.platform != "darwin", reason="sandbox-exec capability is macOS-only") +def test_loopback_audit_refuses_before_subprocesses_or_sentinel_writes(tmp_path, monkeypatch): + """TRT-5/TRT-6: unsupported endpoint authority refuses before touching session roots.""" + from aisle.harness import treatment_confinement as tc + + policy = MacOSPolicy( + **{**_policy(tmp_path).as_dict(), "network_policy": "loopback", "loopback_port": 4321} + ) + monkeypatch.setattr(tc, "_apple_git_runtime", lambda **_: pytest.fail("spawned a probe")) + with pytest.raises(ConfinementError, match="cannot isolate the provider relay"): + run_macos_capability_audit(policy) + assert all( + not list(path.iterdir()) + for path in (*policy.visible_roots, *policy.output_roots, *policy.hidden_roots) + ) diff --git a/tests/unit/test_typed_graph_preflight.py b/tests/unit/test_typed_graph_preflight.py index cd927a6b..e4e9b43d 100644 --- a/tests/unit/test_typed_graph_preflight.py +++ b/tests/unit/test_typed_graph_preflight.py @@ -1,7 +1,7 @@ """MON-6/MON-13: every staged worker must pass admission before graph transport starts.""" import json -from dataclasses import asdict, replace +from dataclasses import replace from pathlib import Path import pytest @@ -47,7 +47,7 @@ def _stage(tmp_path, *, share_home=False, direct_python=True): compiled = compile_macos_profile(policy) profile.write_text(compiled.text) launch.update( - policy=asdict(policy), + policy=policy.as_dict(), environment=environment, environment_record=environment_record, profile_path=str(profile), diff --git a/tests/unit/test_typed_graph_stage.py b/tests/unit/test_typed_graph_stage.py index 013c6255..af24d5fc 100644 --- a/tests/unit/test_typed_graph_stage.py +++ b/tests/unit/test_typed_graph_stage.py @@ -2,7 +2,6 @@ import copy import json -from dataclasses import asdict from pathlib import Path import pytest @@ -45,7 +44,7 @@ def _declarations(inputs): "timeout_s", ) } - launch["policy"] = asdict(launch["policy"]) + launch["policy"] = launch["policy"].as_dict() launch.update(bundle=bundle, bundle_manifest=manifest, source_roots=[ROOT, inputs["snapshot"]]) launch = json.loads(json.dumps(launch, default=str)) return {n["id"]: copy.deepcopy(launch) for n in graph["nodes"] if n["id"] in names} diff --git a/tests/unit/test_typed_node_host.py b/tests/unit/test_typed_node_host.py index 90ba7699..ce0f9e51 100644 --- a/tests/unit/test_typed_node_host.py +++ b/tests/unit/test_typed_node_host.py @@ -2,7 +2,6 @@ import hashlib import json -from dataclasses import asdict from pathlib import Path import pytest @@ -24,7 +23,7 @@ def _config(tmp_path): for name in ("node", "module", "outputs"): inputs.pop(name) output = inputs.pop("output") - inputs["policy"] = asdict(inputs["policy"]) + inputs["policy"] = inputs["policy"].as_dict() config = { "schema_version": "aisle.typed-node-host.v1", "purpose": "expert_parity", diff --git a/tests/unit/test_worker_capability.py b/tests/unit/test_worker_capability.py index 9161db23..e44944f4 100644 --- a/tests/unit/test_worker_capability.py +++ b/tests/unit/test_worker_capability.py @@ -13,10 +13,12 @@ def test_actual_worker_capability_report_and_cleanup(tmp_path): """TRT-6: all required observations bind to the actual policy and preserve worker assets.""" from aisle.harness.treatment_confinement import ( - _REQUIRED_CASE_IDS, compile_macos_profile, + required_case_ids, wrap_verified_command, ) + + _REQUIRED_CASE_IDS = required_case_ids("deny-external") from aisle.harness.worker_capability import audit_worker_capability inputs = _launch_inputs(tmp_path, direct_python=True) diff --git a/tools/matched_campaign.py b/tools/matched_campaign.py index 30c451f6..99f83cb8 100644 --- a/tools/matched_campaign.py +++ b/tools/matched_campaign.py @@ -57,12 +57,7 @@ def launch(destination: Path) -> dict: raise AdmissionError( "process requires confinement, ambient and launch bindings" ) from exc - policy = MacOSPolicy( - **{ - key: value if key == "network_policy" else tuple(Path(p) for p in value) - for key, value in declared.items() - } - ) + policy = MacOSPolicy.from_canonical(declared) compiled = compile_macos_profile(policy) if ( attestation.get("adapter", {}).get("sha256")