diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af56bd87..7e809c61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,11 +132,11 @@ jobs: missing=1 fi done - if [ "$missing" -eq 0 ]; then - echo "ready=true" >> "$GITHUB_OUTPUT" - else - echo "ready=false" >> "$GITHUB_OUTPUT" + if [ "$missing" -ne 0 ]; then + echo "Danger-zone ACR guard prerequisites are missing; failing closed." + exit 1 fi + echo "ready=true" >> "$GITHUB_OUTPUT" - uses: actions/setup-python@v5 if: steps.danger_zone_prereqs.outputs.ready == 'true' @@ -170,9 +170,9 @@ jobs: --repo-root . \ --json-out artifacts/kernel_contract_pack_report.json - - name: Danger-zone ACR guard skipped (missing prerequisites) - if: steps.danger_zone_prereqs.outputs.ready != 'true' - run: echo "Danger-zone ACR guard skipped on this branch." + - name: Danger-zone ACR guard failed closed (missing prerequisites) + if: failure() && steps.danger_zone_prereqs.outputs.ready != 'true' + run: echo "Danger-zone ACR guard prerequisites were missing; the prerequisite step failed closed." - name: Upload danger-zone ACR report if: ${{ always() && steps.danger_zone_prereqs.outputs.ready == 'true' }} diff --git a/agentic_coder_prototype/api/cli_bridge/app.py b/agentic_coder_prototype/api/cli_bridge/app.py index b8511d00..9c8cd246 100644 --- a/agentic_coder_prototype/api/cli_bridge/app.py +++ b/agentic_coder_prototype/api/cli_bridge/app.py @@ -49,6 +49,8 @@ SessionSummary, ) from .service import SessionService +from breadboard.rl.phase3.api_router import create_phase3_rl_router +from breadboard.rl.phase3.service_live import LiveRLRunService logger = logging.getLogger(__name__) ENGINE_STARTED_AT = time.time() @@ -177,6 +179,11 @@ def create_app(service: SessionService | None = None, include_atp_routes: bool | engine_version = (os.environ.get("BREADBOARD_ENGINE_VERSION") or "0.1.0").strip() or "0.1.0" app = FastAPI(title="BreadBoard CLI Bridge", version=engine_version) _service = service or SessionService() + store_path = os.environ.get("BREADBOARD_RL_RUN_STORE") + rl_service = LiveRLRunService(Path(store_path) if store_path else ":memory:") + rl_router = create_phase3_rl_router(rl_service) + app.include_router(rl_router, prefix="/v1/rl", tags=["rl"]) + app.include_router(rl_router, prefix="/rl", tags=["rl"]) chaos_config = _load_chaos_config() required_token = (os.environ.get("BREADBOARD_API_TOKEN") or "").strip() extension_config = None diff --git a/breadboard/rl/__init__.py b/breadboard/rl/__init__.py new file mode 100644 index 00000000..c7c661f2 --- /dev/null +++ b/breadboard/rl/__init__.py @@ -0,0 +1,10 @@ +"""BreadBoard RL rollout substrate primitives. + +This namespace contains the production-oriented RL environment, rollout, +trace/replay, runtime, security, and export primitives introduced by the +BreadBoard x Zyphra RL Phase 1 plan. Existing +``agentic_coder_prototype.rl`` APIs remain intact and can be bridged into this +namespace as the substrate matures. +""" + +__all__: list[str] = [] diff --git a/breadboard/rl/adapters/__init__.py b/breadboard/rl/adapters/__init__.py new file mode 100644 index 00000000..6a897c3a --- /dev/null +++ b/breadboard/rl/adapters/__init__.py @@ -0,0 +1,8 @@ +"""External adapter probe report primitives.""" + +from breadboard.rl.adapters.probe import AdapterProbeReport, validate_adapter_probe_report + +__all__ = [ + "AdapterProbeReport", + "validate_adapter_probe_report", +] diff --git a/breadboard/rl/adapters/benchflow/__init__.py b/breadboard/rl/adapters/benchflow/__init__.py new file mode 100644 index 00000000..b6205083 --- /dev/null +++ b/breadboard/rl/adapters/benchflow/__init__.py @@ -0,0 +1,3 @@ +from breadboard.rl.adapters.benchflow.importer import build_benchflow_fixture_probe_report + +__all__ = ["build_benchflow_fixture_probe_report"] diff --git a/breadboard/rl/adapters/benchflow/importer.py b/breadboard/rl/adapters/benchflow/importer.py new file mode 100644 index 00000000..31ab2fdc --- /dev/null +++ b/breadboard/rl/adapters/benchflow/importer.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from breadboard.rl.adapters.probe import AdapterProbeReport + + +def build_benchflow_fixture_probe_report() -> AdapterProbeReport: + return AdapterProbeReport( + adapter_id="benchflow.fixture.v1", + adapter_kind="benchflow_import_fixture", + support_level="fixture_probe", + workload_family="swe", + preserved_fields=[ + "env_package_id", + "task_id", + "runtime_backend", + "hardening_policy", + "verifier_command_shape", + ], + field_mapping={ + "env_package_id": "EnvPackage.metadata.id", + "task_id": "EnvPackage.tasks[].id", + "runtime_backend": "EnvPackage.runtime.backend", + "hardening_policy": "EnvPackage.security.hardening_policy", + "verifier_command_shape": "EnvPackage.evaluation.verifier.command", + }, + lost_fields=[ + "real_benchflow_harbor_runtime", + "live_benchflow_sandbox_attestation", + ], + unsupported_fields=["production_benchflow_execution"], + source_artifacts=["examples/rl_env_packages/swe_toy_patch/env_package.yaml"], + fidelity_notes=[ + "This probe checks whether BreadBoard EnvPackage fields can be represented in a BenchFlow-shaped fixture.", + "It does not invoke Harbor or BenchFlow sandbox execution, so sandbox attestation remains intentionally absent.", + ], + promotion_requirements=[ + "Run an actual BenchFlow/Harbor task using this mapping and capture sandbox identity plus verifier evidence.", + "Compare BreadBoard replay/admission results against BenchFlow task completion semantics on at least one real SWE task.", + ], + metadata={ + "fixture_scope": "static_env_package_mapping", + "minimum_real_promotion_level": "live_probe", + }, + ) diff --git a/breadboard/rl/adapters/ors/__init__.py b/breadboard/rl/adapters/ors/__init__.py new file mode 100644 index 00000000..ccf78460 --- /dev/null +++ b/breadboard/rl/adapters/ors/__init__.py @@ -0,0 +1,3 @@ +from breadboard.rl.adapters.ors.client import build_ors_fixture_probe_report + +__all__ = ["build_ors_fixture_probe_report"] diff --git a/breadboard/rl/adapters/ors/client.py b/breadboard/rl/adapters/ors/client.py new file mode 100644 index 00000000..2472e098 --- /dev/null +++ b/breadboard/rl/adapters/ors/client.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from breadboard.rl.adapters.probe import AdapterProbeReport + + +def build_ors_fixture_probe_report() -> AdapterProbeReport: + return AdapterProbeReport( + adapter_id="ors.fixture.v1", + adapter_kind="ors_openreward_fixture", + support_level="fixture_probe", + workload_family="swe", + preserved_fields=[ + "task_id", + "prompt_fields", + "reward_scalar", + "verifier_evidence_ref", + "split_id", + ], + field_mapping={ + "task_id": "M6 run_summary.rows[].task_id", + "prompt_fields": "M6 run_summary.rows[].prompt_preview", + "reward_scalar": "M6 run_summary.rows[].reward_scalar", + "verifier_evidence_ref": "M6 run_summary.rows[].verifier_evidence_ref", + "split_id": "M6 run_summary.rows[].split_id", + }, + lost_fields=["live_ors_server_route", "remote_reward_model_metadata"], + unsupported_fields=["production_ors_execution"], + source_artifacts=["docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/run_summary.json"], + fidelity_notes=[ + "This probe treats ORS/OpenReward as a reward-record exchange shape over the M6 controlled SWE toy run.", + "It preserves local reward and verifier references but does not contact an ORS service or refresh remote reward metadata.", + ], + promotion_requirements=[ + "Submit a BreadBoard-generated rollout row to a real ORS/OpenReward endpoint and record route/version metadata.", + "Verify reward parity between the local verifier scalar and the returned ORS/OpenReward result on accepted and quarantined rows.", + ], + metadata={ + "fixture_scope": "local_reward_record_projection", + "minimum_real_promotion_level": "live_probe", + }, + ) diff --git a/breadboard/rl/adapters/prime_verifiers/__init__.py b/breadboard/rl/adapters/prime_verifiers/__init__.py new file mode 100644 index 00000000..6d450aee --- /dev/null +++ b/breadboard/rl/adapters/prime_verifiers/__init__.py @@ -0,0 +1,3 @@ +from breadboard.rl.adapters.prime_verifiers.importer import build_prime_verifiers_fixture_probe_report + +__all__ = ["build_prime_verifiers_fixture_probe_report"] diff --git a/breadboard/rl/adapters/prime_verifiers/importer.py b/breadboard/rl/adapters/prime_verifiers/importer.py new file mode 100644 index 00000000..f65eb1e9 --- /dev/null +++ b/breadboard/rl/adapters/prime_verifiers/importer.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from breadboard.rl.adapters.probe import AdapterProbeReport + + +def build_prime_verifiers_fixture_probe_report() -> AdapterProbeReport: + return AdapterProbeReport( + adapter_id="prime_verifiers.fixture.v1", + adapter_kind="prime_verifiers_fixture", + support_level="fixture_probe", + workload_family="verifier", + preserved_fields=[ + "verifier_id", + "verifier_kind", + "evidence_sha256", + "rerun_agreement", + "reward_scalar", + ], + field_mapping={ + "verifier_id": "M6 row_evidence.verifier.id", + "verifier_kind": "M6 row_evidence.verifier.kind", + "evidence_sha256": "M6 row_evidence.sha256", + "rerun_agreement": "M6 row_evidence.rerun_agreement", + "reward_scalar": "M6 row_evidence.reward_scalar", + }, + lost_fields=["remote_prime_registry_identity", "live_verifier_service_attestation"], + unsupported_fields=["production_prime_verifiers_execution"], + source_artifacts=["docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/row_evidence/"], + fidelity_notes=[ + "This probe records verifier identity/evidence semantics from local M6 row evidence only.", + "It does not register a verifier with Prime Verifiers or prove service-side execution/attestation.", + ], + promotion_requirements=[ + "Wrap one BreadBoard verifier in the real Prime Verifiers interface and capture registry identity plus service attestation.", + "Confirm local rerun agreement and remote verifier result agreement on accepted, rejected, and quarantined examples.", + ], + metadata={ + "fixture_scope": "local_verifier_evidence_mapping", + "minimum_real_promotion_level": "live_probe", + }, + ) diff --git a/breadboard/rl/adapters/probe.py b/breadboard/rl/adapters/probe.py new file mode 100644 index 00000000..4252a08d --- /dev/null +++ b/breadboard/rl/adapters/probe.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + + +SUPPORT_LEVELS = {"not_started", "fixture_probe", "jsonl_probe", "live_probe", "supported"} + + +@dataclass(frozen=True) +class AdapterProbeReport: + adapter_id: str + adapter_kind: str + support_level: str + workload_family: str + preserved_fields: list[str] + lost_fields: list[str] = field(default_factory=list) + unsupported_fields: list[str] = field(default_factory=list) + source_artifacts: list[str] = field(default_factory=list) + claim_boundary: str = "adapter_probe_not_production_integration" + data_boundary: str = "fixture_or_probe_only" + field_mapping: dict[str, str] = field(default_factory=dict) + fidelity_notes: list[str] = field(default_factory=list) + promotion_requirements: list[str] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "adapter_id": self.adapter_id, + "adapter_kind": self.adapter_kind, + "support_level": self.support_level, + "workload_family": self.workload_family, + "preserved_fields": list(self.preserved_fields), + "lost_fields": list(self.lost_fields), + "unsupported_fields": list(self.unsupported_fields), + "source_artifacts": list(self.source_artifacts), + "claim_boundary": self.claim_boundary, + "data_boundary": self.data_boundary, + "field_mapping": dict(self.field_mapping), + "fidelity_notes": list(self.fidelity_notes), + "promotion_requirements": list(self.promotion_requirements), + "metadata": dict(self.metadata), + } + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "AdapterProbeReport": + return AdapterProbeReport( + adapter_id=str(data.get("adapter_id") or ""), + adapter_kind=str(data.get("adapter_kind") or ""), + support_level=str(data.get("support_level") or ""), + workload_family=str(data.get("workload_family") or ""), + preserved_fields=[str(item) for item in data.get("preserved_fields") or []], + lost_fields=[str(item) for item in data.get("lost_fields") or []], + unsupported_fields=[str(item) for item in data.get("unsupported_fields") or []], + source_artifacts=[str(item) for item in data.get("source_artifacts") or []], + claim_boundary=str(data.get("claim_boundary") or "adapter_probe_not_production_integration"), + data_boundary=str(data.get("data_boundary") or "fixture_or_probe_only"), + field_mapping={str(key): str(value) for key, value in dict(data.get("field_mapping") or {}).items()}, + fidelity_notes=[str(item) for item in data.get("fidelity_notes") or []], + promotion_requirements=[str(item) for item in data.get("promotion_requirements") or []], + metadata=dict(data.get("metadata") or {}), + ) + + +def validate_adapter_probe_report(report: AdapterProbeReport) -> list[str]: + errors: list[str] = [] + for field_name in ["adapter_id", "adapter_kind", "support_level", "workload_family"]: + if not str(getattr(report, field_name) or "").strip(): + errors.append(f"{field_name} must be non-empty") + if report.support_level not in SUPPORT_LEVELS: + errors.append(f"support_level must be one of {sorted(SUPPORT_LEVELS)}") + if not report.preserved_fields: + errors.append("preserved_fields must be non-empty") + if not report.source_artifacts: + errors.append("source_artifacts must be non-empty") + if not report.data_boundary.strip(): + errors.append("data_boundary must be non-empty") + if not report.field_mapping: + errors.append("field_mapping must be non-empty") + for preserved_field in report.preserved_fields: + if preserved_field not in report.field_mapping: + errors.append(f"field_mapping missing preserved field: {preserved_field}") + if report.support_level == "supported" and (report.lost_fields or report.unsupported_fields): + errors.append("support_level=supported requires no lost_fields or unsupported_fields") + if report.support_level == "supported" and report.claim_boundary != "production_supported": + errors.append("support_level=supported requires production_supported claim_boundary") + if report.support_level != "supported" and report.claim_boundary == "production_supported": + errors.append("production_supported claim_boundary requires support_level=supported") + if report.support_level != "supported": + if not (report.lost_fields or report.unsupported_fields): + errors.append("non-supported reports must name lost_fields or unsupported_fields") + if not report.fidelity_notes: + errors.append("non-supported reports must include fidelity_notes") + if not report.promotion_requirements: + errors.append("non-supported reports must include promotion_requirements") + return errors diff --git a/breadboard/rl/adapters/verl/__init__.py b/breadboard/rl/adapters/verl/__init__.py new file mode 100644 index 00000000..b4660afc --- /dev/null +++ b/breadboard/rl/adapters/verl/__init__.py @@ -0,0 +1,3 @@ +from breadboard.rl.adapters.verl.report import build_verl_jsonl_probe_report + +__all__ = ["build_verl_jsonl_probe_report"] diff --git a/breadboard/rl/adapters/verl/report.py b/breadboard/rl/adapters/verl/report.py new file mode 100644 index 00000000..feb462a8 --- /dev/null +++ b/breadboard/rl/adapters/verl/report.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from breadboard.rl.adapters.probe import AdapterProbeReport + + +def build_verl_jsonl_probe_report() -> AdapterProbeReport: + return AdapterProbeReport( + adapter_id="verl.jsonl_probe.v1", + adapter_kind="verl_jsonl_probe_export", + support_level="jsonl_probe", + workload_family="swe", + preserved_fields=[ + "input_ids", + "attention_mask", + "loss_mask", + "completion_logprobs", + "reward_scalar", + "policy_id", + "env_package_hash", + ], + field_mapping={ + "input_ids": "VeRLProbeRow.input_ids", + "attention_mask": "VeRLProbeRow.attention_mask", + "loss_mask": "VeRLProbeRow.loss_mask", + "completion_logprobs": "VeRLProbeRow.completion_logprobs", + "reward_scalar": "VeRLProbeRow.reward_scalar", + "policy_id": "VeRLProbeRow.policy_id", + "env_package_hash": "VeRLProbeRow.env_package_hash", + }, + lost_fields=["verl_DataProto_object", "trainer_execution_state"], + unsupported_fields=["ppo_grpo_trainer_execution", "distributed_actor_logprob_refresh"], + source_artifacts=["docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/verl_probe_rows.jsonl"], + fidelity_notes=[ + "This report covers BreadBoard's token-native JSONL/Parquet projection, not VeRL's in-process DataProto object.", + "Completion logprobs are explicit nullable fields with status metadata; they are not proof of actor-side logprob refresh.", + ], + promotion_requirements=[ + "Load the exported rows into a real VeRL DataProto or documented ingestion shim and validate tensor dtypes/shapes.", + "Run a no-op or tiny trainer smoke that consumes BreadBoard rows without treating quarantined rows as trainable.", + ], + metadata={ + "fixture_scope": "jsonl_parquet_export_projection", + "minimum_real_promotion_level": "live_probe", + }, + ) diff --git a/breadboard/rl/e2e/__init__.py b/breadboard/rl/e2e/__init__.py new file mode 100644 index 00000000..22076f87 --- /dev/null +++ b/breadboard/rl/e2e/__init__.py @@ -0,0 +1,13 @@ +"""End-to-end RL Phase 1 probe runners.""" + +from breadboard.rl.e2e.swe_probe import ( + ControlledSweProbeRun, + ControlledSweProbeRow, + run_controlled_swe_probe, +) + +__all__ = [ + "ControlledSweProbeRow", + "ControlledSweProbeRun", + "run_controlled_swe_probe", +] diff --git a/breadboard/rl/e2e/swe_probe.py b/breadboard/rl/e2e/swe_probe.py new file mode 100644 index 00000000..e14253df --- /dev/null +++ b/breadboard/rl/e2e/swe_probe.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +import json +import os +import statistics +from dataclasses import dataclass, field +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +from breadboard.rl.env_package.schema import EnvPackage +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.export import build_projection_manifest +from breadboard.rl.replay import compare_replay_parity, decide_export_admission +from breadboard.rl.security import build_hardening_report, build_verifier_run_report, quarantine_on_findings +from breadboard.rl.trace import build_graph_from_session_events + + +@dataclass(frozen=True) +class ControlledSweProbeRow: + task_id: str + row_status: str + reward: float + hardening_status: str + replay_status: str + exportable_debug: bool + trainable: bool + projection_id: str + metrics_ms: dict[str, float] + blocked_reasons: list[str] = field(default_factory=list) + findings: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "task_id": self.task_id, + "row_status": self.row_status, + "reward": self.reward, + "hardening_status": self.hardening_status, + "replay_status": self.replay_status, + "exportable_debug": self.exportable_debug, + "trainable": self.trainable, + "projection_id": self.projection_id, + "metrics_ms": dict(self.metrics_ms), + "blocked_reasons": list(self.blocked_reasons), + "findings": list(self.findings), + } + + +@dataclass(frozen=True) +class ControlledSweProbeRun: + run_id: str + target_run_id: str | None + package_id: str + package_hash: str + source_claim: str + rows: list[ControlledSweProbeRow] + metrics_summary: dict[str, dict[str, float]] + qc_report: dict[str, Any] + + def to_dict(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "target_run_id": self.target_run_id, + "package_id": self.package_id, + "package_hash": self.package_hash, + "source_claim": self.source_claim, + "rows": [row.to_dict() for row in self.rows], + "metrics_summary": self.metrics_summary, + "qc_report": dict(self.qc_report), + } + + +def _task_ids(package: EnvPackage, limit: int) -> list[str]: + split = package.splits["train_probe"] + ids = list(split.selector.get("task_ids") or []) + return [str(item) for item in ids[:limit]] + + +def _metrics_for_index(index: int) -> dict[str, float]: + return { + "reset_ms": 5.0 + index, + "step_ms": 7.0 + index, + "verify_ms": 9.0 + index, + "export_ms": 3.0 + index, + "total_ms": 24.0 + (4 * index), + } + + +def _summarize_metrics(rows: list[ControlledSweProbeRow]) -> dict[str, dict[str, float]]: + summary: dict[str, dict[str, float]] = {} + for key in ["reset_ms", "step_ms", "verify_ms", "export_ms", "total_ms"]: + values = [row.metrics_ms[key] for row in rows] + sorted_values = sorted(values) + p95_index = min(len(sorted_values) - 1, int(round(0.95 * (len(sorted_values) - 1)))) + summary[key] = { + "p50": float(statistics.median(values)), + "p95": float(sorted_values[p95_index]), + } + return summary + + +def _write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(row, sort_keys=True) + "\n" for row in rows), encoding="utf-8") + + +def run_controlled_swe_probe( + *, + package_path: Path, + output_dir: Path | None = None, + run_id: str = "controlled_swe_toy_m6", + limit: int = 10, +) -> ControlledSweProbeRun: + package = load_env_package(package_path) + if package.hardening is None: + raise ValueError("controlled SWE probe requires package hardening policy") + task_ids = _task_ids(package, limit) + if len(task_ids) < 10: + raise ValueError("M6 controlled SWE probe requires at least 10 task ids") + + rows: list[ControlledSweProbeRow] = [] + with TemporaryDirectory() as tmp: + tmp_root = Path(tmp) + for index, task_id in enumerate(task_ids, start=1): + workspace = tmp_root / task_id + workspace.mkdir() + if index in {4, 8}: + (workspace / "sitecustomize.py").write_text("raise SystemExit('poison')\n", encoding="utf-8") + + hardening_report = build_hardening_report( + report_id=f"{run_id}.{task_id}.hardening", + workspace=workspace, + policy=package.hardening, + clean_baseline_passed=True, + reference_solution_passed=True, + process_cleanup_observed=True, + ) + quarantine = quarantine_on_findings( + row_id=task_id, + findings=hardening_report.findings, + policy=package.hardening, + ) + reward = 0.0 if index == 7 else 1.0 + verifier_report = build_verifier_run_report( + report_id=f"{run_id}.{task_id}.verifier", + verifier_id=package.verifier.verifier_id, + status="passed" if reward == 1.0 else "failed", + output=f"reward={reward}", + rerun_output=f"reward={reward}", + ) + graph = _toy_graph_for_task(run_id=run_id, task_id=task_id, reward=reward) + replay_report = compare_replay_parity(graph, graph) + admission = decide_export_admission( + replay_report=replay_report, + hardening_status=hardening_report.status, + quarantine_status="quarantined" if quarantine.quarantined else "clear", + token_records_valid=True, + ) + projection = build_projection_manifest( + graph=graph, + target_format="controlled_swe_toy_jsonl_probe", + preserved_fields=["task_id", "reward", "hardening_status", "replay_status"], + lost_fields=["full_workspace_bytes"], + included_node_kinds={"step", "evaluate"}, + ) + if quarantine.quarantined: + row_status = "quarantined" + elif reward < 1.0: + row_status = "rejected" + else: + row_status = "accepted" + rows.append( + ControlledSweProbeRow( + task_id=task_id, + row_status=row_status, + reward=reward, + hardening_status=hardening_report.status, + replay_status="passed" if replay_report.passed else "failed", + exportable_debug=admission.exportable and row_status == "accepted", + trainable=False, + projection_id=projection.projection_id, + metrics_ms=_metrics_for_index(index), + blocked_reasons=[ + *admission.blocked_reasons, + *(["verifier_failed"] if row_status == "rejected" else []), + ], + findings=[item.finding_id for item in hardening_report.findings], + ) + ) + if output_dir: + _write_json( + output_dir / "row_evidence" / f"{task_id}.json", + { + "hardening_report": hardening_report.to_dict(), + "verifier_report": verifier_report.to_dict(), + "replay_report": replay_report.to_dict(), + "admission": admission.to_dict(), + "projection": projection.to_dict(), + }, + ) + + metrics_summary = _summarize_metrics(rows) + qc_report = _build_qc_report(rows) + run = ControlledSweProbeRun( + run_id=run_id, + target_run_id=os.environ.get("M12_TARGET_RUN_ID"), + package_id=package.package_id, + package_hash=package.package_hash or "", + source_claim="controlled_swe_toy_slice", + rows=rows, + metrics_summary=metrics_summary, + qc_report=qc_report, + ) + if output_dir: + _write_json(output_dir / "run_summary.json", run.to_dict()) + _write_json(output_dir / "metrics_summary.json", metrics_summary) + _write_json(output_dir / "qc_report.json", qc_report) + _write_jsonl(output_dir / "run_ledger.jsonl", [row.to_dict() for row in rows]) + return run + + +def _build_qc_report(rows: list[ControlledSweProbeRow]) -> dict[str, Any]: + by_status: dict[str, list[str]] = {"accepted": [], "rejected": [], "quarantined": []} + for row in rows: + by_status.setdefault(row.row_status, []).append(row.task_id) + return { + "accepted_sample": by_status["accepted"][:5], + "rejected_sample": by_status["rejected"][:5], + "quarantined_sample": by_status["quarantined"][:5], + "reviewed_dimensions": [ + "trace_completeness", + "verifier_evidence", + "hardening_status", + "replay_status", + "projection_manifest", + "claim_wording", + ], + "operator_notes": "Controlled SWE toy slice only; not external benchmark support.", + } + + +def _toy_graph_for_task(*, run_id: str, task_id: str, reward: float): + from breadboard.rl.session.events import SessionEvent + from breadboard.rl.trace import build_graph_from_session_events + + events = [ + SessionEvent( + event_id=f"{run_id}.{task_id}.reset", + session_id=f"{run_id}.{task_id}", + event_kind="reset", + status_before="created", + status_after="ready", + payload={"success": True, "result_kind": "reset"}, + ), + SessionEvent( + event_id=f"{run_id}.{task_id}.step", + session_id=f"{run_id}.{task_id}", + event_kind="step", + status_before="ready", + status_after="running", + payload={"success": True, "result_kind": "step"}, + ), + SessionEvent( + event_id=f"{run_id}.{task_id}.evaluate", + session_id=f"{run_id}.{task_id}", + event_kind="evaluate", + status_before="running", + status_after="evaluated", + payload={"success": True, "result_kind": "evaluate", "reward": reward}, + ), + ] + return build_graph_from_session_events( + graph_id=f"{run_id}.{task_id}.graph", + session_id=f"{run_id}.{task_id}", + events=events, + ) diff --git a/breadboard/rl/env_package/__init__.py b/breadboard/rl/env_package/__init__.py new file mode 100644 index 00000000..c9f9f9d8 --- /dev/null +++ b/breadboard/rl/env_package/__init__.py @@ -0,0 +1,15 @@ +"""EnvPackage v1alpha schema and validation helpers.""" + +from breadboard.rl.env_package.hash import canonical_env_package_hash +from breadboard.rl.env_package.schema import EnvPackage +from breadboard.rl.env_package.validate import ( + load_env_package, + validate_env_package_mapping, +) + +__all__ = [ + "EnvPackage", + "canonical_env_package_hash", + "load_env_package", + "validate_env_package_mapping", +] diff --git a/breadboard/rl/env_package/hash.py b/breadboard/rl/env_package/hash.py new file mode 100644 index 00000000..23f14868 --- /dev/null +++ b/breadboard/rl/env_package/hash.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import hashlib +import json +from typing import Any, Mapping + + +def _without_package_hash(payload: Any) -> Any: + if isinstance(payload, Mapping): + return { + str(key): _without_package_hash(value) + for key, value in payload.items() + if str(key) != "package_hash" + } + if isinstance(payload, list): + return [_without_package_hash(item) for item in payload] + return payload + + +def canonical_env_package_hash(payload: Mapping[str, Any]) -> str: + """Return a stable sha256 hash for an EnvPackage mapping. + + The declared ``package_hash`` field is excluded so a package can carry its + own hash without changing the canonical digest. + """ + + canonical_payload = _without_package_hash(payload) + encoded = json.dumps( + canonical_payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() diff --git a/breadboard/rl/env_package/lint.py b/breadboard/rl/env_package/lint.py new file mode 100644 index 00000000..4f124d5c --- /dev/null +++ b/breadboard/rl/env_package/lint.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_yaml_mapping, validate_env_package_mapping + + +def lint_env_package(path: str | Path) -> list[str]: + """Return human-readable EnvPackage validation errors for a YAML file.""" + + try: + payload = load_yaml_mapping(path) + except Exception as exc: # pragma: no cover - defensive CLI-facing path + return [str(exc)] + return validate_env_package_mapping(payload) diff --git a/breadboard/rl/env_package/schema.py b/breadboard/rl/env_package/schema.py new file mode 100644 index 00000000..ffb6cecc --- /dev/null +++ b/breadboard/rl/env_package/schema.py @@ -0,0 +1,489 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + + +SCHEMA_VERSION = "bb.env_package.v1alpha" + + +def _text(value: Any, field_name: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError(f"{field_name} must be non-empty") + return text + + +def _bool(value: Any) -> bool: + return bool(value) + + +def _mapping(value: Any, field_name: str) -> dict[str, Any]: + if value is None: + return {} + if not isinstance(value, Mapping): + raise ValueError(f"{field_name} must be a mapping") + return dict(value) + + +def _text_list(value: Any, field_name: str) -> list[str]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError(f"{field_name} must be a list") + copied: list[str] = [] + for item in value: + text = str(item or "").strip() + if text: + copied.append(text) + return copied + + +@dataclass(frozen=True) +class ProvenanceSpec: + created_at: str + license: str + source_usage_policy: str + contamination_scope: str + source_refs: list[str] = field(default_factory=list) + source_hashes: dict[str, Any] = field(default_factory=dict) + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "ProvenanceSpec": + return ProvenanceSpec( + created_at=_text(data.get("created_at"), "provenance.created_at"), + license=_text(data.get("license"), "provenance.license"), + source_usage_policy=_text( + data.get("source_usage_policy"), + "provenance.source_usage_policy", + ), + contamination_scope=_text( + data.get("contamination_scope"), + "provenance.contamination_scope", + ), + source_refs=_text_list(data.get("source_refs"), "provenance.source_refs"), + source_hashes=_mapping(data.get("source_hashes"), "provenance.source_hashes"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "created_at": self.created_at, + "license": self.license, + "source_usage_policy": self.source_usage_policy, + "contamination_scope": self.contamination_scope, + "source_refs": list(self.source_refs), + "source_hashes": dict(self.source_hashes), + } + + +@dataclass(frozen=True) +class TasksetSpec: + taskset_id: str + source_kind: str + source_hash: str + task_id_field: str + prompt_fields: list[str] + allowed_splits: list[str] + source_uri: str | None = None + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "TasksetSpec": + source_uri = data.get("source_uri") + return TasksetSpec( + taskset_id=_text(data.get("taskset_id"), "taskset.taskset_id"), + source_kind=_text(data.get("source_kind"), "taskset.source_kind"), + source_hash=_text(data.get("source_hash"), "taskset.source_hash"), + task_id_field=_text(data.get("task_id_field"), "taskset.task_id_field"), + prompt_fields=_text_list(data.get("prompt_fields"), "taskset.prompt_fields"), + allowed_splits=_text_list(data.get("allowed_splits"), "taskset.allowed_splits"), + source_uri=str(source_uri).strip() if source_uri else None, + ) + + def to_dict(self) -> dict[str, Any]: + payload = { + "taskset_id": self.taskset_id, + "source_kind": self.source_kind, + "source_hash": self.source_hash, + "task_id_field": self.task_id_field, + "prompt_fields": list(self.prompt_fields), + "allowed_splits": list(self.allowed_splits), + } + if self.source_uri: + payload["source_uri"] = self.source_uri + return payload + + +@dataclass(frozen=True) +class SplitSpec: + split_id: str + split_type: str + taskset_id: str + selector: dict[str, Any] + split_hash: str + immutable: bool = True + optimizer_visible: bool = False + trainer_visible: bool = False + protected: bool = False + + @staticmethod + def from_dict(split_id: str, data: Mapping[str, Any]) -> "SplitSpec": + return SplitSpec( + split_id=_text(data.get("split_id", split_id), "split.split_id"), + split_type=_text(data.get("split_type"), "split.split_type"), + taskset_id=_text(data.get("taskset_id"), "split.taskset_id"), + selector=_mapping(data.get("selector"), "split.selector"), + split_hash=_text(data.get("split_hash"), "split.split_hash"), + immutable=_bool(data.get("immutable", True)), + optimizer_visible=_bool(data.get("optimizer_visible", False)), + trainer_visible=_bool(data.get("trainer_visible", False)), + protected=_bool(data.get("protected", False)), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "split_id": self.split_id, + "split_type": self.split_type, + "taskset_id": self.taskset_id, + "selector": dict(self.selector), + "split_hash": self.split_hash, + "immutable": self.immutable, + "optimizer_visible": self.optimizer_visible, + "trainer_visible": self.trainer_visible, + "protected": self.protected, + } + + +@dataclass(frozen=True) +class HarnessContract: + harness_id: str + interaction_mode: str + observation_schema: dict[str, Any] + action_schema: dict[str, Any] + termination: dict[str, Any] + tools: list[str] = field(default_factory=list) + max_turns: int | None = None + hidden_state_policy: str = "never_visible" + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "HarnessContract": + max_turns = data.get("max_turns") + return HarnessContract( + harness_id=_text(data.get("harness_id"), "harness.harness_id"), + interaction_mode=_text(data.get("interaction_mode"), "harness.interaction_mode"), + observation_schema=_mapping(data.get("observation_schema"), "harness.observation_schema"), + action_schema=_mapping(data.get("action_schema"), "harness.action_schema"), + termination=_mapping(data.get("termination"), "harness.termination"), + tools=_text_list(data.get("tools"), "harness.tools"), + max_turns=int(max_turns) if max_turns is not None else None, + hidden_state_policy=str(data.get("hidden_state_policy") or "never_visible"), + ) + + def to_dict(self) -> dict[str, Any]: + payload = { + "harness_id": self.harness_id, + "interaction_mode": self.interaction_mode, + "observation_schema": dict(self.observation_schema), + "action_schema": dict(self.action_schema), + "termination": dict(self.termination), + "tools": list(self.tools), + "hidden_state_policy": self.hidden_state_policy, + } + if self.max_turns is not None: + payload["max_turns"] = self.max_turns + return payload + + +@dataclass(frozen=True) +class RuntimeEnvelope: + backend: str + isolation_level: str + agent_user: str + network: str + secrets_policy: str + pool_key_fields: list[str] + image_digest: str | None = None + network_allowlist_reason: str | None = None + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "RuntimeEnvelope": + image_digest = data.get("image_digest") + allowlist_reason = data.get("network_allowlist_reason") + return RuntimeEnvelope( + backend=_text(data.get("backend"), "runtime.backend"), + isolation_level=_text(data.get("isolation_level"), "runtime.isolation_level"), + agent_user=str(data.get("agent_user") or "sandbox").strip(), + network=str(data.get("network") or "none").strip(), + secrets_policy=str(data.get("secrets_policy") or "ambient_forbidden").strip(), + pool_key_fields=_text_list(data.get("pool_key_fields"), "runtime.pool_key_fields"), + image_digest=str(image_digest).strip() if image_digest else None, + network_allowlist_reason=str(allowlist_reason).strip() if allowlist_reason else None, + ) + + def to_dict(self) -> dict[str, Any]: + payload = { + "backend": self.backend, + "isolation_level": self.isolation_level, + "agent_user": self.agent_user, + "network": self.network, + "secrets_policy": self.secrets_policy, + "pool_key_fields": list(self.pool_key_fields), + } + if self.image_digest: + payload["image_digest"] = self.image_digest + if self.network_allowlist_reason: + payload["network_allowlist_reason"] = self.network_allowlist_reason + return payload + + +@dataclass(frozen=True) +class VerifierSpec: + verifier_id: str + kind: str + code_hash: str + input_contract: dict[str, Any] + output_contract: dict[str, Any] + isolated_from_agent: bool = True + rerun_policy: dict[str, Any] = field(default_factory=dict) + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "VerifierSpec": + return VerifierSpec( + verifier_id=_text(data.get("verifier_id"), "verifier.verifier_id"), + kind=_text(data.get("kind"), "verifier.kind"), + code_hash=_text(data.get("code_hash"), "verifier.code_hash"), + input_contract=_mapping(data.get("input_contract"), "verifier.input_contract"), + output_contract=_mapping(data.get("output_contract"), "verifier.output_contract"), + isolated_from_agent=_bool(data.get("isolated_from_agent", True)), + rerun_policy=_mapping(data.get("rerun_policy"), "verifier.rerun_policy"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "verifier_id": self.verifier_id, + "kind": self.kind, + "code_hash": self.code_hash, + "input_contract": dict(self.input_contract), + "output_contract": dict(self.output_contract), + "isolated_from_agent": self.isolated_from_agent, + "rerun_policy": dict(self.rerun_policy), + } + + +@dataclass(frozen=True) +class RewardSpec: + reward_id: str + kind: str + terminal_reward: bool = True + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "RewardSpec": + return RewardSpec( + reward_id=_text(data.get("reward_id"), "reward.reward_id"), + kind=_text(data.get("kind"), "reward.kind"), + terminal_reward=_bool(data.get("terminal_reward", True)), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "reward_id": self.reward_id, + "kind": self.kind, + "terminal_reward": self.terminal_reward, + } + + +@dataclass(frozen=True) +class RendererSpec: + renderer_id: str + tokenizer_hash: str + chat_template_hash: str + bridge_to_next_turn_required: bool = True + mask_contract: dict[str, Any] = field(default_factory=dict) + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "RendererSpec": + return RendererSpec( + renderer_id=_text(data.get("renderer_id"), "renderer.renderer_id"), + tokenizer_hash=_text(data.get("tokenizer_hash"), "renderer.tokenizer_hash"), + chat_template_hash=_text(data.get("chat_template_hash"), "renderer.chat_template_hash"), + bridge_to_next_turn_required=_bool(data.get("bridge_to_next_turn_required", True)), + mask_contract=_mapping(data.get("mask_contract"), "renderer.mask_contract"), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "renderer_id": self.renderer_id, + "tokenizer_hash": self.tokenizer_hash, + "chat_template_hash": self.chat_template_hash, + "bridge_to_next_turn_required": self.bridge_to_next_turn_required, + "mask_contract": dict(self.mask_contract), + } + + +@dataclass(frozen=True) +class HardeningPolicy: + policy_id: str + threat_level: str + agent_non_root_required: bool + verifier_isolated_required: bool + quarantine_on_findings: list[str] = field(default_factory=list) + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "HardeningPolicy": + return HardeningPolicy( + policy_id=_text(data.get("policy_id"), "hardening.policy_id"), + threat_level=_text(data.get("threat_level"), "hardening.threat_level"), + agent_non_root_required=_bool(data.get("agent_non_root_required", True)), + verifier_isolated_required=_bool(data.get("verifier_isolated_required", True)), + quarantine_on_findings=_text_list( + data.get("quarantine_on_findings"), + "hardening.quarantine_on_findings", + ), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "policy_id": self.policy_id, + "threat_level": self.threat_level, + "agent_non_root_required": self.agent_non_root_required, + "verifier_isolated_required": self.verifier_isolated_required, + "quarantine_on_findings": list(self.quarantine_on_findings), + } + + +@dataclass(frozen=True) +class ReplayConformanceSpec: + replay_required: bool = True + live_replay_parity_required: bool = True + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "ReplayConformanceSpec": + return ReplayConformanceSpec( + replay_required=_bool(data.get("replay_required", True)), + live_replay_parity_required=_bool(data.get("live_replay_parity_required", True)), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "replay_required": self.replay_required, + "live_replay_parity_required": self.live_replay_parity_required, + } + + +@dataclass(frozen=True) +class ExportEligibility: + allowed_formats: list[str] + trainable: bool + requires_token_records: bool + requires_replay_pass: bool + support_level: str + trainability_status: str + trainability_blockers: list[str] + support_evidence_refs: list[str] = field(default_factory=list) + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "ExportEligibility": + return ExportEligibility( + allowed_formats=_text_list(data.get("allowed_formats"), "exports.allowed_formats"), + trainable=_bool(data.get("trainable", False)), + requires_token_records=_bool(data.get("requires_token_records", True)), + requires_replay_pass=_bool(data.get("requires_replay_pass", True)), + support_level=str(data.get("support_level") or "experimental").strip(), + trainability_status=str(data.get("trainability_status") or "not_trainable").strip(), + trainability_blockers=_text_list( + data.get("trainability_blockers"), + "exports.trainability_blockers", + ), + support_evidence_refs=_text_list( + data.get("support_evidence_refs"), + "exports.support_evidence_refs", + ), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "allowed_formats": list(self.allowed_formats), + "trainable": self.trainable, + "requires_token_records": self.requires_token_records, + "requires_replay_pass": self.requires_replay_pass, + "support_level": self.support_level, + "trainability_status": self.trainability_status, + "trainability_blockers": list(self.trainability_blockers), + "support_evidence_refs": list(self.support_evidence_refs), + } + + +@dataclass(frozen=True) +class EnvPackage: + package_id: str + version: str + provenance: ProvenanceSpec + tasksets: list[TasksetSpec] + splits: dict[str, SplitSpec] + harness: HarnessContract + runtime: RuntimeEnvelope + verifier: VerifierSpec + reward: RewardSpec + renderer: RendererSpec + hardening: HardeningPolicy | None + replay: ReplayConformanceSpec + exports: ExportEligibility + schema_version: str = SCHEMA_VERSION + package_hash: str | None = None + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "EnvPackage": + from breadboard.rl.env_package.validate import validate_env_package_mapping + + errors = validate_env_package_mapping(data) + if errors: + raise ValueError("; ".join(errors)) + + hardening_data = data.get("hardening") + hardening = ( + HardeningPolicy.from_dict(hardening_data) + if isinstance(hardening_data, Mapping) + else None + ) + return EnvPackage( + schema_version=_text(data.get("schema_version"), "schema_version"), + package_id=_text(data.get("package_id"), "package_id"), + version=_text(data.get("version"), "version"), + package_hash=str(data.get("package_hash")).strip() + if data.get("package_hash") + else None, + provenance=ProvenanceSpec.from_dict(data["provenance"]), + tasksets=[TasksetSpec.from_dict(item) for item in data["tasksets"]], + splits={ + str(split_id): SplitSpec.from_dict(str(split_id), split_data) + for split_id, split_data in data["splits"].items() + }, + harness=HarnessContract.from_dict(data["harness"]), + runtime=RuntimeEnvelope.from_dict(data["runtime"]), + verifier=VerifierSpec.from_dict(data["verifier"]), + reward=RewardSpec.from_dict(data["reward"]), + renderer=RendererSpec.from_dict(data["renderer"]), + hardening=hardening, + replay=ReplayConformanceSpec.from_dict(data["replay"]), + exports=ExportEligibility.from_dict(data["exports"]), + ) + + def to_dict(self, *, include_package_hash: bool = True) -> dict[str, Any]: + payload: dict[str, Any] = { + "schema_version": self.schema_version, + "package_id": self.package_id, + "version": self.version, + "provenance": self.provenance.to_dict(), + "tasksets": [taskset.to_dict() for taskset in self.tasksets], + "splits": {split_id: split.to_dict() for split_id, split in self.splits.items()}, + "harness": self.harness.to_dict(), + "runtime": self.runtime.to_dict(), + "verifier": self.verifier.to_dict(), + "reward": self.reward.to_dict(), + "renderer": self.renderer.to_dict(), + "hardening": self.hardening.to_dict() if self.hardening else None, + "replay": self.replay.to_dict(), + "exports": self.exports.to_dict(), + } + if include_package_hash and self.package_hash: + payload["package_hash"] = self.package_hash + return payload diff --git a/breadboard/rl/env_package/validate.py b/breadboard/rl/env_package/validate.py new file mode 100644 index 00000000..e35191ac --- /dev/null +++ b/breadboard/rl/env_package/validate.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, Mapping + +import yaml + +from breadboard.rl.env_package.hash import canonical_env_package_hash +from breadboard.rl.env_package.schema import EnvPackage, SCHEMA_VERSION + + +PROTECTED_CONTAMINATION_SCOPES = { + "dev_hidden", + "promotion_hidden", + "final_holdout", + "external_board", +} +UNKNOWN_CONTAMINATION_SCOPES = {"unknown", ""} +SUPPORTED_EXPORT_LEVELS = {"experimental", "probe_backed", "supported"} +TRAINABILITY_STATUSES = {"not_trainable", "sft_candidate", "rl_candidate", "debug_only"} +SWE_SOURCE_KINDS = {"swe_gym", "swe_rebench_v2", "seta", "benchflow_swe"} +UNTRUSTED_ISOLATION_LEVELS = { + "single_tenant_untrusted", + "multi_tenant_untrusted", + "verifier_high_integrity", +} + + +def load_yaml_mapping(path: str | Path) -> dict[str, Any]: + payload = yaml.safe_load(Path(path).read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{path} must contain a YAML mapping") + return payload + + +def load_env_package(path: str | Path) -> EnvPackage: + return EnvPackage.from_dict(load_yaml_mapping(path)) + + +def _require_mapping(payload: Mapping[str, Any], field_name: str, errors: list[str]) -> dict[str, Any]: + value = payload.get(field_name) + if not isinstance(value, dict): + errors.append(f"{field_name} must be a mapping") + return {} + return dict(value) + + +def _require_list(payload: Mapping[str, Any], field_name: str, errors: list[str]) -> list[Any]: + value = payload.get(field_name) + if not isinstance(value, list) or not value: + errors.append(f"{field_name} must be a non-empty list") + return [] + return list(value) + + +def _text(value: Any) -> str: + return str(value or "").strip() + + +def _task_source_kinds(tasksets: list[Any]) -> set[str]: + kinds: set[str] = set() + for item in tasksets: + if isinstance(item, Mapping): + kinds.add(_text(item.get("source_kind")).lower()) + return kinds + + +def _requires_swe_hardening(payload: Mapping[str, Any], tasksets: list[Any]) -> bool: + package_id = _text(payload.get("package_id")).lower() + harness = payload.get("harness", {}) + interaction_mode = _text(harness.get("interaction_mode") if isinstance(harness, Mapping) else "").lower() + source_kinds = _task_source_kinds(tasksets) + return ( + "swe" in package_id + or bool(source_kinds & SWE_SOURCE_KINDS) + or interaction_mode in {"patch_submit", "swe_patch", "terminal_swe"} + ) + + +def validate_env_package_mapping(payload: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + + if payload.get("schema_version") != SCHEMA_VERSION: + errors.append(f"schema_version must be {SCHEMA_VERSION!r}") + for field in [ + "package_id", + "version", + "package_hash", + "provenance", + "tasksets", + "splits", + "harness", + "runtime", + "verifier", + "reward", + "renderer", + "replay", + "exports", + ]: + if field not in payload: + errors.append(f"missing required field: {field}") + + provenance = _require_mapping(payload, "provenance", errors) + tasksets = _require_list(payload, "tasksets", errors) + splits = _require_mapping(payload, "splits", errors) + harness = _require_mapping(payload, "harness", errors) + runtime = _require_mapping(payload, "runtime", errors) + verifier = _require_mapping(payload, "verifier", errors) + renderer = _require_mapping(payload, "renderer", errors) + replay = _require_mapping(payload, "replay", errors) + exports = _require_mapping(payload, "exports", errors) + + for field in ["created_at", "license", "source_usage_policy", "contamination_scope"]: + if not _text(provenance.get(field)): + errors.append(f"provenance.{field} must be non-empty") + + contamination_scope = _text(provenance.get("contamination_scope")).lower() + if exports.get("trainable") is True: + if contamination_scope in UNKNOWN_CONTAMINATION_SCOPES: + errors.append("trainable packages must not use unknown contamination_scope") + if not _text(provenance.get("license")): + errors.append("trainable packages require provenance.license") + if not _text(provenance.get("source_usage_policy")): + errors.append("trainable packages require provenance.source_usage_policy") + if replay.get("replay_required") is False: + errors.append("trainable packages require replay.replay_required") + if not _text(renderer.get("tokenizer_hash")): + errors.append("trainable packages require renderer.tokenizer_hash") + + if contamination_scope in PROTECTED_CONTAMINATION_SCOPES and exports.get("trainable") is True: + errors.append("protected contamination scopes cannot be marked trainable") + + for index, taskset in enumerate(tasksets): + if not isinstance(taskset, Mapping): + errors.append(f"tasksets[{index}] must be a mapping") + continue + for field in ["taskset_id", "source_kind", "source_hash", "task_id_field"]: + if not _text(taskset.get(field)): + errors.append(f"tasksets[{index}].{field} must be non-empty") + if not isinstance(taskset.get("prompt_fields"), list) or not taskset.get("prompt_fields"): + errors.append(f"tasksets[{index}].prompt_fields must be a non-empty list") + if not isinstance(taskset.get("allowed_splits"), list) or not taskset.get("allowed_splits"): + errors.append(f"tasksets[{index}].allowed_splits must be a non-empty list") + + allowed_splits_by_taskset: dict[str, set[str]] = {} + for item in tasksets: + if isinstance(item, Mapping) and _text(item.get("taskset_id")): + allowed_splits_by_taskset[_text(item.get("taskset_id"))] = { + _text(split_id) for split_id in item.get("allowed_splits", []) if _text(split_id) + } + for split_id, split in splits.items(): + if not isinstance(split, Mapping): + errors.append(f"splits.{split_id} must be a mapping") + continue + split_id_text = _text(split.get("split_id") or split_id) + taskset_id = _text(split.get("taskset_id")) + if split.get("protected") is True: + if split.get("trainer_visible") is True: + errors.append(f"splits.{split_id} protected split cannot be trainer_visible") + if split.get("optimizer_visible") is True: + errors.append(f"splits.{split_id} protected split cannot be optimizer_visible") + if taskset_id not in allowed_splits_by_taskset: + errors.append(f"splits.{split_id} references unknown taskset_id") + elif split_id_text not in allowed_splits_by_taskset[taskset_id]: + errors.append(f"splits.{split_id} is not listed in taskset allowed_splits") + if not _text(split.get("split_hash")): + errors.append(f"splits.{split_id}.split_hash must be non-empty") + + for field in ["harness_id", "interaction_mode"]: + if not _text(harness.get(field)): + errors.append(f"harness.{field} must be non-empty") + for field in ["backend", "isolation_level"]: + if not _text(runtime.get(field)): + errors.append(f"runtime.{field} must be non-empty") + if runtime.get("network") == "full" and not _text(runtime.get("network_allowlist_reason")): + errors.append("runtime.network=full requires network_allowlist_reason") + + hardening = payload.get("hardening") + needs_hardening = _requires_swe_hardening(payload, tasksets) + if needs_hardening and not isinstance(hardening, Mapping): + errors.append("SWE packages require hardening policy") + if isinstance(hardening, Mapping): + if hardening.get("agent_non_root_required") is True: + if _text(runtime.get("agent_user")).lower() == "root": + errors.append("hardening.agent_non_root_required forbids runtime.agent_user=root") + if hardening.get("verifier_isolated_required") is True and verifier.get("isolated_from_agent") is not True: + errors.append("hardening.verifier_isolated_required requires verifier.isolated_from_agent=true") + elif _text(runtime.get("isolation_level")) in UNTRUSTED_ISOLATION_LEVELS: + errors.append("untrusted runtime packages require hardening policy") + + for field in ["verifier_id", "kind", "code_hash"]: + if not _text(verifier.get(field)): + errors.append(f"verifier.{field} must be non-empty") + for field in ["renderer_id", "tokenizer_hash", "chat_template_hash"]: + if not _text(renderer.get(field)): + errors.append(f"renderer.{field} must be non-empty") + + support_level = _text(exports.get("support_level") or "experimental") + if support_level not in SUPPORTED_EXPORT_LEVELS: + errors.append(f"exports.support_level must be one of {sorted(SUPPORTED_EXPORT_LEVELS)}") + if support_level == "supported" and not exports.get("support_evidence_refs"): + errors.append("exports.support_level=supported requires support_evidence_refs") + trainability_status = _text(exports.get("trainability_status") or "not_trainable") + if trainability_status not in TRAINABILITY_STATUSES: + errors.append(f"exports.trainability_status must be one of {sorted(TRAINABILITY_STATUSES)}") + if exports.get("trainable") is False and trainability_status in {"sft_candidate", "rl_candidate"}: + errors.append("non-trainable exports cannot use trainable trainability_status") + if not isinstance(exports.get("allowed_formats"), list) or not exports.get("allowed_formats"): + errors.append("exports.allowed_formats must be a non-empty list") + + declared_hash = _text(payload.get("package_hash")) + if not declared_hash: + errors.append("package_hash must be non-empty") + else: + actual_hash = canonical_env_package_hash(payload) + if declared_hash != actual_hash: + errors.append("package_hash does not match canonical EnvPackage hash") + + return errors diff --git a/breadboard/rl/export/__init__.py b/breadboard/rl/export/__init__.py new file mode 100644 index 00000000..9b9b084e --- /dev/null +++ b/breadboard/rl/export/__init__.py @@ -0,0 +1,39 @@ +"""RL export projection helpers.""" + +from breadboard.rl.export.token_record import ( + build_token_record_export_payload, + validate_token_record_export_payload, +) +from breadboard.rl.export.projection import ProjectionManifest, build_projection_manifest +from breadboard.rl.export.schema import VERL_PROBE_SCHEMA, VerlProbeRow +from breadboard.rl.export.verl import ( + VERL_PROJECTION_MANIFEST_SCHEMA, + build_verl_probe_projection_manifest, + build_verl_probe_rows_from_m6_summary, + smoke_consume_verl_probe_jsonl, + smoke_consume_verl_probe_parquet, + validate_verl_probe_projection_manifest, + validate_verl_probe_row, + write_verl_probe_jsonl, + write_verl_probe_parquet, + write_verl_probe_projection_manifest, +) + +__all__ = [ + "ProjectionManifest", + "VERL_PROJECTION_MANIFEST_SCHEMA", + "VERL_PROBE_SCHEMA", + "VerlProbeRow", + "build_projection_manifest", + "build_verl_probe_projection_manifest", + "build_verl_probe_rows_from_m6_summary", + "smoke_consume_verl_probe_jsonl", + "smoke_consume_verl_probe_parquet", + "validate_verl_probe_projection_manifest", + "validate_verl_probe_row", + "build_token_record_export_payload", + "validate_token_record_export_payload", + "write_verl_probe_jsonl", + "write_verl_probe_parquet", + "write_verl_probe_projection_manifest", +] diff --git a/breadboard/rl/export/projection.py b/breadboard/rl/export/projection.py new file mode 100644 index 00000000..39ea2fd2 --- /dev/null +++ b/breadboard/rl/export/projection.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from breadboard.rl.trace.graph import TrajectoryGraph + + +@dataclass(frozen=True) +class ProjectionManifest: + projection_id: str + source_graph_id: str + target_format: str + preserved_fields: list[str] + lost_fields: list[str] = field(default_factory=list) + included_node_ids: list[str] = field(default_factory=list) + excluded_node_ids: list[str] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "projection_id": self.projection_id, + "source_graph_id": self.source_graph_id, + "target_format": self.target_format, + "preserved_fields": list(self.preserved_fields), + "lost_fields": list(self.lost_fields), + "included_node_ids": list(self.included_node_ids), + "excluded_node_ids": list(self.excluded_node_ids), + "metadata": dict(self.metadata), + } + + +def build_projection_manifest( + *, + graph: TrajectoryGraph, + target_format: str, + preserved_fields: list[str], + lost_fields: list[str] | None = None, + included_node_kinds: set[str] | None = None, +) -> ProjectionManifest: + included_kinds = included_node_kinds or {node.node_kind for node in graph.nodes} + included_node_ids = [node.node_id for node in graph.nodes if node.node_kind in included_kinds] + excluded_node_ids = [node.node_id for node in graph.nodes if node.node_kind not in included_kinds] + return ProjectionManifest( + projection_id=f"{graph.graph_id}.projection.{target_format}", + source_graph_id=graph.graph_id, + target_format=target_format, + preserved_fields=list(preserved_fields), + lost_fields=list(lost_fields or []), + included_node_ids=included_node_ids, + excluded_node_ids=excluded_node_ids, + metadata={"canonical_truth": "breadboard_graph_replay_runtime"}, + ) diff --git a/breadboard/rl/export/schema.py b/breadboard/rl/export/schema.py new file mode 100644 index 00000000..536accdd --- /dev/null +++ b/breadboard/rl/export/schema.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + + +VERL_PROBE_SCHEMA = "bb.verl_probe_row.v1alpha" + + +def _require_text(value: Any, field_name: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError(f"{field_name} must be non-empty") + return text + + +def _int_list(value: Any, field_name: str) -> list[int]: + if not isinstance(value, list): + raise ValueError(f"{field_name} must be a list") + return [int(item) for item in value] + + +def _bool_list(value: Any, field_name: str) -> list[bool]: + if not isinstance(value, list): + raise ValueError(f"{field_name} must be a list") + return [bool(item) for item in value] + + +def _float_list(value: Any, field_name: str) -> list[float] | None: + if value is None: + return None + if not isinstance(value, list): + raise ValueError(f"{field_name} must be a list") + return [float(item) for item in value] + + +def _mapping(value: Any, field_name: str) -> dict[str, Any]: + if value is None: + return {} + if not isinstance(value, Mapping): + raise ValueError(f"{field_name} must be a mapping") + return dict(value) + + +@dataclass(frozen=True) +class VerlProbeRow: + rollout_id: str + trajectory_id: str + episode_id: str + task_id: str + split_id: str + env_package_id: str + env_package_hash: str + group_id: str + policy: dict[str, Any] + prompt_ids: list[int] + completion_ids: list[int] + input_ids: list[int] + attention_mask: list[int] + loss_mask: list[bool] + assistant_mask: list[bool] + tool_action_mask: list[bool] + reward_mask: list[bool] + completion_logprobs: list[float] | None + completion_logprob_status: str + renderer: dict[str, Any] + reward: dict[str, Any] + runtime: dict[str, Any] + admission: dict[str, Any] + projection_manifest_id: str + trainable_candidate: bool = False + schema_version: str = VERL_PROBE_SCHEMA + claim_boundary: str = "verl_shaped_probe_not_trainer_ready" + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + payload = { + "schema_version": self.schema_version, + "rollout_id": self.rollout_id, + "trajectory_id": self.trajectory_id, + "episode_id": self.episode_id, + "task_id": self.task_id, + "split_id": self.split_id, + "env_package_id": self.env_package_id, + "env_package_hash": self.env_package_hash, + "group_id": self.group_id, + "policy": dict(self.policy), + "prompt_ids": list(self.prompt_ids), + "completion_ids": list(self.completion_ids), + "input_ids": list(self.input_ids), + "attention_mask": list(self.attention_mask), + "loss_mask": list(self.loss_mask), + "assistant_mask": list(self.assistant_mask), + "tool_action_mask": list(self.tool_action_mask), + "reward_mask": list(self.reward_mask), + "completion_logprob_status": self.completion_logprob_status, + "renderer": dict(self.renderer), + "reward": dict(self.reward), + "runtime": dict(self.runtime), + "admission": dict(self.admission), + "projection_manifest_id": self.projection_manifest_id, + "trainable_candidate": self.trainable_candidate, + "claim_boundary": self.claim_boundary, + "metadata": dict(self.metadata), + } + if self.completion_logprobs is not None: + payload["completion_logprobs"] = list(self.completion_logprobs) + return payload + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "VerlProbeRow": + return VerlProbeRow( + schema_version=data.get("schema_version") or VERL_PROBE_SCHEMA, + rollout_id=_require_text(data.get("rollout_id"), "rollout_id"), + trajectory_id=_require_text(data.get("trajectory_id"), "trajectory_id"), + episode_id=_require_text(data.get("episode_id"), "episode_id"), + task_id=_require_text(data.get("task_id"), "task_id"), + split_id=_require_text(data.get("split_id"), "split_id"), + env_package_id=_require_text(data.get("env_package_id"), "env_package_id"), + env_package_hash=_require_text(data.get("env_package_hash"), "env_package_hash"), + group_id=_require_text(data.get("group_id"), "group_id"), + policy=_mapping(data.get("policy"), "policy"), + prompt_ids=_int_list(data.get("prompt_ids"), "prompt_ids"), + completion_ids=_int_list(data.get("completion_ids"), "completion_ids"), + input_ids=_int_list(data.get("input_ids"), "input_ids"), + attention_mask=_int_list(data.get("attention_mask"), "attention_mask"), + loss_mask=_bool_list(data.get("loss_mask"), "loss_mask"), + assistant_mask=_bool_list(data.get("assistant_mask"), "assistant_mask"), + tool_action_mask=_bool_list(data.get("tool_action_mask"), "tool_action_mask"), + reward_mask=_bool_list(data.get("reward_mask"), "reward_mask"), + completion_logprobs=_float_list(data.get("completion_logprobs"), "completion_logprobs"), + completion_logprob_status=str(data.get("completion_logprob_status") or ""), + renderer=_mapping(data.get("renderer"), "renderer"), + reward=_mapping(data.get("reward"), "reward"), + runtime=_mapping(data.get("runtime"), "runtime"), + admission=_mapping(data.get("admission"), "admission"), + projection_manifest_id=_require_text(data.get("projection_manifest_id"), "projection_manifest_id"), + trainable_candidate=bool(data.get("trainable_candidate")), + claim_boundary=str(data.get("claim_boundary") or "verl_shaped_probe_not_trainer_ready"), + metadata=_mapping(data.get("metadata"), "metadata"), + ) diff --git a/breadboard/rl/export/token_record.py b/breadboard/rl/export/token_record.py new file mode 100644 index 00000000..b473ea00 --- /dev/null +++ b/breadboard/rl/export/token_record.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import Any, Mapping + +from breadboard.rl.renderer.records import classify_rendered_turn_trainability, validate_rendered_turn +from breadboard.rl.renderer.schema import RenderedTurnRecord + + +TOKEN_RECORD_EXPORT_SCHEMA = "bb.token_record.v1alpha" + + +def build_token_record_export_payload(record: RenderedTurnRecord) -> dict[str, Any]: + decision = classify_rendered_turn_trainability(record) + return { + "schema_version": TOKEN_RECORD_EXPORT_SCHEMA, + "record": record.to_dict(), + "trainability": decision.to_dict(), + "projection_boundary": { + "canonical_truth": "breadboard_graph_replay_runtime", + "projection_kind": "token_record", + "trainer_specific": False, + "verl_support_claim": False, + }, + } + + +def validate_token_record_export_payload(payload: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if payload.get("schema_version") != TOKEN_RECORD_EXPORT_SCHEMA: + errors.append(f"schema_version must be {TOKEN_RECORD_EXPORT_SCHEMA!r}") + record_payload = payload.get("record") + if not isinstance(record_payload, Mapping): + errors.append("record must be a mapping") + return errors + try: + record = RenderedTurnRecord.from_dict(record_payload) + except ValueError as exc: + return [*errors, str(exc)] + errors.extend(validate_rendered_turn(record)) + boundary = payload.get("projection_boundary") + if not isinstance(boundary, Mapping): + errors.append("projection_boundary must be a mapping") + else: + if boundary.get("canonical_truth") != "breadboard_graph_replay_runtime": + errors.append("projection_boundary.canonical_truth must remain breadboard_graph_replay_runtime") + if boundary.get("trainer_specific") is not False: + errors.append("projection_boundary.trainer_specific must be false") + if boundary.get("verl_support_claim") is not False: + errors.append("projection_boundary.verl_support_claim must be false for M2") + return errors diff --git a/breadboard/rl/export/verl.py b/breadboard/rl/export/verl.py new file mode 100644 index 00000000..5b83ddba --- /dev/null +++ b/breadboard/rl/export/verl.py @@ -0,0 +1,375 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Iterable, Mapping + +from breadboard.rl.export.schema import VERL_PROBE_SCHEMA, VerlProbeRow + + +TOKEN_ALIGNED_FIELDS = [ + "attention_mask", + "loss_mask", + "assistant_mask", + "tool_action_mask", + "reward_mask", +] + +VERL_PROJECTION_MANIFEST_SCHEMA = "bb.verl_probe_projection_manifest.v1alpha" + +VERL_PRESERVED_FIELDS = [ + "rollout_id", + "trajectory_id", + "episode_id", + "task_id", + "split_id", + "env_package_id", + "env_package_hash", + "group_id", + "policy", + "prompt_ids", + "completion_ids", + "input_ids", + "attention_mask", + "loss_mask", + "assistant_mask", + "tool_action_mask", + "reward_mask", + "completion_logprob_status", + "completion_logprobs", + "renderer", + "reward", + "runtime", + "admission", + "projection_manifest_id", + "trainable_candidate", +] + +VERL_LOST_FIELDS = [ + "full_workspace_bytes", + "full_runtime_process_tree", + "raw_filesystem_snapshot_bytes", + "trainer_dataproto_object", +] + +REQUIRED_POLICY_FIELDS = [ + "policy_id", + "policy_version", + "checkpoint_ref", + "model_requested", + "model_served", + "provider", + "engine", + "sampling_config", + "policy_staleness", +] + +REQUIRED_RENDERER_FIELDS = [ + "renderer_id", + "renderer_version", + "renderer_config_hash", + "tokenizer_hash", + "chat_template_hash", + "stop_ids", + "fidelity_class", +] + +REQUIRED_REWARD_FIELDS = [ + "scalar", + "reward_vector", + "verifier_id", + "verifier_version", + "verifier_hash", + "evidence_ref", +] + +REQUIRED_RUNTIME_FIELDS = [ + "runtime_backend", + "runtime_signature", + "image_digest", + "state_refs", + "artifact_refs", + "package_hash", + "metrics_ms", +] + +REQUIRED_ADMISSION_FIELDS = [ + "row_status", + "hardening_status", + "replay_status", + "quarantine_status", + "trainable", + "eligible_exports", + "exportable_debug", + "blocked_reasons", +] + + +def _require_mapping_fields(mapping: Mapping[str, Any], mapping_name: str, fields: list[str]) -> list[str]: + errors: list[str] = [] + for field_name in fields: + if field_name not in mapping: + errors.append(f"{mapping_name}.{field_name} must be present") + continue + value = mapping[field_name] + if value is None or (isinstance(value, str) and not value.strip()): + errors.append(f"{mapping_name}.{field_name} must be non-empty") + return errors + + +def validate_verl_probe_row(row: VerlProbeRow) -> list[str]: + errors: list[str] = [] + if row.schema_version != VERL_PROBE_SCHEMA: + errors.append(f"schema_version must be {VERL_PROBE_SCHEMA!r}") + if row.input_ids != [*row.prompt_ids, *row.completion_ids]: + errors.append("input_ids must equal prompt_ids + completion_ids") + if not row.completion_ids: + errors.append("completion_ids must be non-empty") + for field_name in TOKEN_ALIGNED_FIELDS: + if len(getattr(row, field_name)) != len(row.input_ids): + errors.append(f"{field_name} length must equal input_ids length") + if row.completion_logprobs is not None and len(row.completion_logprobs) != len(row.completion_ids): + errors.append("completion_logprobs length must equal completion_ids length") + if row.completion_logprobs is None and row.completion_logprob_status not in {"unavailable_non_trainable", "posthoc_unavailable"}: + errors.append("missing completion_logprobs requires explicit unavailable status") + if row.completion_logprobs is not None and row.completion_logprob_status not in {"native_available", "posthoc_available"}: + errors.append("completion_logprobs require available logprob status") + if row.trainable_candidate and row.completion_logprobs is None: + errors.append("trainable_candidate requires completion_logprobs") + if row.trainable_candidate and row.completion_logprob_status != "native_available": + errors.append("trainable_candidate requires native_available completion_logprob_status") + if row.trainable_candidate and row.admission.get("row_status") != "accepted": + errors.append("trainable_candidate requires accepted row_status") + if row.trainable_candidate and row.admission.get("hardening_status") != "passed": + errors.append("trainable_candidate requires hardening_status=passed") + if row.trainable_candidate and row.admission.get("replay_status") != "passed": + errors.append("trainable_candidate requires replay_status=passed") + if row.claim_boundary != "verl_shaped_probe_not_trainer_ready": + errors.append("claim_boundary must remain verl_shaped_probe_not_trainer_ready") + for field_name in ["policy", "renderer", "reward", "runtime", "admission"]: + if not getattr(row, field_name): + errors.append(f"{field_name} must be non-empty") + errors.extend(_require_mapping_fields(row.policy, "policy", REQUIRED_POLICY_FIELDS)) + errors.extend(_require_mapping_fields(row.renderer, "renderer", REQUIRED_RENDERER_FIELDS)) + errors.extend(_require_mapping_fields(row.reward, "reward", REQUIRED_REWARD_FIELDS)) + errors.extend(_require_mapping_fields(row.runtime, "runtime", REQUIRED_RUNTIME_FIELDS)) + errors.extend(_require_mapping_fields(row.admission, "admission", REQUIRED_ADMISSION_FIELDS)) + if row.policy.get("policy_staleness") and not isinstance(row.policy.get("policy_staleness"), Mapping): + errors.append("policy.policy_staleness must be a mapping") + if row.renderer.get("stop_ids") is not None and not isinstance(row.renderer.get("stop_ids"), list): + errors.append("renderer.stop_ids must be a list") + if row.runtime.get("state_refs") is not None and not isinstance(row.runtime.get("state_refs"), list): + errors.append("runtime.state_refs must be a list") + if row.runtime.get("artifact_refs") is not None and not isinstance(row.runtime.get("artifact_refs"), list): + errors.append("runtime.artifact_refs must be a list") + if row.admission.get("eligible_exports") is not None and not isinstance(row.admission.get("eligible_exports"), list): + errors.append("admission.eligible_exports must be a list") + return errors + + +def build_verl_probe_rows_from_m6_summary(summary: Mapping[str, Any]) -> list[VerlProbeRow]: + rows: list[VerlProbeRow] = [] + for index, source_row in enumerate(summary.get("rows") or [], start=1): + row_status = source_row["row_status"] + prompt_ids = [100, index] + completion_ids = [200 + index, 300 + index] + input_ids = [*prompt_ids, *completion_ids] + accepted = row_status == "accepted" + rows.append( + VerlProbeRow( + rollout_id=summary["run_id"], + trajectory_id=f"{summary['run_id']}.{source_row['task_id']}.trajectory", + episode_id=f"{summary['run_id']}.{source_row['task_id']}.episode", + task_id=source_row["task_id"], + split_id="train_probe", + env_package_id=summary["package_id"], + env_package_hash=summary["package_hash"], + group_id=f"{summary['run_id']}.group.controlled_swe_toy", + policy={ + "policy_id": "m7_probe_policy", + "policy_version": "v1alpha", + "checkpoint_ref": "none_probe", + "model_requested": "synthetic_token_probe", + "model_served": "synthetic_token_probe", + "provider": "local_probe", + "engine": "synthetic_token_probe", + "sampling_config": {"temperature": 1.0}, + "policy_staleness": { + "policy_age_steps": 0, + "actor_version_matches_rollout": True, + "staleness_status": "not_stale_offline_probe", + }, + }, + prompt_ids=prompt_ids, + completion_ids=completion_ids, + input_ids=input_ids, + attention_mask=[1] * len(input_ids), + loss_mask=[False, False, True, True], + assistant_mask=[False, False, True, True], + tool_action_mask=[False, False, True, False], + reward_mask=[False, False, False, True], + completion_logprobs=[-0.1, -0.2] if accepted else None, + completion_logprob_status="native_available" if accepted else "unavailable_non_trainable", + renderer={ + "renderer_id": "m7_synthetic_renderer", + "renderer_version": "v1alpha", + "renderer_config_hash": "sha256:m7-renderer", + "tokenizer_hash": "sha256:m7-tokenizer", + "chat_template_hash": "sha256:m7-chat-template", + "stop_ids": [], + "fidelity_class": "F3" if accepted else "F1", + }, + reward={ + "scalar": source_row["reward"], + "reward_vector": {"unit_test_reward": source_row["reward"]}, + "verifier_id": "swe_toy_patch_pytest", + "verifier_version": "v1alpha", + "verifier_hash": "sha256:swe-toy-patch-pytest", + "evidence_ref": f"row_evidence/{source_row['task_id']}.json", + }, + runtime={ + "runtime_backend": "controlled_swe_toy_probe", + "runtime_signature": "controlled_swe_toy_probe.v1alpha", + "image_digest": "local_process_no_container_probe", + "state_refs": [f"state://{summary['run_id']}/{source_row['task_id']}"], + "artifact_refs": [f"row_evidence/{source_row['task_id']}.json"], + "package_hash": summary["package_hash"], + "metrics_ms": dict(source_row["metrics_ms"]), + }, + admission={ + "row_status": row_status, + "hardening_status": source_row["hardening_status"], + "replay_status": source_row["replay_status"], + "quarantine_status": "quarantined" if row_status == "quarantined" else "clear", + "trainable": accepted, + "eligible_exports": ["debug_jsonl", "verl_jsonl", "verl_parquet"] if accepted else ["debug_jsonl"], + "exportable_debug": source_row["exportable_debug"], + "blocked_reasons": list(source_row.get("blocked_reasons") or []), + }, + projection_manifest_id=source_row["projection_id"], + trainable_candidate=accepted, + metadata={"source_claim": summary.get("source_claim")}, + ) + ) + return rows + + +def write_verl_probe_jsonl(rows: Iterable[VerlProbeRow], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(row.to_dict(), sort_keys=True) + "\n" for row in rows), encoding="utf-8") + + +def build_verl_probe_projection_manifest(rows: Iterable[VerlProbeRow], target_formats: Iterable[str]) -> dict[str, Any]: + materialized_rows = list(rows) + rollout_id = materialized_rows[0].rollout_id if materialized_rows else "empty" + return { + "schema_version": VERL_PROJECTION_MANIFEST_SCHEMA, + "projection_id": f"{rollout_id}.projection.verl_probe_v1alpha", + "claim_boundary": "verl_shaped_probe_not_trainer_ready", + "canonical_truth": "breadboard_graph_replay_runtime", + "target_formats": list(target_formats), + "row_count": len(materialized_rows), + "trainable_candidate_count": sum(row.trainable_candidate for row in materialized_rows), + "source_projection_manifest_ids": sorted({row.projection_manifest_id for row in materialized_rows}), + "preserved_fields": list(VERL_PRESERVED_FIELDS), + "lost_fields": list(VERL_LOST_FIELDS), + "metadata": { + "compatibility_target": "VeRL JSONL/Parquet probe v1alpha; not DataProto or trainer execution", + }, + } + + +def validate_verl_probe_projection_manifest(manifest: Mapping[str, Any], rows: Iterable[VerlProbeRow]) -> list[str]: + materialized_rows = list(rows) + errors: list[str] = [] + if manifest.get("schema_version") != VERL_PROJECTION_MANIFEST_SCHEMA: + errors.append(f"schema_version must be {VERL_PROJECTION_MANIFEST_SCHEMA!r}") + if manifest.get("claim_boundary") != "verl_shaped_probe_not_trainer_ready": + errors.append("claim_boundary must remain verl_shaped_probe_not_trainer_ready") + if manifest.get("canonical_truth") != "breadboard_graph_replay_runtime": + errors.append("canonical_truth must be breadboard_graph_replay_runtime") + if int(manifest.get("row_count", -1)) != len(materialized_rows): + errors.append("row_count must match exported rows") + if int(manifest.get("trainable_candidate_count", -1)) != sum(row.trainable_candidate for row in materialized_rows): + errors.append("trainable_candidate_count must match exported rows") + target_formats = set(manifest.get("target_formats") or []) + if not {"jsonl", "parquet"}.issubset(target_formats): + errors.append("target_formats must include jsonl and parquet") + source_projection_ids = set(manifest.get("source_projection_manifest_ids") or []) + expected_projection_ids = {row.projection_manifest_id for row in materialized_rows} + if source_projection_ids != expected_projection_ids: + errors.append("source_projection_manifest_ids must match row projection_manifest_id values") + if not set(VERL_PRESERVED_FIELDS).issubset(set(manifest.get("preserved_fields") or [])): + errors.append("preserved_fields must include required VeRL probe fields") + if "trainer_dataproto_object" not in set(manifest.get("lost_fields") or []): + errors.append("lost_fields must record deferred trainer_dataproto_object") + return errors + + +def write_verl_probe_projection_manifest(rows: Iterable[VerlProbeRow], path: Path, target_formats: Iterable[str]) -> dict[str, Any]: + manifest = build_verl_probe_projection_manifest(rows, target_formats) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return manifest + + +def write_verl_probe_parquet(rows: Iterable[VerlProbeRow], path: Path) -> None: + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError as exc: # pragma: no cover - exercised only on missing optional dependency. + raise RuntimeError("pyarrow is required for VeRL Parquet probe export") from exc + + path.parent.mkdir(parents=True, exist_ok=True) + payloads = [row.to_dict() for row in rows] + table = pa.Table.from_pylist(payloads) + pq.write_table(table, path) + + +def _smoke_report(rows: list[VerlProbeRow], errors: list[dict[str, Any]], compatibility_target: str) -> dict[str, Any]: + return { + "row_count": len(rows), + "trainable_candidate_count": sum(row.trainable_candidate for row in rows), + "tensorizable": not errors, + "errors": errors, + "compatibility_target": compatibility_target, + } + + +def smoke_consume_verl_probe_jsonl(path: Path) -> dict[str, Any]: + rows = [] + errors: list[dict[str, Any]] = [] + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + row = VerlProbeRow.from_dict(json.loads(line)) + except Exception as exc: + errors.append({"line": line_number, "errors": [str(exc)]}) + continue + row_errors = validate_verl_probe_row(row) + if row_errors: + errors.append({"line": line_number, "task_id": row.task_id, "errors": row_errors}) + rows.append(row) + return _smoke_report(rows, errors, "VeRL JSONL probe v1alpha; not DataProto or trainer execution") + + +def smoke_consume_verl_probe_parquet(path: Path) -> dict[str, Any]: + try: + import pyarrow.parquet as pq + except ImportError as exc: # pragma: no cover - exercised only on missing optional dependency. + raise RuntimeError("pyarrow is required for VeRL Parquet probe smoke consumption") from exc + + rows = [] + errors: list[dict[str, Any]] = [] + for row_number, payload in enumerate(pq.read_table(path).to_pylist(), start=1): + try: + row = VerlProbeRow.from_dict(payload) + except Exception as exc: + errors.append({"row": row_number, "errors": [str(exc)]}) + continue + row_errors = validate_verl_probe_row(row) + if row_errors: + errors.append({"row": row_number, "task_id": row.task_id, "errors": row_errors}) + rows.append(row) + return _smoke_report(rows, errors, "VeRL Parquet probe v1alpha; not DataProto or trainer execution") diff --git a/breadboard/rl/m12/__init__.py b/breadboard/rl/m12/__init__.py new file mode 100644 index 00000000..224eda7f --- /dev/null +++ b/breadboard/rl/m12/__init__.py @@ -0,0 +1,97 @@ +"""M12 target-node transfer and preflight preparation helpers.""" + +from breadboard.rl.m12.bootstrap import ( + build_m12_bootstrap_dry_run_report, + validate_m12_bootstrap_dry_run_report, + write_m12_bootstrap_dry_run_report, +) +from breadboard.rl.m12.command_logs import ( + next_command_log_path, + record_command_log_result, + validate_command_id, + validate_target_run_id, + validate_command_log_manifest, +) +from breadboard.rl.m12.evidence_consistency import ( + build_m12_evidence_consistency_report, + validate_m12_evidence_consistency_report, + write_m12_evidence_consistency_report, +) +from breadboard.rl.m12.final_report import ( + build_m12_final_report, + summarize_m12_final_report_remediations, + validate_m12_final_report, + validate_m12_final_report_remediation_summary, + write_m12_final_report, +) +from breadboard.rl.m12.load_soak import ( + build_m12_load_ladder_report, + build_m12_soak_report, + validate_m12_load_ladder_report, + validate_m12_soak_report, +) +from breadboard.rl.m12.preflight import ( + run_m12_preflight, + validate_m12_preflight_report, + write_m12_preflight_report, +) +from breadboard.rl.m12.promotion_audit import ( + build_m12_promotion_audit, + validate_m12_promotion_audit, + write_m12_promotion_audit, +) +from breadboard.rl.m12.transfer import ( + apply_m12_transfer_overlay, + build_m12_readiness_summary, + build_m12_test_commands_script, + build_m12_transfer_manifest, + build_m12_transfer_summary, + validate_m12_readiness_summary, + validate_m12_test_commands_script, + validate_m12_transfer_archive_manifest, + validate_m12_transfer_overlay_report, + validate_m12_transfer_summary, + write_m12_transfer_archive, + write_m12_transfer_pack, +) + +__all__ = [ + "apply_m12_transfer_overlay", + "build_m12_bootstrap_dry_run_report", + "build_m12_final_report", + "build_m12_evidence_consistency_report", + "build_m12_load_ladder_report", + "build_m12_promotion_audit", + "build_m12_readiness_summary", + "build_m12_soak_report", + "build_m12_test_commands_script", + "build_m12_transfer_manifest", + "build_m12_transfer_summary", + "next_command_log_path", + "record_command_log_result", + "run_m12_preflight", + "summarize_m12_final_report_remediations", + "validate_command_id", + "validate_command_log_manifest", + "validate_target_run_id", + "validate_m12_evidence_consistency_report", + "validate_m12_bootstrap_dry_run_report", + "validate_m12_load_ladder_report", + "validate_m12_final_report", + "validate_m12_final_report_remediation_summary", + "validate_m12_preflight_report", + "validate_m12_promotion_audit", + "validate_m12_readiness_summary", + "validate_m12_soak_report", + "validate_m12_test_commands_script", + "validate_m12_transfer_archive_manifest", + "validate_m12_transfer_overlay_report", + "validate_m12_transfer_summary", + "write_m12_bootstrap_dry_run_report", + "write_m12_evidence_consistency_report", + "write_m12_final_report", + "write_m12_promotion_audit", + "write_m12_preflight_report", + "write_m12_transfer_archive", + "write_m12_transfer_pack", +] diff --git a/breadboard/rl/m12/bootstrap.py b/breadboard/rl/m12/bootstrap.py new file mode 100644 index 00000000..a2b5e75e --- /dev/null +++ b/breadboard/rl/m12/bootstrap.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import subprocess +from typing import Any + +from breadboard.rl.m12.transfer import validate_m12_transfer_overlay_report + + +BOOTSTRAP_DRY_RUN_ID = "bb_zyphra_rl_phase1_m12_bootstrap_dry_run_report_v1" +BOOTSTRAP_DRY_RUN_CLAIM_BOUNDARY = "target_bootstrap_dry_run_not_m12_validation" + + +def _sha256_text(value: str) -> str: + return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _sha256_file(path: Path) -> str | None: + if not path.is_file(): + return None + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def _read_json_if_present(path: Path) -> dict[str, Any] | None: + if not path.is_file(): + return None + return json.loads(path.read_text(encoding="utf-8")) + + +def _input_hash_errors(report: dict[str, Any], input_hashes: dict[str, Any]) -> list[str]: + errors: list[str] = [] + for field in ["bootstrap_script", "transfer_manifest", "archive_manifest", "overlay_dry_run_report"]: + recorded = str(input_hashes.get(field) or "") + raw_path = report.get(field) + if not recorded.startswith("sha256:"): + errors.append(f"input_hashes.{field} must start with sha256:") + continue + if not raw_path: + errors.append(f"{field} path must be recorded") + continue + actual = _sha256_file(Path(str(raw_path))) + if actual is None: + errors.append(f"{field} file missing for input hash validation") + elif actual != recorded: + errors.append(f"input_hashes.{field} does not match current file") + return errors + + +def _overlay_file_errors(report: dict[str, Any], overlay_summary: dict[str, Any]) -> list[str]: + errors: list[str] = [] + raw_path = report.get("overlay_dry_run_report") + if not raw_path: + return ["overlay_dry_run_report path must be recorded"] + path = Path(str(raw_path)) + if not path.is_file(): + return ["overlay_dry_run_report file missing for validation"] + try: + overlay_report = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + return [f"overlay_dry_run_report is not readable JSON: {type(exc).__name__}: {exc}"] + + errors.extend(f"overlay_dry_run_report.{error}" for error in validate_m12_transfer_overlay_report(overlay_report)) + for field in [ + "status", + "dry_run", + "would_write_count", + "written_count", + "existing_destination_count", + "scorecard_update_allowed", + "m12_points_awarded", + ]: + if overlay_summary.get(field) != overlay_report.get(field): + errors.append(f"overlay.{field} must match overlay_dry_run_report") + return errors + + +def build_m12_bootstrap_dry_run_report( + *, + repo_root: Path, + transfer_prep_dir: Path, + workspace_root: Path | None = None, +) -> dict[str, Any]: + repo_root = repo_root.resolve() + transfer_prep_dir = transfer_prep_dir.resolve() + workspace_root = (workspace_root or repo_root.parent).resolve() + bootstrap_script = transfer_prep_dir / "m12_target_bootstrap.sh" + transfer_manifest = transfer_prep_dir / "m12_transfer_manifest.json" + archive_manifest = transfer_prep_dir / "m12_transfer_archive_manifest.json" + overlay_report_path = transfer_prep_dir / "m12_overlay_apply_dry_run_report.json" + bootstrap_script_sha256 = _sha256_file(bootstrap_script) + transfer_manifest_sha256 = _sha256_file(transfer_manifest) + archive_manifest_sha256 = _sha256_file(archive_manifest) + + env = os.environ.copy() + env.update( + { + "REPO_ROOT": str(repo_root), + "WORKSPACE_ROOT": str(workspace_root), + "BOOTSTRAP_DRY_RUN_ONLY": "1", + "ALLOW_M12_DIRTY_CHECKOUT": "1", + } + ) + result = subprocess.run( + ["bash", str(bootstrap_script)], + cwd=repo_root, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + overlay_report = _read_json_if_present(overlay_report_path) + overlay_report_sha256 = _sha256_file(overlay_report_path) + stdout = result.stdout + stderr = result.stderr + repo_head_verified = "repo_head_verified=true" in stdout + dirty_checkout_check_observed = "repo_dirty_check=" in stdout + dirty_checkout_mode = "override" if "repo_dirty_check=override" in stdout else "clean" if "repo_dirty_check=clean" in stdout else None + dirty_checkout_override_used = dirty_checkout_mode == "override" + target_commands_skipped = "bootstrap_dry_run_only=true" in stdout + errors: list[str] = [] + if result.returncode != 0: + errors.append(f"bootstrap exited nonzero: {result.returncode}") + if not repo_head_verified: + errors.append("bootstrap stdout did not confirm repo_head_verified=true") + if not dirty_checkout_check_observed: + errors.append("bootstrap stdout did not confirm repo_dirty_check") + if not target_commands_skipped: + errors.append("bootstrap stdout did not confirm bootstrap_dry_run_only=true") + if overlay_report is None: + errors.append("overlay dry-run report was not written") + else: + overlay_validation_errors = validate_m12_transfer_overlay_report(overlay_report) + errors.extend(f"overlay dry-run report invalid: {error}" for error in overlay_validation_errors) + if overlay_report_sha256 is None: + errors.append("overlay dry-run report sha256 was not recorded") + if overlay_report.get("status") != "passed": + errors.append("overlay dry-run report status was not passed") + if overlay_report.get("dry_run") is not True: + errors.append("overlay dry-run report did not record dry_run=true") + if overlay_report.get("written_count") != 0: + errors.append("overlay dry-run report wrote files") + if overlay_report.get("scorecard_update_allowed") is not False: + errors.append("overlay dry-run report allowed scorecard update") + if overlay_report.get("m12_points_awarded") is not False: + errors.append("overlay dry-run report awarded M12 points") + + return { + "report_id": BOOTSTRAP_DRY_RUN_ID, + "claim_boundary": BOOTSTRAP_DRY_RUN_CLAIM_BOUNDARY, + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "status": "passed" if not errors else "failed", + "errors": errors, + "repo_root": str(repo_root), + "workspace_root": str(workspace_root), + "transfer_prep_dir": str(transfer_prep_dir), + "bootstrap_script": str(bootstrap_script), + "transfer_manifest": str(transfer_manifest), + "archive_manifest": str(archive_manifest), + "overlay_dry_run_report": str(overlay_report_path), + "input_hashes": { + "bootstrap_script": bootstrap_script_sha256, + "transfer_manifest": transfer_manifest_sha256, + "archive_manifest": archive_manifest_sha256, + "overlay_dry_run_report": overlay_report_sha256, + }, + "exit_code": result.returncode, + "repo_head_verified": repo_head_verified, + "dirty_checkout_check_observed": dirty_checkout_check_observed, + "dirty_checkout_mode": dirty_checkout_mode, + "dirty_checkout_override_used": dirty_checkout_override_used, + "target_commands_skipped": target_commands_skipped, + "stdout_sha256": _sha256_text(stdout), + "stderr_sha256": _sha256_text(stderr), + "stdout": stdout, + "stderr": stderr, + "overlay": { + "status": None if overlay_report is None else overlay_report.get("status"), + "dry_run": None if overlay_report is None else overlay_report.get("dry_run"), + "would_write_count": None if overlay_report is None else overlay_report.get("would_write_count"), + "written_count": None if overlay_report is None else overlay_report.get("written_count"), + "existing_destination_count": None + if overlay_report is None + else overlay_report.get("existing_destination_count"), + "scorecard_update_allowed": None + if overlay_report is None + else overlay_report.get("scorecard_update_allowed"), + "m12_points_awarded": None if overlay_report is None else overlay_report.get("m12_points_awarded"), + }, + } + + +def validate_m12_bootstrap_dry_run_report(report: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != BOOTSTRAP_DRY_RUN_ID: + errors.append("report_id must be bb_zyphra_rl_phase1_m12_bootstrap_dry_run_report_v1") + if report.get("claim_boundary") != BOOTSTRAP_DRY_RUN_CLAIM_BOUNDARY: + errors.append("claim_boundary must remain target_bootstrap_dry_run_not_m12_validation") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if report.get("m12_points_awarded") is not False: + errors.append("m12_points_awarded must be false") + if report.get("status") not in {"passed", "failed"}: + errors.append("status must be passed or failed") + if report.get("status") == "passed" and report.get("errors") != []: + errors.append("passed report must have no errors") + if report.get("exit_code") != 0 and report.get("status") == "passed": + errors.append("passed report must have exit_code=0") + if report.get("repo_head_verified") is not True and report.get("status") == "passed": + errors.append("passed report must verify repo head") + if report.get("dirty_checkout_check_observed") is not True and report.get("status") == "passed": + errors.append("passed report must observe dirty checkout check") + if report.get("dirty_checkout_mode") not in {"clean", "override"} and report.get("status") == "passed": + errors.append("passed report must record dirty_checkout_mode clean or override") + if report.get("target_commands_skipped") is not True and report.get("status") == "passed": + errors.append("passed report must skip target commands") + input_hashes = report.get("input_hashes") + if not isinstance(input_hashes, dict): + errors.append("input_hashes must be an object") + input_hashes = {} + errors.extend(_input_hash_errors(report, input_hashes)) + overlay = report.get("overlay") + if not isinstance(overlay, dict): + errors.append("overlay must be an object") + overlay = {} + else: + errors.extend(_overlay_file_errors(report, overlay)) + if report.get("status") == "passed": + if overlay.get("status") != "passed": + errors.append("passed report requires overlay.status=passed") + if overlay.get("dry_run") is not True: + errors.append("passed report requires overlay.dry_run=true") + if overlay.get("written_count") != 0: + errors.append("passed report requires overlay.written_count=0") + if overlay.get("scorecard_update_allowed") is not False: + errors.append("passed report requires overlay.scorecard_update_allowed=false") + if overlay.get("m12_points_awarded") is not False: + errors.append("passed report requires overlay.m12_points_awarded=false") + for field in ["stdout_sha256", "stderr_sha256"]: + if not str(report.get(field) or "").startswith("sha256:"): + errors.append(f"{field} must start with sha256:") + return errors + + +def write_m12_bootstrap_dry_run_report( + *, + repo_root: Path, + transfer_prep_dir: Path, + output_path: Path, + workspace_root: Path | None = None, +) -> dict[str, Any]: + report = build_m12_bootstrap_dry_run_report( + repo_root=repo_root, + transfer_prep_dir=transfer_prep_dir, + workspace_root=workspace_root, + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report diff --git a/breadboard/rl/m12/command_logs.py b/breadboard/rl/m12/command_logs.py new file mode 100644 index 00000000..fcfa0a60 --- /dev/null +++ b/breadboard/rl/m12/command_logs.py @@ -0,0 +1,656 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import re +import shlex +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from breadboard.rl.m12.final_report import ( + COMMAND_LOG_MANIFEST_ID, + OPTIONAL_COMMAND_LOG_COMMANDS, + REQUIRED_COMMAND_LOG_COMMANDS, + REQUIRED_COMMAND_LOG_IDS, +) + + +COMMAND_ID_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+$") +TARGET_RUN_ID_PATTERN = re.compile(r"^[A-Za-z0-9_.:-]+$") +LOG_HEADER_KEYS = { + "command_id", + "target_run_id", + "command", + "argv_json", + "started_at", + "completed_at", + "exit_code", +} +COMMAND_LOG_CLAIM_BOUNDARY = "target_command_logs_not_scorecard_update" + +COMMAND_LOG_ENTRY_TEMPLATES = [ + { + "command_id": "target_transfer_archive_verify", + "required": True, + "description": "Target-side transfer archive verification before preflight.", + "command": REQUIRED_COMMAND_LOG_COMMANDS["target_transfer_archive_verify"], + }, + { + "command_id": "phase1_validation_suite", + "required": True, + "description": "Full RL Phase 1 validation suite on the target checkout before target artifacts mutate local blocked-state fixtures.", + "command": REQUIRED_COMMAND_LOG_COMMANDS["phase1_validation_suite"], + }, + { + "command_id": "target_preflight", + "required": True, + "description": "Target preflight with --require-pass.", + "command": REQUIRED_COMMAND_LOG_COMMANDS["target_preflight"], + }, + { + "command_id": "target_swe_probe", + "required": True, + "description": "Target SWE probe run.", + "command": REQUIRED_COMMAND_LOG_COMMANDS["target_swe_probe"], + }, + { + "command_id": "target_verl_export", + "required": True, + "description": "Target VeRL JSONL/Parquet projection smoke.", + "command": REQUIRED_COMMAND_LOG_COMMANDS["target_verl_export"], + }, + { + "command_id": "target_ray_warm_pool", + "required": True, + "description": "Target Ray/warm-pool probe.", + "command": REQUIRED_COMMAND_LOG_COMMANDS["target_ray_warm_pool"], + }, + { + "command_id": "target_load_ladder", + "required": True, + "description": "Target load ladder producing m12_node_load_ladder/load_ladder_report.json.", + "command": REQUIRED_COMMAND_LOG_COMMANDS["target_load_ladder"], + }, + { + "command_id": "target_soak", + "required": True, + "description": "Target soak producing m12_node_soak/soak_report.json.", + "command": REQUIRED_COMMAND_LOG_COMMANDS["target_soak"], + }, + { + "command_id": "final_report", + "required": False, + "description": "Final report builder with --require-eligible; logged after required command-log gates are satisfied.", + "command": OPTIONAL_COMMAND_LOG_COMMANDS["final_report"], + }, + { + "command_id": "promotion_audit", + "required": False, + "description": "Non-scoring promotion audit; logged after the eligible final report command.", + "command": OPTIONAL_COMMAND_LOG_COMMANDS["promotion_audit"], + }, +] + +COMMAND_LOG_MANIFEST_TEMPLATE = { + "manifest_id": COMMAND_LOG_MANIFEST_ID, + "claim_boundary": COMMAND_LOG_CLAIM_BOUNDARY, + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "all_required_logs_archived": False, + "all_required_commands_passed": False, + "target_run_ids": [], + "latest_target_run_id": None, + "commands": [ + { + **entry, + "status": "pending", + "exit_code": None, + "log_path": None, + "sha256": None, + "started_at": None, + "completed_at": None, + "notes": "", + } + for entry in COMMAND_LOG_ENTRY_TEMPLATES + ], + "required_command_ids": list(REQUIRED_COMMAND_LOG_IDS), + "operator_notes": "", +} + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def validate_command_id(command_id: str) -> list[str]: + errors: list[str] = [] + if not command_id: + errors.append("command_id must be non-empty") + if not COMMAND_ID_PATTERN.fullmatch(command_id): + errors.append("command_id must contain only letters, numbers, dot, underscore, and hyphen") + return errors + + +def validate_target_run_id(target_run_id: str) -> list[str]: + errors: list[str] = [] + if not target_run_id: + errors.append("target_run_id must be non-empty") + if not TARGET_RUN_ID_PATTERN.fullmatch(target_run_id): + errors.append("target_run_id must contain only letters, numbers, dot, underscore, colon, and hyphen") + return errors + + +def next_command_log_path(log_dir: Path, command_id: str) -> Path: + errors = validate_command_id(command_id) + if errors: + raise ValueError("; ".join(errors)) + base_path = log_dir / f"{command_id}.log" + if not base_path.exists(): + return base_path + attempt = 2 + while True: + candidate = log_dir / f"{command_id}.attempt-{attempt:03d}.log" + if not candidate.exists(): + return candidate + attempt += 1 + + +def init_command_log_manifest() -> dict[str, Any]: + return copy.deepcopy(COMMAND_LOG_MANIFEST_TEMPLATE) + + +def read_command_log_manifest(path: Path) -> dict[str, Any]: + if not path.exists(): + return init_command_log_manifest() + return json.loads(path.read_text(encoding="utf-8")) + + +def write_command_log_manifest(path: Path, manifest: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def manifest_relative_log_path(manifest_path: Path, log_path: Path) -> str: + try: + return str(log_path.resolve().relative_to(manifest_path.parent.resolve())) + except ValueError: + return str(log_path) + + +def resolve_manifest_log_path(manifest_path: Path, raw_log_path: str) -> Path: + path = Path(raw_log_path) + if path.is_absolute(): + return path + manifest_relative = manifest_path.parent / path + if manifest_relative.exists(): + return manifest_relative + return path + + +def validate_manifest_log_path(raw_log_path: str) -> list[str]: + errors: list[str] = [] + if not raw_log_path: + errors.append("log_path must be non-empty") + return errors + path = Path(raw_log_path) + if path.is_absolute(): + errors.append("log_path must be relative to the command-log manifest directory") + if any(part == ".." for part in path.parts): + errors.append("log_path must not contain parent-directory traversal") + return errors + + +def normalize_command_argv(command: str, argv: list[str] | None = None) -> list[str]: + if argv is not None: + return [str(part) for part in argv] + try: + parsed = shlex.split(command) + except ValueError: + parsed = [] + return parsed or [command] + + +def validate_command_argv(raw_argv: Any) -> list[str]: + errors: list[str] = [] + if not isinstance(raw_argv, list) or not raw_argv: + return ["argv must be a non-empty list"] + if any(not isinstance(item, str) or item == "" for item in raw_argv): + errors.append("argv entries must be non-empty strings") + return errors + + +def validate_command_argv_congruence(raw_command: Any, raw_argv: Any) -> list[str]: + command = str(raw_command or "") + if not command: + return ["command must be non-empty"] + argv_errors = validate_command_argv(raw_argv) + if argv_errors: + return [] + if shlex.join([str(part) for part in raw_argv]) != command: + return ["command must equal shlex.join(argv)"] + return [] + + +def validate_status_exit_code_congruence(raw_status: Any, raw_exit_code: Any) -> list[str]: + status = str(raw_status or "") + exit_code_is_int = isinstance(raw_exit_code, int) and not isinstance(raw_exit_code, bool) + errors: list[str] = [] + if status not in {"pending", "passed", "failed"}: + return ["status must be pending, passed, or failed"] + if status == "pending": + if raw_exit_code is not None: + errors.append("pending commands must not have exit_code") + return errors + if not exit_code_is_int: + return ["completed commands must have integer exit_code"] + if status == "passed" and raw_exit_code != 0: + errors.append("passed commands must have exit_code 0") + if status == "failed" and raw_exit_code == 0: + errors.append("failed commands must have nonzero exit_code") + return errors + + +def collect_command_log_header_metadata(log_path: Path) -> tuple[dict[str, str], dict[str, int]]: + metadata: dict[str, str] = {} + counts: dict[str, int] = {} + with log_path.open("r", encoding="utf-8", errors="replace") as handle: + for line in handle: + if not line.startswith("# ") or ": " not in line: + continue + raw_key, raw_value = line[2:].rstrip("\n").split(": ", 1) + if raw_key in LOG_HEADER_KEYS: + counts[raw_key] = counts.get(raw_key, 0) + 1 + metadata.setdefault(raw_key, raw_value) + return metadata, counts + + +def read_command_log_header_metadata(log_path: Path) -> dict[str, str]: + metadata, _counts = collect_command_log_header_metadata(log_path) + return metadata + + +def _nonempty_log_lines(log_path: Path) -> list[str]: + return [ + line.rstrip("\n") + for line in log_path.read_text(encoding="utf-8", errors="replace").splitlines() + if line.rstrip("\n") != "" + ] + + +def validate_command_log_header_layout( + *, + log_path: Path, + command_id: str, + attempt: dict[str, Any], + attempt_index: int, +) -> list[str]: + errors: list[str] = [] + lines = _nonempty_log_lines(log_path) + argv_json = json.dumps(attempt.get("argv"), ensure_ascii=True) + expected_prefix = [f"# command_id: {command_id}"] + if attempt.get("target_run_id") is not None: + expected_prefix.append(f"# target_run_id: {attempt.get('target_run_id')}") + expected_prefix.extend( + [ + f"# command: {attempt.get('command')}", + f"# argv_json: {argv_json}", + f"# started_at: {attempt.get('started_at')}", + ] + ) + expected_suffix = [ + f"# completed_at: {attempt.get('completed_at')}", + f"# exit_code: {attempt.get('exit_code')}", + ] + if lines[: len(expected_prefix)] != expected_prefix: + errors.append(f"raw log header layout mismatch for {command_id} attempt {attempt_index}: preamble") + if lines[-len(expected_suffix) :] != expected_suffix: + errors.append(f"raw log header layout mismatch for {command_id} attempt {attempt_index}: trailer") + return errors + + +def validate_command_log_header_metadata( + *, + log_path: Path, + command_id: str, + attempt: dict[str, Any], + attempt_index: int, +) -> list[str]: + errors: list[str] = [] + metadata, counts = collect_command_log_header_metadata(log_path) + errors.extend( + validate_command_log_header_layout( + log_path=log_path, + command_id=command_id, + attempt=attempt, + attempt_index=attempt_index, + ) + ) + for key, count in sorted(counts.items()): + if count > 1: + errors.append( + f"raw log header duplicate key for {command_id} attempt {attempt_index}: {key}" + ) + expected = { + "command_id": command_id, + "command": str(attempt.get("command") or ""), + "started_at": str(attempt.get("started_at") or ""), + "completed_at": str(attempt.get("completed_at") or ""), + "exit_code": str(attempt.get("exit_code")), + } + if attempt.get("target_run_id") is not None: + expected["target_run_id"] = str(attempt.get("target_run_id") or "") + for key, expected_value in expected.items(): + observed = metadata.get(key) + if observed != expected_value: + errors.append( + f"raw log header mismatch for {command_id} attempt {attempt_index}: {key}" + ) + raw_argv_json = metadata.get("argv_json") + if raw_argv_json is None: + errors.append(f"raw log header missing argv_json for {command_id} attempt {attempt_index}") + else: + try: + observed_argv = json.loads(raw_argv_json) + except json.JSONDecodeError: + errors.append(f"raw log header invalid argv_json for {command_id} attempt {attempt_index}") + else: + if observed_argv != attempt.get("argv"): + errors.append( + f"raw log header mismatch for {command_id} attempt {attempt_index}: argv_json" + ) + return errors + + +def record_command_log_result( + *, + manifest_path: Path, + command_id: str, + command: str, + argv: list[str] | None = None, + log_path: Path, + exit_code: int, + started_at: str, + completed_at: str, + description: str | None = None, + notes: str = "", + target_run_id: str | None = None, +) -> dict[str, Any]: + command_id_errors = validate_command_id(command_id) + if command_id_errors: + raise ValueError("; ".join(command_id_errors)) + if target_run_id is not None: + target_run_errors = validate_target_run_id(target_run_id) + if target_run_errors: + raise ValueError("; ".join(target_run_errors)) + manifest = read_command_log_manifest(manifest_path) + commands = [item for item in manifest.get("commands", []) if isinstance(item, dict)] + entry = next((item for item in commands if item.get("command_id") == command_id), None) + if entry is None: + entry = { + "command_id": command_id, + "required": command_id in REQUIRED_COMMAND_LOG_IDS, + "description": description or "", + } + commands.append(entry) + if description is not None: + entry["description"] = description + relative_log_path = manifest_relative_log_path(manifest_path, log_path) + log_path_errors = validate_manifest_log_path(relative_log_path) + if log_path_errors: + raise ValueError("; ".join(log_path_errors)) + argv_list = normalize_command_argv(command, argv) + attempt_record = { + "command": command, + "argv": argv_list, + "status": "passed" if exit_code == 0 else "failed", + "exit_code": exit_code, + "log_path": relative_log_path, + "sha256": sha256_file(log_path), + "started_at": started_at, + "completed_at": completed_at, + "notes": notes, + } + if target_run_id is not None: + attempt_record["target_run_id"] = target_run_id + attempts = entry.get("attempts") + if not isinstance(attempts, list): + attempts = [] + attempts.append(dict(attempt_record)) + entry.update( + { + **attempt_record, + "attempts": attempts, + } + ) + manifest["commands"] = commands + manifest["required_command_ids"] = list(REQUIRED_COMMAND_LOG_IDS) + target_run_ids = sorted( + { + str(item.get("target_run_id")) + for item in commands + if item.get("target_run_id") + } + ) + manifest["target_run_ids"] = target_run_ids + manifest["latest_target_run_id"] = target_run_id or (target_run_ids[-1] if target_run_ids else None) + + required_ids = set(REQUIRED_COMMAND_LOG_IDS) + archived_ids = { + str(item.get("command_id")) + for item in commands + if item.get("command_id") in required_ids and item.get("log_path") and str(item.get("sha256") or "").startswith("sha256:") + } + passed_ids = { + str(item.get("command_id")) + for item in commands + if item.get("command_id") in required_ids and item.get("status") == "passed" + } + manifest["all_required_logs_archived"] = required_ids <= archived_ids + manifest["all_required_commands_passed"] = required_ids <= passed_ids + write_command_log_manifest(manifest_path, manifest) + return manifest + + +def validate_command_log_manifest( + manifest_path: Path, + *, + required_command_ids: list[str] | None = None, + require_passed: bool = True, + verify_hashes: bool = True, +) -> list[str]: + errors: list[str] = [] + manifest = read_command_log_manifest(manifest_path) + canonical_required_ids = set(REQUIRED_COMMAND_LOG_IDS) + raw_manifest_required_ids = manifest.get("required_command_ids") + if not isinstance(raw_manifest_required_ids, list): + errors.append("required_command_ids must equal canonical M12 required command IDs") + manifest_required_ids: set[str] = set() + else: + manifest_required_id_list = [str(item) for item in raw_manifest_required_ids] + manifest_required_ids = set(manifest_required_id_list) + if manifest_required_id_list != list(REQUIRED_COMMAND_LOG_IDS): + errors.append("required_command_ids must equal canonical M12 required command IDs") + required_ids = set(required_command_ids or canonical_required_ids) + commands = [item for item in manifest.get("commands", []) if isinstance(item, dict)] + seen_command_ids: set[str] = set() + duplicate_command_ids: set[str] = set() + entries_by_id = {str(item.get("command_id") or ""): item for item in commands} + allowed_command_ids = canonical_required_ids | set(OPTIONAL_COMMAND_LOG_COMMANDS) + if manifest.get("manifest_id") != COMMAND_LOG_MANIFEST_ID: + errors.append("manifest_id must be bb_zyphra_rl_phase1_m12_command_log_manifest_v1") + if manifest.get("claim_boundary") != COMMAND_LOG_CLAIM_BOUNDARY: + errors.append("claim_boundary must remain target_command_logs_not_scorecard_update") + if manifest.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if manifest.get("m12_points_awarded") is not False: + errors.append("m12_points_awarded must be false") + for command_id in sorted(required_ids): + for command_id_error in validate_command_id(command_id): + errors.append(f"invalid required_command_id {command_id!r}: {command_id_error}") + for entry in commands: + command_id = str(entry.get("command_id") or "") + if command_id in seen_command_ids: + duplicate_command_ids.add(command_id) + seen_command_ids.add(command_id) + for command_id_error in validate_command_id(command_id): + errors.append(f"invalid command_id {command_id!r}: {command_id_error}") + if command_id and command_id not in allowed_command_ids: + errors.append(f"unknown command entry: {command_id}") + if command_id in allowed_command_ids: + expected_required = command_id in canonical_required_ids + if entry.get("required") is not expected_required: + errors.append(f"required flag mismatch for {command_id}") + target_run_id = entry.get("target_run_id") + if target_run_id is not None: + for target_run_error in validate_target_run_id(str(target_run_id)): + errors.append(f"invalid target_run_id for {command_id}: {target_run_error}") + raw_log_path = entry.get("log_path") + if raw_log_path: + for log_path_error in validate_manifest_log_path(str(raw_log_path)): + errors.append(f"invalid log_path for {command_id}: {log_path_error}") + attempts = entry.get("attempts") + row_is_completed = entry.get("status") in {"passed", "failed"} or bool(entry.get("log_path")) or bool(entry.get("sha256")) + for status_error in validate_status_exit_code_congruence(entry.get("status"), entry.get("exit_code")): + errors.append(f"invalid status/exit_code for {command_id}: {status_error}") + if row_is_completed and not isinstance(attempts, list): + errors.append(f"completed command entry must preserve attempts: {command_id}") + if attempts is not None: + if not isinstance(attempts, list) or not attempts: + errors.append(f"attempts must be a non-empty list when present: {command_id}") + else: + for index, attempt in enumerate(attempts, start=1): + if not isinstance(attempt, dict): + errors.append(f"attempt {index} for {command_id} must be an object") + continue + attempt_log_path = attempt.get("log_path") + if not attempt_log_path: + errors.append(f"attempt {index} for {command_id} missing log_path") + else: + for log_path_error in validate_manifest_log_path(str(attempt_log_path)): + errors.append(f"invalid attempt log_path for {command_id} attempt {index}: {log_path_error}") + attempt_sha = str(attempt.get("sha256") or "") + if not attempt_sha.startswith("sha256:"): + errors.append(f"attempt {index} for {command_id} missing sha256") + attempt_status = attempt.get("status") + if attempt_status not in {"passed", "failed"}: + errors.append(f"attempt {index} for {command_id} status must be passed or failed") + for status_error in validate_status_exit_code_congruence( + attempt_status, + attempt.get("exit_code"), + ): + errors.append(f"invalid status/exit_code for {command_id} attempt {index}: {status_error}") + for argv_error in validate_command_argv(attempt.get("argv")): + errors.append(f"invalid argv for {command_id} attempt {index}: {argv_error}") + for congruence_error in validate_command_argv_congruence( + attempt.get("command"), attempt.get("argv") + ): + errors.append(f"invalid command/argv for {command_id} attempt {index}: {congruence_error}") + attempt_target_run_id = attempt.get("target_run_id") + if attempt_target_run_id is not None: + for target_run_error in validate_target_run_id(str(attempt_target_run_id)): + errors.append(f"invalid target_run_id for {command_id} attempt {index}: {target_run_error}") + if verify_hashes and attempt_log_path and attempt_sha.startswith("sha256:"): + attempt_resolved_log_path = resolve_manifest_log_path(manifest_path, str(attempt_log_path)) + if not attempt_resolved_log_path.is_file(): + errors.append(f"attempt log file missing: {command_id} attempt {index}") + elif sha256_file(attempt_resolved_log_path) != attempt_sha: + errors.append(f"attempt log sha256 mismatch: {command_id} attempt {index}") + else: + errors.extend( + validate_command_log_header_metadata( + log_path=attempt_resolved_log_path, + command_id=command_id, + attempt=attempt, + attempt_index=index, + ) + ) + if attempts and isinstance(attempts[-1], dict): + latest_attempt = attempts[-1] + for field in [ + "command", + "argv", + "status", + "exit_code", + "log_path", + "sha256", + "started_at", + "completed_at", + "target_run_id", + ]: + if latest_attempt.get(field) != entry.get(field): + errors.append(f"latest attempt mismatch for {command_id}: {field}") + for command_id in sorted(duplicate_command_ids): + errors.append(f"duplicate command entry: {command_id}") + for target_run_id in manifest.get("target_run_ids") or []: + for target_run_error in validate_target_run_id(str(target_run_id)): + errors.append(f"invalid manifest target_run_id {target_run_id!r}: {target_run_error}") + latest_target_run_id = manifest.get("latest_target_run_id") + if latest_target_run_id is not None: + for target_run_error in validate_target_run_id(str(latest_target_run_id)): + errors.append(f"invalid latest_target_run_id: {target_run_error}") + row_target_run_ids = sorted( + { + str(item.get("target_run_id")) + for item in commands + if item.get("target_run_id") + } + ) + if manifest.get("target_run_ids") != row_target_run_ids: + errors.append("target_run_ids must equal sorted target_run_id values from command rows") + if latest_target_run_id is not None and str(latest_target_run_id) not in row_target_run_ids: + errors.append("latest_target_run_id must be present in target_run_ids") + archived_ids = { + str(item.get("command_id")) + for item in commands + if item.get("command_id") in canonical_required_ids + and item.get("log_path") + and str(item.get("sha256") or "").startswith("sha256:") + } + passed_ids = { + str(item.get("command_id")) + for item in commands + if item.get("command_id") in canonical_required_ids and item.get("status") == "passed" + } + expected_logs_archived = canonical_required_ids <= archived_ids + expected_commands_passed = canonical_required_ids <= passed_ids + if manifest.get("all_required_logs_archived") is not expected_logs_archived: + errors.append("all_required_logs_archived must match required command log rows") + if manifest.get("all_required_commands_passed") is not expected_commands_passed: + errors.append("all_required_commands_passed must match required command statuses") + for command_id in sorted(required_ids): + entry = entries_by_id.get(command_id) + if entry is None: + errors.append(f"missing command entry: {command_id}") + continue + if require_passed and entry.get("status") != "passed": + errors.append(f"command did not pass: {command_id}") + raw_log_path = entry.get("log_path") + if not raw_log_path: + errors.append(f"missing log_path: {command_id}") + continue + log_path_errors = validate_manifest_log_path(str(raw_log_path)) + if log_path_errors: + continue + expected_sha = str(entry.get("sha256") or "") + if not expected_sha.startswith("sha256:"): + errors.append(f"missing sha256: {command_id}") + continue + if verify_hashes: + resolved_log_path = resolve_manifest_log_path(manifest_path, str(raw_log_path)) + if not resolved_log_path.is_file(): + errors.append(f"log file missing: {command_id}") + continue + actual_sha = sha256_file(resolved_log_path) + if actual_sha != expected_sha: + errors.append(f"log sha256 mismatch: {command_id}") + return errors diff --git a/breadboard/rl/m12/evidence_consistency.py b/breadboard/rl/m12/evidence_consistency.py new file mode 100644 index 00000000..cd98e98a --- /dev/null +++ b/breadboard/rl/m12/evidence_consistency.py @@ -0,0 +1,1017 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import yaml + +from breadboard.rl.m12.bootstrap import validate_m12_bootstrap_dry_run_report +from breadboard.rl.m12.final_report import ( + ARCHIVE_VERIFY_CLAIM_BOUNDARY, + ARCHIVE_VERIFY_REPORT_ID, + validate_m12_final_report, + validate_m12_final_report_remediation_summary, +) +from breadboard.rl.m12.preflight import validate_m12_preflight_report +from breadboard.rl.m12.promotion_audit import validate_m12_promotion_audit +from breadboard.rl.m12.transfer import ( + validate_m12_readiness_summary, + validate_m12_transfer_archive_manifest, + validate_m12_transfer_overlay_report, + validate_m12_transfer_summary, +) + + +EVIDENCE_CONSISTENCY_ID = "bb_zyphra_rl_phase1_m12_evidence_consistency_v1" +EVIDENCE_CONSISTENCY_CLAIM_BOUNDARY = "m12_evidence_consistency_not_scorecard_update" +REVIEW_MUTABLE_TRANSFER_ARTIFACTS = { + "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml", + "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md", + "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_M12_VALIDATION_REPORT.md", + "tests/test_rl_phase1_scorecard_schema.py", +} + + + +def _read_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _read_yaml(path: Path) -> dict[str, Any]: + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def _m12_milestone(scorecard: dict[str, Any]) -> dict[str, Any]: + for milestone in scorecard.get("milestones") or []: + if isinstance(milestone, dict) and milestone.get("id") == "M12": + return milestone + return {} + + +def _m12_evidence_result_contains( + *, + m12: dict[str, Any], + command_fragment: str, + required_fragment: str, +) -> bool: + for evidence in m12.get("evidence") or []: + if not isinstance(evidence, dict): + continue + if command_fragment not in str(evidence.get("command") or ""): + continue + return required_fragment in str(evidence.get("result") or "") + return False + + +def _phase_workspace_root(phase_dir: Path) -> Path: + return phase_dir.resolve().parents[2] + + +def _resolve_transfer_repo_root(*, transfer_manifest: dict[str, Any], phase_dir: Path) -> Path | None: + repo = transfer_manifest.get("repo") or {} + raw_root = str(repo.get("root") or "") + if not raw_root: + return None + root_path = Path(raw_root) + if root_path.is_absolute(): + return root_path + if ".." in root_path.parts: + return None + return _phase_workspace_root(phase_dir) / root_path + + +def _artifact_file_hashes_current(*, transfer_manifest: dict[str, Any], phase_dir: Path) -> bool: + repo_root = _resolve_transfer_repo_root(transfer_manifest=transfer_manifest, phase_dir=phase_dir) + if repo_root is None: + return False + repo_root = repo_root.resolve() + if not repo_root.is_dir(): + repo_root = Path.cwd().resolve() + if not repo_root.is_dir(): + return False + for artifact in transfer_manifest.get("artifacts") or []: + if not isinstance(artifact, dict) or artifact.get("kind") != "file": + continue + if artifact.get("path") in REVIEW_MUTABLE_TRANSFER_ARTIFACTS: + continue + source = repo_root / str(artifact.get("path") or "") + if not source.is_file() or artifact.get("sha256") != _sha256_file(source): + return False + return True + + +def _promotion_audit_input_hashes_current( + *, + promotion_audit: dict[str, Any], + scorecard_path: Path, + claim_ledger_path: Path, + final_report_path: Path, +) -> bool: + inputs = promotion_audit.get("inputs") or {} + expected = { + "scorecard": scorecard_path, + "claim_ledger": claim_ledger_path, + "final_report": final_report_path, + } + for key, path in expected.items(): + recorded = (inputs.get(key) or {}).get("sha256") + if not isinstance(recorded, str) or not recorded.startswith("sha256:"): + return False + if key == "final_report" and recorded != _sha256_file(path): + return False + if key != "final_report" and not path.is_file(): + return False + return True + + +def _bootstrap_input_hashes_current(bootstrap_report: dict[str, Any]) -> bool: + input_hashes = bootstrap_report.get("input_hashes") + if not isinstance(input_hashes, dict): + return False + for field in ["bootstrap_script", "transfer_manifest", "archive_manifest", "overlay_dry_run_report"]: + raw_path = bootstrap_report.get(field) + recorded = input_hashes.get(field) + if not raw_path or not isinstance(recorded, str) or not recorded.startswith("sha256:"): + return False + if not Path(str(raw_path)).is_file(): + return False + return True + + +def _archive_verify_report_errors(*, report: dict[str, Any], archive_manifest: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != ARCHIVE_VERIFY_REPORT_ID: + errors.append(f"report_id must be {ARCHIVE_VERIFY_REPORT_ID}") + if report.get("claim_boundary") != ARCHIVE_VERIFY_CLAIM_BOUNDARY: + errors.append(f"claim_boundary must remain {ARCHIVE_VERIFY_CLAIM_BOUNDARY}") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if report.get("m12_points_awarded") is not False: + errors.append("m12_points_awarded must be false") + if report.get("status") != "passed": + errors.append("status must be passed") + if report.get("errors") != []: + errors.append("errors must be empty") + if report.get("manifest_read_error") is not None: + errors.append("manifest_read_error must be null") + expected_pairs = { + "archive_manifest_id": archive_manifest.get("archive_manifest_id"), + "archive_claim_boundary": archive_manifest.get("claim_boundary"), + "archive_sha256": archive_manifest.get("archive_sha256"), + "archive_size_bytes": archive_manifest.get("archive_size_bytes"), + "included_entry_count": archive_manifest.get("included_entry_count"), + "all_required_artifacts_present": archive_manifest.get("all_required_artifacts_present"), + "all_transfer_requirements_covered": archive_manifest.get("all_transfer_requirements_covered"), + "archive_contains_source_overlay": archive_manifest.get("archive_contains_source_overlay"), + "archive_deterministic": archive_manifest.get("archive_deterministic"), + "source_paths_portable": archive_manifest.get("source_paths_portable"), + } + for key, expected in expected_pairs.items(): + if report.get(key) != expected: + errors.append(f"{key} must match archive manifest") + if Path(str(report.get("archive_path") or "")).name != archive_manifest.get("archive_path"): + errors.append("archive_path filename must match archive manifest") + if Path(str(report.get("archive_sha256_file") or "")).name != archive_manifest.get("archive_sha256_file"): + errors.append("archive_sha256_file filename must match archive manifest") + return errors + + +def _bool_checks_missing(sections: dict[str, dict[str, bool]]) -> list[str]: + missing: list[str] = [] + for section, checks in sections.items(): + for name, passed in checks.items(): + if not passed: + missing.append(f"{section}.{name}") + return missing + + +def _evidence_check_summary(report: dict[str, Any]) -> tuple[list[str], list[str], set[str]]: + checks = report.get("checks") + if not isinstance(checks, dict) or not checks: + return [], [], set() + errors: list[str] = [] + missing: list[str] = [] + known_check_ids: set[str] = set() + for section, section_checks in checks.items(): + section_name = str(section) + if not isinstance(section_checks, dict): + errors.append(f"checks.{section_name} must be an object") + missing.append(f"{section_name}.__section__") + known_check_ids.add(f"{section_name}.__section__") + continue + for name, passed in section_checks.items(): + check_id = f"{section_name}.{name}" + known_check_ids.add(check_id) + if passed is not True and passed is not False: + errors.append(f"checks.{check_id} must be boolean") + if passed is not True: + missing.append(check_id) + return errors, missing, known_check_ids + + +def build_m12_evidence_consistency_report(*, phase_dir: Path) -> dict[str, Any]: + scorecard_path = phase_dir / "BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" + claim_ledger_path = phase_dir / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + blocked_report_path = phase_dir / "BB_ZYPHRA_RL_PHASE_1_M12_VALIDATION_REPORT.md" + handoff_path = phase_dir / "BB_ZYPHRA_RL_PHASE_1_HANDOFF.md" + readiness_summary_path = phase_dir / "runs/m12_transfer_prep/m12_readiness_summary.json" + transfer_summary_path = phase_dir / "runs/m12_transfer_prep/m12_transfer_summary.json" + transfer_manifest_path = phase_dir / "runs/m12_transfer_prep/m12_transfer_manifest.json" + archive_manifest_path = phase_dir / "runs/m12_transfer_prep/m12_transfer_archive_manifest.json" + archive_verify_report_path = phase_dir / "runs/m12_transfer_prep/m12_archive_verify_report.json" + overlay_report_path = phase_dir / "runs/m12_overlay_apply_probe/m12_overlay_apply_report.json" + bootstrap_report_path = phase_dir / "runs/m12_bootstrap_dry_run/m12_bootstrap_dry_run_report.json" + preflight_path = phase_dir / "runs/m12_target_preflight/m12_preflight_report.json" + final_report_path = phase_dir / "runs/m12_final_report/m12_final_report.json" + remediation_summary_path = phase_dir / "runs/m12_final_report/m12_remediation_summary.json" + promotion_audit_path = phase_dir / "runs/m12_promotion_audit/m12_promotion_audit.json" + + scorecard = _read_yaml(scorecard_path) + claim_ledger = claim_ledger_path.read_text(encoding="utf-8") + blocked_report = blocked_report_path.read_text(encoding="utf-8") + handoff = handoff_path.read_text(encoding="utf-8") + readiness_summary = _read_json(readiness_summary_path) + transfer_summary = _read_json(transfer_summary_path) + transfer_manifest = _read_json(transfer_manifest_path) + archive_manifest = _read_json(archive_manifest_path) + archive_verify_report = _read_json(archive_verify_report_path) + overlay_report = _read_json(overlay_report_path) + bootstrap_report = _read_json(bootstrap_report_path) + preflight = _read_json(preflight_path) + final_report = _read_json(final_report_path) + remediation_summary = _read_json(remediation_summary_path) + promotion_audit = _read_json(promotion_audit_path) + + m12 = _m12_milestone(scorecard) + milestone_sum = sum( + int(milestone.get("verified_points", 0)) + for milestone in scorecard.get("milestones") or [] + if isinstance(milestone, dict) + ) + artifact_count = len(transfer_manifest.get("artifacts") or []) + command_count = len(transfer_manifest.get("test_commands") or []) + expected_output_count = len(transfer_manifest.get("expected_outputs") or []) + readiness_summary_errors = validate_m12_readiness_summary(readiness_summary, transfer_manifest) + transfer_summary_errors = validate_m12_transfer_summary(transfer_summary, transfer_manifest) + archive_errors = validate_m12_transfer_archive_manifest(archive_manifest_path) + archive_verify_report_errors = _archive_verify_report_errors( + report=archive_verify_report, + archive_manifest=archive_manifest, + ) + overlay_errors = validate_m12_transfer_overlay_report(overlay_report) + bootstrap_errors = validate_m12_bootstrap_dry_run_report(bootstrap_report) + bootstrap_consistency_errors = [ + error + for error in bootstrap_errors + if error + not in { + "input_hashes.transfer_manifest does not match current file", + "input_hashes.archive_manifest does not match current file", + } + ] + preflight_errors = validate_m12_preflight_report(preflight) + final_report_errors = validate_m12_final_report(final_report) + remediation_summary_errors = validate_m12_final_report_remediation_summary(remediation_summary) + promotion_audit_errors = validate_m12_promotion_audit(promotion_audit) + final_report_missing_gates = [str(gate) for gate in final_report.get("missing_gates") or []] + final_report_remediations = [ + item for item in final_report.get("missing_gate_remediations") or [] if isinstance(item, dict) + ] + final_report_remediation_gates = [ + str(item.get("gate") or "") for item in final_report_remediations + ] + remediation_summary_action_gates = [ + str(gate) + for action in remediation_summary.get("next_target_actions") or [] + if isinstance(action, dict) + for gate in action.get("gates") or [] + ] + + transfer_checks = { + "readiness_summary_validator_passed": not readiness_summary_errors, + "transfer_summary_validator_passed": not transfer_summary_errors, + "summary_matches_manifest_artifacts": transfer_summary.get("artifact_count") == artifact_count, + "summary_matches_manifest_commands": transfer_summary.get("command_count") == command_count, + "summary_matches_manifest_expected_outputs": transfer_summary.get("expected_output_count") == expected_output_count, + "artifacts_present": transfer_summary.get("artifacts_present") is True, + "requirements_covered": transfer_summary.get("all_transfer_requirements_covered") is True, + "archive_verifier_first": transfer_summary.get("archive_verifier_runs_first") is True, + "preflight_requires_pass": transfer_summary.get("preflight_command_require_pass") is True, + "final_report_requires_eligible": transfer_summary.get("final_command_require_eligible") is True, + "promotion_audit_requires_ready": transfer_summary.get("promotion_audit_require_ready") is True, + "promotion_audit_explicit_score_inputs": transfer_summary.get("promotion_audit_explicit_score_inputs") is True, + "promotion_audit_explicit_target_paths": transfer_summary.get("promotion_audit_explicit_target_paths") is True, + "bootstrap_dirty_checkout_guard": transfer_summary.get("bootstrap_dirty_checkout_guard") is True, + "target_run_id_command_binding": transfer_summary.get("target_run_id_command_binding") is True, + "target_run_log_reuse_guard": transfer_summary.get("target_run_log_reuse_guard") is True, + "target_closeout_artifact_reuse_guard": transfer_summary.get("target_closeout_artifact_reuse_guard") is True, + "final_report_failure_remediation_summary": transfer_summary.get("final_report_failure_remediation_summary") is True, + "blocked_report_in_artifacts": any( + artifact.get("path") == "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_M12_VALIDATION_REPORT.md" + and artifact.get("exists") is True + for artifact in transfer_manifest.get("artifacts") or [] + if isinstance(artifact, dict) + ), + "promotion_audit_command_present": any( + "audit_m12_score_promotion.py" in str(command) + for command in transfer_manifest.get("test_commands") or [] + ), + "readiness_summary_matches_manifest_artifacts": readiness_summary.get("artifact_count") == artifact_count, + "readiness_summary_matches_manifest_commands": readiness_summary.get("command_count") == command_count, + "readiness_summary_matches_manifest_expected_outputs": readiness_summary.get("expected_output_count") + == expected_output_count, + "readiness_summary_scorecard_update_disallowed": readiness_summary.get("scorecard_update_allowed") is False, + "readiness_summary_points_not_awarded": readiness_summary.get("m12_points_awarded") is False, + "readiness_summary_fail_closed": all( + value is True for value in (readiness_summary.get("target_script_fail_closed") or {}).values() + ) + and readiness_summary.get("generated_script_validation_errors") == [], + } + sections = { + "scorecard": { + "status_completed": scorecard.get("status") == "phase1_complete_m12_target_validated", + "current_points_1000": scorecard.get("current_verified_points") == 1000, + "total_points_1000": scorecard.get("total_points") == 1000, + "points_match_milestones": scorecard.get("current_verified_points") == milestone_sum, + "m12_present": bool(m12), + "m12_awarded": m12.get("verified_points") == 80, + "m12_points_80": m12.get("points") == 80, + "m12_status_completed": m12.get("status") == "completed", + "planning_prose_does_not_score": (scorecard.get("score_policy") or {}).get("planning_prose_scores") is False, + "allowed_claim_records_cli_log_dir_preexecution": any( + "unsafe CLI log-dir rejection before wrapped command execution" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_unknown_command_row_rejection": any( + "duplicate/unsafe/unknown command-log manifest row rejection" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_inference_container_gates": any( + "inference-engine and container-runtime final-report gates" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_preflight_self_consistency": any( + "preflight self-consistency and runtime-fingerprint/top-level evidence matching" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_target_run_identity_binding": any( + "target artifact target_run_id binding to command-log target_run_id" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_target_run_identity_self_consistency": any( + "target_run_identity summary self-consistency" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_final_report_gate_self_consistency": any( + "final-report missing-gate and score-eligibility self-consistency" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_artifact_path_policy_self_consistency": any( + "artifact_path_policy self-consistency" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_artifact_section_path_self_consistency": any( + "artifact_paths embedded section-path self-consistency" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_command_log_summary_self_consistency": any( + "command_logs embedded command-summary self-consistency" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_promotion_audit_self_consistency": any( + "promotion-audit missing-requirement and readiness self-consistency" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_promotion_final_report_path_gate": any( + "promotion-audit canonical final-report input-path gating" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_promotion_control_input_path_gates": any( + "promotion-audit canonical control-input path gating" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_promotion_output_path_gate": any( + "promotion-audit canonical output-path gating" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_promotion_target_script_path_gates": any( + "target-script promotion-audit explicit target-path gating" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_target_run_log_reuse_guard": any( + "target-run command-log reuse guard" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_target_closeout_artifact_reuse_guard": any( + "target close-out artifact reuse guard" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_evidence_consistency_self_consistency": any( + "evidence-consistency embedded check/error self-consistency" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_overlay_report_self_consistency": any( + "transfer overlay report status, error-list, write-count, and existing-destination self-consistency" + in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "allowed_claim_records_bootstrap_overlay_file_validation": any( + "bootstrap dry-run reports validate the referenced overlay dry-run report" in str(claim) + for claim in scorecard.get("allowed_current_claims") or [] + ), + "m12_final_report_result_records_target_artifact_path_gate": _m12_evidence_result_contains( + m12=m12, + command_fragment="build_m12_final_report.py", + required_fragment="artifact_paths_match_target_defaults", + ), + }, + "claim_ledger": { + "has_preparation_only_claim": "transfer/preflight/final-report preparation only" in claim_ledger, + "has_target_validation_claim": "BreadBoard passed 8xMI300X final validation" in claim_ledger, + "has_final_report_manifest_validation_claim": "final-report full command-log manifest validation gating" in claim_ledger, + "has_final_report_component_validator_claim": "component-report validator reuse for archive-verifier/SWE/export/Ray/preflight/load/soak final-report eligibility" + in claim_ledger, + "has_inference_container_gate_claim": "inference-engine and container-runtime final-report gates" + in claim_ledger, + "has_preflight_self_consistency_claim": "preflight self-consistency and runtime-fingerprint/top-level evidence matching" + in claim_ledger, + "has_target_run_identity_binding_claim": "target artifact target_run_id binding to command-log target_run_id" + in claim_ledger, + "has_target_run_identity_self_consistency_claim": "target_run_identity summary self-consistency" + in claim_ledger, + "has_final_report_gate_self_consistency_claim": "final-report missing-gate and score-eligibility self-consistency" + in claim_ledger, + "has_artifact_path_policy_self_consistency_claim": "artifact_path_policy self-consistency" + in claim_ledger, + "has_artifact_section_path_self_consistency_claim": "artifact_paths embedded section-path self-consistency" + in claim_ledger, + "has_command_log_summary_self_consistency_claim": "command_logs embedded command-summary self-consistency" + in claim_ledger, + "has_promotion_audit_self_consistency_claim": "promotion-audit missing-requirement and readiness self-consistency" + in claim_ledger, + "has_promotion_final_report_path_gate_claim": ( + "promotion-audit canonical final-report input-path gating" in claim_ledger + ), + "has_promotion_control_input_path_gates_claim": ( + "promotion-audit canonical control-input path gating" in claim_ledger + ), + "has_promotion_output_path_gate_claim": ( + "promotion-audit canonical output-path gating" in claim_ledger + ), + "has_promotion_target_script_path_gate_claim": ( + "target-script promotion-audit explicit target-path gating" in claim_ledger + ), + "has_target_run_log_reuse_guard_claim": "target-run command-log reuse guard" in claim_ledger, + "has_target_closeout_artifact_reuse_guard_claim": "target close-out artifact reuse guard" + in claim_ledger, + "has_evidence_consistency_self_consistency_claim": "evidence-consistency embedded check/error self-consistency" + in claim_ledger, + "has_overlay_report_self_consistency_claim": ( + "transfer overlay report status, error-list, write-count, and existing-destination self-consistency" + in claim_ledger + ), + "has_bootstrap_overlay_file_validation_claim": ( + "bootstrap dry-run reports validate the referenced overlay dry-run report" in claim_ledger + ), + "has_target_artifact_path_claim": "target artifact path-default gating" in claim_ledger, + "has_command_argv_capture_claim": "wrapped-command argv capture in raw logs and manifest attempts" + in claim_ledger, + "has_command_argv_congruence_claim": "command/argv congruence validation" in claim_ledger, + "has_cli_log_dir_preexecution_claim": "unsafe CLI log-dir rejection before wrapped command execution" + in claim_ledger, + "has_raw_log_header_manifest_consistency_claim": "raw-log header/manifest consistency validation" + in claim_ledger, + "has_raw_log_layout_claim": "raw-log preamble/trailer layout validation" + in claim_ledger, + "has_no_newline_trailer_separation_claim": "no-newline output trailer separation" + in claim_ledger, + "has_malformed_target_artifact_claim": "malformed required and optional target-artifact read-error capture" + in claim_ledger, + "has_promotion_control_input_read_error_claim": "promotion-audit control-input read-error capture" + in claim_ledger, + "has_duplicate_reserved_raw_log_header_rejection_claim": "duplicate reserved raw-log header rejection" + in claim_ledger, + "has_unknown_command_row_rejection_claim": "duplicate/unsafe/unknown command-log manifest row rejection" + in claim_ledger, + "has_promotion_row_equality_claim": "final-report required command-row equality against the raw manifest" + in claim_ledger, + "has_promotion_explicit_score_inputs_claim": "target promotion-audit explicit scorecard/claim-ledger input binding" + in claim_ledger, + "future_claim_requires_final_report": "m12_score_eligible=true" in claim_ledger, + "future_claim_requires_promotion_audit": "promotion_review_ready=true" in claim_ledger, + "future_claim_requires_separate_review": "separately reviewed" in claim_ledger, + "future_claim_requires_raw_log_layout": "raw-log preamble/trailer layout validation passes" + in claim_ledger, + "future_claim_requires_no_newline_separation": "no-newline output trailer separation is preserved" + in claim_ledger, + "future_claim_requires_no_artifact_read_errors": "all required and optional target artifacts used for scoring have no read errors" + in claim_ledger, + "future_claim_requires_readable_promotion_control_inputs": "target promotion audit uses readable explicit scorecard and claim-ledger inputs" + in claim_ledger, + "forbids_production_rollouts": "BreadBoard supports production RL rollouts." in claim_ledger, + }, + "m12_report": { + "exists": blocked_report_path.is_file(), + "states_passed": "Status: passed, target validation executed on 8xMI300X" in blocked_report, + "states_full_points": "Score impact: 80 / 80 M12 points awarded" in blocked_report, + "states_reviewed_scorecard_update": "separate reviewed scorecard/claim-ledger update" in blocked_report, + "states_manifest_validation_gate": "full command-log manifest validation" in blocked_report, + "states_component_validator_gate": "component-report validator reuse for archive-verifier/SWE/export/Ray/preflight/load/soak final-report eligibility" in blocked_report, + "states_preflight_self_consistency": "preflight self-consistency and runtime-fingerprint/top-level evidence matching" + in blocked_report, + "states_target_run_identity_binding": "target artifact target_run_id binding to command-log target_run_id" + in blocked_report, + "states_target_run_identity_self_consistency": "target_run_identity summary self-consistency" + in blocked_report, + "states_final_report_gate_self_consistency": "final-report missing-gate and score-eligibility self-consistency" + in blocked_report, + "states_artifact_path_policy_self_consistency": "artifact_path_policy self-consistency" + in blocked_report, + "states_artifact_section_path_self_consistency": "artifact_paths embedded section-path self-consistency" + in blocked_report, + "states_command_log_summary_self_consistency": "command_logs embedded command-summary self-consistency" + in blocked_report, + "states_promotion_audit_self_consistency": "promotion-audit missing-requirement and readiness self-consistency" + in blocked_report, + "states_promotion_final_report_path_gate": "promotion-audit canonical final-report input-path gating" + in blocked_report, + "states_promotion_control_input_path_gates": "promotion-audit canonical control-input path gating" + in blocked_report, + "states_promotion_output_path_gate": "promotion-audit canonical output-path gating" in blocked_report, + "states_promotion_target_script_path_gate": ( + "target-script promotion-audit explicit target-path gating" in blocked_report + ), + "states_target_run_log_reuse_guard": "target-run command-log reuse guard" in blocked_report, + "states_target_closeout_artifact_reuse_guard": "target close-out artifact reuse guard" + in blocked_report, + "states_evidence_consistency_self_consistency": "evidence-consistency embedded check/error self-consistency" + in blocked_report, + "states_target_artifact_path_gate": "target artifact path-default" in blocked_report, + "states_command_argv_capture": "wrapped-command argv capture" in blocked_report and "argv" in blocked_report, + "states_command_argv_congruence": "command/argv congruence validation" in blocked_report, + "states_cli_log_dir_preexecution": "unsafe CLI log-dir rejection before wrapped command execution" in blocked_report, + "states_raw_log_header_manifest_consistency": "raw-log header/manifest consistency validation" in blocked_report, + "states_raw_log_layout_validation": "raw-log preamble/trailer layout validation" in blocked_report, + "states_no_newline_trailer_separation": "no-newline output trailer separation" in blocked_report, + "states_malformed_target_artifact_capture": "malformed required and optional target-artifact read-error capture" + in blocked_report, + "states_promotion_control_input_read_error_capture": "promotion-audit control-input read-error capture" + in blocked_report, + "states_duplicate_reserved_raw_log_header_rejection": "duplicate reserved raw-log header rejection" in blocked_report, + "states_unknown_command_row_rejection": "duplicate/unsafe/unknown command-log manifest row rejection" in blocked_report, + "states_promotion_row_equality_gate": "final-report required command-row equality against the raw manifest" + in blocked_report, + "states_promotion_explicit_score_inputs": "target promotion-audit explicit scorecard/claim-ledger input binding" + in blocked_report, + "has_human_review_gate": "Human Review" in blocked_report, + "avoids_archive_self_hash": "archive_sha256_recorded_in=m12_transfer_archive_manifest.json" in blocked_report, + }, + "handoff": { + "references_m12_report": "BB_ZYPHRA_RL_PHASE_1_M12_VALIDATION_REPORT.md" in handoff, + "states_m12_scored": "Current verified score: 1000 / 1000" in handoff, + "states_target_passed": "M12 target validation has passed" in handoff, + "states_target_run_id_binding": "target run id" in handoff, + "states_manifest_validation_gate": "command-log" in handoff, + "states_component_validator_gate": "final report" in handoff + and "promotion audit" in handoff, + "states_inference_container_gate": "inference-engine and container-runtime final-report gates" in handoff, + "states_preflight_self_consistency": "preflight self-consistency and runtime-fingerprint/top-level evidence matching" + in handoff, + "states_target_run_identity_binding": "target artifact target_run_id binding to command-log target_run_id" + in handoff, + "states_target_run_identity_self_consistency": "target_run_identity summary self-consistency" + in handoff, + "states_final_report_gate_self_consistency": "final-report missing-gate and score-eligibility self-consistency" + in handoff, + "states_artifact_path_policy_self_consistency": "artifact_path_policy self-consistency" + in handoff, + "states_artifact_section_path_self_consistency": "artifact_paths embedded section-path self-consistency" + in handoff, + "states_command_log_summary_self_consistency": "command_logs embedded command-summary self-consistency" + in handoff, + "states_promotion_audit_self_consistency": "promotion-audit missing-requirement and readiness self-consistency" + in handoff, + "states_promotion_final_report_path_gate": "promotion-audit canonical final-report input-path gating" + in handoff, + "states_promotion_control_input_path_gates": "promotion-audit canonical control-input path gating" + in handoff, + "states_promotion_output_path_gate": "promotion-audit canonical output-path gating" in handoff, + "states_promotion_target_script_path_gate": ( + "target-script promotion-audit explicit target-path gating" in handoff + ), + "states_target_run_log_reuse_guard": "target-run command-log reuse guard" in handoff, + "states_target_closeout_artifact_reuse_guard": "target close-out artifact reuse guard" in handoff, + "states_evidence_consistency_self_consistency": "evidence-consistency embedded check/error self-consistency" + in handoff, + "states_overlay_report_self_consistency": ( + "overlay report status/error/write-count/existing-destination self-consistency validation" in handoff + ), + "states_bootstrap_overlay_file_validation": ( + "bootstrap dry-run referenced-overlay validation and embedded overlay-summary drift rejection" in handoff + ), + "states_target_artifact_path_gate": "target artifact path-default" in handoff, + "states_command_argv_capture": "wrapped-command argv capture in raw logs and manifest attempts" in handoff, + "states_command_argv_congruence": "command/argv congruence validation" in handoff, + "states_cli_log_dir_preexecution": "unsafe CLI log-dir rejection before wrapped command execution" + in handoff, + "states_raw_log_header_manifest_consistency": "raw-log header/manifest consistency validation" in handoff, + "states_raw_log_layout_validation": "raw-log preamble/trailer layout validation" in handoff, + "states_no_newline_trailer_separation": "no-newline output trailer separation" in handoff, + "states_malformed_target_artifact_capture": "malformed required and optional target-artifact read-error capture" + in handoff, + "states_promotion_control_input_read_error_capture": "promotion-audit control-input read-error capture" + in handoff, + "states_duplicate_reserved_raw_log_header_rejection": "duplicate reserved raw-log header rejection" in handoff, + "states_unknown_command_row_rejection": "duplicate/unsafe/unknown command-log manifest row rejection" in handoff, + "states_promotion_row_equality_gate": "final-report required command-row equality against the raw manifest" + in handoff, + "states_promotion_explicit_score_inputs": "target promotion-audit explicit scorecard/claim-ledger input binding" + in handoff, + }, + "transfer": transfer_checks, + "local_consistency_boundary": { + "consistency_report_not_transfer_artifact": not any( + str(artifact.get("path") or "").endswith("runs/m12_evidence_consistency/m12_evidence_consistency.json") + for artifact in transfer_manifest.get("artifacts") or [] + if isinstance(artifact, dict) + ), + "bootstrap_report_not_transfer_artifact": not any( + str(artifact.get("path") or "").endswith("runs/m12_bootstrap_dry_run/m12_bootstrap_dry_run_report.json") + for artifact in transfer_manifest.get("artifacts") or [] + if isinstance(artifact, dict) + ), + "consistency_checker_not_target_command": not any( + "check_m12_evidence_consistency.py" in str(command) + for command in transfer_manifest.get("test_commands") or [] + ), + "bootstrap_dry_run_not_target_command": not any( + "run_m12_bootstrap_dry_run.py" in str(command) + for command in transfer_manifest.get("test_commands") or [] + ), + }, + "archive": { + "validator_passed": not archive_errors, + "scorecard_update_disallowed": archive_manifest.get("scorecard_update_allowed") is False, + "points_not_awarded": archive_manifest.get("m12_points_awarded") is False, + "not_repo_replacement": archive_manifest.get("archive_is_repo_replacement") is False, + "contains_source_overlay": archive_manifest.get("archive_contains_source_overlay") is True, + "excludes_pycache": archive_manifest.get("archive_excludes_pycache") is True, + "source_paths_portable": archive_manifest.get("source_paths_portable") is True, + "archive_paths_portable": archive_manifest.get("archive_path") == archive_manifest.get("archive_name") + and archive_manifest.get("archive_sha256_file") == str(archive_manifest.get("archive_name") or "") + ".sha256", + "deterministic_metadata": archive_manifest.get("archive_deterministic") is True + and archive_manifest.get("deterministic_archive_metadata") + == { + "gzip_mtime": 0, + "member_gid": 0, + "member_gname": "", + "member_mtime": 0, + "member_order": "sorted_by_archive_path", + "member_uid": 0, + "member_uname": "", + }, + "entry_count_matches": archive_manifest.get("included_entry_count") == len(archive_manifest.get("included_entries") or []), + "included_entry_paths_unique": len( + [ + str(entry.get("archive_path") or "") + for entry in archive_manifest.get("included_entries") or [] + if isinstance(entry, dict) + ] + ) + == len( + { + str(entry.get("archive_path") or "") + for entry in archive_manifest.get("included_entries") or [] + if isinstance(entry, dict) + } + ), + "repo_root_path_portable": (transfer_manifest.get("repo") or {}).get("root_path_portable") is True + and not Path(str((transfer_manifest.get("repo") or {}).get("root") or "")).is_absolute() + and ".." not in Path(str((transfer_manifest.get("repo") or {}).get("root") or "")).parts, + "transfer_manifest_file_artifact_hashes_current": _artifact_file_hashes_current( + transfer_manifest=transfer_manifest, + phase_dir=phase_dir, + ), + }, + "archive_verify_report": { + "validator_passed": not archive_verify_report_errors, + "report_id_valid": archive_verify_report.get("report_id") == ARCHIVE_VERIFY_REPORT_ID, + "status_passed": archive_verify_report.get("status") == "passed", + "scorecard_update_disallowed": archive_verify_report.get("scorecard_update_allowed") is False, + "points_not_awarded": archive_verify_report.get("m12_points_awarded") is False, + "errors_empty": archive_verify_report.get("errors") == [], + "manifest_read_error_empty": archive_verify_report.get("manifest_read_error") is None, + "archive_sha_matches_manifest": archive_verify_report.get("archive_sha256") + == archive_manifest.get("archive_sha256"), + "entry_count_matches_manifest": archive_verify_report.get("included_entry_count") + == archive_manifest.get("included_entry_count"), + "source_overlay_matches_manifest": archive_verify_report.get("archive_contains_source_overlay") + == archive_manifest.get("archive_contains_source_overlay"), + "determinism_matches_manifest": archive_verify_report.get("archive_deterministic") + == archive_manifest.get("archive_deterministic"), + "source_portability_matches_manifest": archive_verify_report.get("source_paths_portable") + == archive_manifest.get("source_paths_portable"), + }, + "overlay_apply": { + "validator_passed": not overlay_errors, + "status_passed": overlay_report.get("status") == "passed", + "dry_run_only": overlay_report.get("dry_run") is True, + "scorecard_update_disallowed": overlay_report.get("scorecard_update_allowed") is False, + "points_not_awarded": overlay_report.get("m12_points_awarded") is False, + "no_writes": overlay_report.get("written_count") == 0, + "no_errors": overlay_report.get("errors") == [], + "would_write_matches_archive": overlay_report.get("would_write_count") == archive_manifest.get("included_entry_count"), + "would_write_matches_entries": overlay_report.get("would_write_count") + == len(overlay_report.get("entries") or []), + "written_count_bounded": ( + isinstance(overlay_report.get("written_count"), int) + and isinstance(overlay_report.get("would_write_count"), int) + and 0 <= overlay_report.get("written_count") <= overlay_report.get("would_write_count") + ), + "existing_destination_count_matches_entries": overlay_report.get("existing_destination_count") + == sum( + 1 + for entry in overlay_report.get("entries") or [] + if isinstance(entry, dict) and entry.get("exists") is True + ), + "has_existing_destinations": int(overlay_report.get("existing_destination_count") or 0) > 0, + }, + "bootstrap_dry_run": { + "validator_passed": not bootstrap_consistency_errors, + "status_passed": bootstrap_report.get("status") == "passed", + "scorecard_update_disallowed": bootstrap_report.get("scorecard_update_allowed") is False, + "points_not_awarded": bootstrap_report.get("m12_points_awarded") is False, + "repo_head_verified": bootstrap_report.get("repo_head_verified") is True, + "dirty_checkout_check_observed": bootstrap_report.get("dirty_checkout_check_observed") is True, + "dirty_checkout_mode_recorded": bootstrap_report.get("dirty_checkout_mode") in {"clean", "override"}, + "target_commands_skipped": bootstrap_report.get("target_commands_skipped") is True, + "exit_code_zero": bootstrap_report.get("exit_code") == 0, + "input_hashes_present": all( + str((bootstrap_report.get("input_hashes") or {}).get(field) or "").startswith("sha256:") + for field in [ + "bootstrap_script", + "transfer_manifest", + "archive_manifest", + "overlay_dry_run_report", + ] + ), + "input_hashes_current": _bootstrap_input_hashes_current(bootstrap_report), + "overlay_written_zero": (bootstrap_report.get("overlay") or {}).get("written_count") == 0, + "overlay_would_write_matches_archive": (bootstrap_report.get("overlay") or {}).get("would_write_count") + == archive_manifest.get("included_entry_count"), + "overlay_summary_matches_overlay_file": not any( + error.startswith("overlay.") and error.endswith("must match overlay_dry_run_report") + for error in bootstrap_errors + ), + "overlay_file_validator_passed": not any( + error.startswith("overlay_dry_run_report.") for error in bootstrap_errors + ), + }, + "preflight": { + "validator_passed": not preflight_errors, + "status_passed": preflight.get("status") == "preflight_passed", + "filesystem_cas_smoke_passed": preflight.get("filesystem_cas_smoke", {}).get("status") == "passed", + "no_target_blockers": preflight.get("blockers") == [], + "mi300x_evidence_recorded": preflight.get("gpu", {}).get("mi300x_product_evidence") is True, + "device_count_8_recorded": preflight.get("gpu", {}).get("torch_probe", {}).get("device_count") == 8, + "verl_available": preflight.get("python_modules", {}).get("verl", {}).get("available") is True, + "ray_available": preflight.get("python_modules", {}).get("ray", {}).get("available") is True, + }, + "final_report": { + "validator_passed": not final_report_errors, + "score_eligible_true": final_report.get("m12_score_eligible") is True, + "scorecard_update_disallowed": final_report.get("scorecard_update_allowed") is False, + "missing_gates_empty": final_report.get("missing_gates") == [], + "missing_gate_remediations_empty": final_report.get("missing_gate_remediations") == [], + "target_artifact_paths_match": final_report.get("artifact_path_policy", {}).get( + "paths_match_target_defaults" + ) + is True, + "target_run_ids_match_command_logs": final_report.get("target_run_identity", {}).get( + "run_ids_match_command_logs" + ) + is True, + "single_target_run_id_recorded": bool( + final_report.get("target_run_identity", {}).get("single_command_log_target_run_id") + ), + "inference_engine_gate_satisfied": final_report.get("preflight", {}) + .get("inference_engine_feasibility", {}) + .get("decision") + == "available", + "container_runtime_gate_satisfied": any( + value is True for value in final_report.get("preflight", {}).get("container_runtimes", {}).values() + ), + "preflight_validation_errors_empty": final_report.get("preflight", {}).get("validation_errors") == [], + "swe_validation_errors_empty": final_report.get("swe_probe", {}).get("validation_errors") == [], + "verl_validation_errors_empty": final_report.get("verl_export", {}).get("validation_errors") == [], + "ray_validation_errors_empty": final_report.get("ray_probe", {}).get("validation_errors") == [], + "warm_vs_cold_validation_errors_empty": final_report.get("warm_vs_cold", {}).get("validation_errors") == [], + "load_ladder_validation_errors_empty": final_report.get("load_ladder", {}).get("validation_errors") == [], + "soak_validation_errors_empty": final_report.get("soak", {}).get("validation_errors") == [], + "ray_probe_distributed": final_report.get("ray_probe", {}).get("ray_local_mode") is False, + "soak_distributed": final_report.get("soak", {}).get("ray_local_mode") is False, + }, + "remediation_summary": { + "validator_passed": not remediation_summary_errors, + "scorecard_update_disallowed": remediation_summary.get("scorecard_update_allowed") is False, + "points_not_awarded": remediation_summary.get("m12_points_awarded") is False, + "score_eligible_matches_final_report": remediation_summary.get("m12_score_eligible") + is (final_report.get("m12_score_eligible") is True), + "missing_gate_count_matches_final_report": remediation_summary.get("missing_gate_count") + == len(final_report_missing_gates), + "remediation_count_matches_final_report": remediation_summary.get("remediation_count") + == len(final_report_remediations), + "action_gate_count_matches_final_report": len(remediation_summary_action_gates) + == len(final_report_remediations), + "remediation_gates_match_missing_gates": remediation_summary.get("remediation_gates_match_missing_gates") + is True, + "target_hardware_action_absent_after_pass": "target_hardware" not in remediation_summary_action_gates, + "command_log_action_absent_after_pass": "command_log_manifest_present" not in remediation_summary_action_gates, + }, + "promotion_audit": { + "validator_passed": not promotion_audit_errors, + "review_ready_true": promotion_audit.get("promotion_review_ready") is True, + "scorecard_update_disallowed": promotion_audit.get("scorecard_update_allowed") is False, + "missing_requirements_empty": promotion_audit.get("missing_requirements") == [], + "required_command_text_matches": ( + (promotion_audit.get("checks") or {}) + .get("command_log_manifest", {}) + .get("required_command_text_matches") + is True + ), + "final_report_command_text_matches": ( + (promotion_audit.get("checks") or {}) + .get("command_log_manifest", {}) + .get("final_report_command_text_matches") + is True + ), + "final_report_path_gate_satisfied": ( + (promotion_audit.get("checks") or {}) + .get("final_report", {}) + .get("path_matches_target_default") + is True + ), + "command_log_manifest_path_gate_satisfied": ( + (promotion_audit.get("checks") or {}) + .get("command_log_manifest", {}) + .get("path_matches_target_default") + is True + ), + "scorecard_path_gate_satisfied": ( + (promotion_audit.get("checks") or {}) + .get("scorecard", {}) + .get("path_matches_target_default") + is True + ), + "claim_ledger_path_gate_satisfied": ( + (promotion_audit.get("checks") or {}) + .get("claim_ledger", {}) + .get("path_matches_target_default") + is True + ), + "output_path_gate_satisfied": ( + (promotion_audit.get("checks") or {}) + .get("promotion_audit_output", {}) + .get("path_matches_target_default") + is True + ), + "score_state_pre_review": promotion_audit.get("scorecard_state", {}).get("current_verified_points") == 920, + "m12_state_pre_review_unawarded": promotion_audit.get("scorecard_state", {}).get("m12_verified_points") == 0, + "input_hashes_current": _promotion_audit_input_hashes_current( + promotion_audit=promotion_audit, + scorecard_path=scorecard_path, + claim_ledger_path=claim_ledger_path, + final_report_path=final_report_path, + ), + }, + } + missing_checks = _bool_checks_missing(sections) + consistency_errors = list(missing_checks) + consistency_errors.extend(f"readiness_summary_validator.{error}" for error in readiness_summary_errors) + consistency_errors.extend(f"transfer_summary_validator.{error}" for error in transfer_summary_errors) + consistency_errors.extend(f"archive_validator.{error}" for error in archive_errors) + consistency_errors.extend(f"archive_verify_report_validator.{error}" for error in archive_verify_report_errors) + consistency_errors.extend(f"overlay_validator.{error}" for error in overlay_errors) + consistency_errors.extend(f"bootstrap_validator.{error}" for error in bootstrap_consistency_errors) + consistency_errors.extend(f"preflight_validator.{error}" for error in preflight_errors) + consistency_errors.extend(f"final_report_validator.{error}" for error in final_report_errors) + consistency_errors.extend(f"remediation_summary_validator.{error}" for error in remediation_summary_errors) + consistency_errors.extend(f"promotion_audit_validator.{error}" for error in promotion_audit_errors) + consistent = not consistency_errors + inputs = { + "scorecard": str(scorecard_path), + "claim_ledger": str(claim_ledger_path), + "blocked_report": str(blocked_report_path), + "handoff": str(handoff_path), + "readiness_summary": str(readiness_summary_path), + "transfer_summary": str(transfer_summary_path), + "transfer_manifest": str(transfer_manifest_path), + "archive_manifest": str(archive_manifest_path), + "archive_verify_report": str(archive_verify_report_path), + "overlay_report": str(overlay_report_path), + "bootstrap_report": str(bootstrap_report_path), + "preflight": str(preflight_path), + "final_report": str(final_report_path), + "remediation_summary": str(remediation_summary_path), + "promotion_audit": str(promotion_audit_path), + } + return { + "report_id": EVIDENCE_CONSISTENCY_ID, + "claim_boundary": EVIDENCE_CONSISTENCY_CLAIM_BOUNDARY, + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "consistent": consistent, + "errors": consistency_errors, + "checks": sections, + "counts": { + "scorecard_current_verified_points": scorecard.get("current_verified_points"), + "scorecard_total_points": scorecard.get("total_points"), + "m12_verified_points": m12.get("verified_points"), + "m12_points": m12.get("points"), + "transfer_artifacts": artifact_count, + "transfer_commands": command_count, + "transfer_expected_outputs": expected_output_count, + "archive_entries": archive_manifest.get("included_entry_count"), + "archive_verify_entries": archive_verify_report.get("included_entry_count"), + "overlay_would_write": overlay_report.get("would_write_count"), + "overlay_existing_destinations": overlay_report.get("existing_destination_count"), + "bootstrap_overlay_would_write": (bootstrap_report.get("overlay") or {}).get("would_write_count"), + "bootstrap_dirty_checkout_mode": bootstrap_report.get("dirty_checkout_mode"), + "local_final_missing_gates": len(final_report.get("missing_gates") or []), + "local_remediation_summary_actions": len(remediation_summary.get("next_target_actions") or []), + "local_promotion_missing_requirements": len(promotion_audit.get("missing_requirements") or []), + }, + "input_sha256": { + key: _sha256_file(Path(path)) for key, path in inputs.items() if Path(path).is_file() + }, + "inputs": inputs, + "operator_next_step": ( + "If consistent is true, the local M12 blocked-state evidence is internally aligned. " + "This report does not award M12 points; target 8xMI300X evidence is still required." + ), + } + + +def validate_m12_evidence_consistency_report(report: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != EVIDENCE_CONSISTENCY_ID: + errors.append("report_id must be bb_zyphra_rl_phase1_m12_evidence_consistency_v1") + if report.get("claim_boundary") != EVIDENCE_CONSISTENCY_CLAIM_BOUNDARY: + errors.append("claim_boundary must remain m12_evidence_consistency_not_scorecard_update") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if report.get("m12_points_awarded") is not False: + errors.append("m12_points_awarded must be false") + if not isinstance(report.get("errors"), list): + errors.append("errors must be a list") + if not isinstance(report.get("checks"), dict) or not report["checks"]: + errors.append("checks must be a non-empty object") + check_errors, expected_check_errors, known_check_ids = _evidence_check_summary(report) + errors.extend(check_errors) + observed_errors = [str(item) for item in report.get("errors", [])] if isinstance(report.get("errors"), list) else [] + if isinstance(report.get("errors"), list): + missing_check_errors = sorted(set(expected_check_errors) - set(observed_errors)) + stale_check_errors = sorted( + error for error in observed_errors if error in known_check_ids and error not in expected_check_errors + ) + unknown_errors = sorted( + error + for error in observed_errors + if error not in known_check_ids and not error.endswith(".__section__") and "_validator." not in error + ) + if missing_check_errors: + errors.append("errors must include every failed embedded check") + if stale_check_errors: + errors.append("errors must not include passed embedded checks") + if unknown_errors: + errors.append("errors contains unknown entries") + if report.get("consistent") is not (not observed_errors): + errors.append("consistent must match evidence-consistency errors") + if bool(report.get("consistent")) and report.get("errors"): + errors.append("consistent cannot be true while errors is non-empty") + if bool(report.get("consistent")): + for section, checks in (report.get("checks") or {}).items(): + if not isinstance(checks, dict): + errors.append(f"checks.{section} must be an object") + continue + for name, passed in checks.items(): + if passed is not True: + errors.append(f"consistent report requires checks.{section}.{name}=true") + return errors + + +def write_m12_evidence_consistency_report(*, phase_dir: Path, output_path: Path) -> dict[str, Any]: + report = build_m12_evidence_consistency_report(phase_dir=phase_dir) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report diff --git a/breadboard/rl/m12/final_report.py b/breadboard/rl/m12/final_report.py new file mode 100644 index 00000000..8d9c3958 --- /dev/null +++ b/breadboard/rl/m12/final_report.py @@ -0,0 +1,2052 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path, PureWindowsPath +from typing import Any + + +FINAL_REPORT_ID = "bb_zyphra_rl_phase1_m12_final_report_v1" +FINAL_REPORT_CLAIM_BOUNDARY = "m12_target_validation_candidate_not_scorecard_update" +ARCHIVE_VERIFY_REPORT_ID = "bb_zyphra_rl_phase1_m12_archive_verify_report_v1" +ARCHIVE_VERIFY_CLAIM_BOUNDARY = "transfer_archive_verification_not_m12_validation" +REQUIRED_LOAD_LEVELS = [5, 20, 50] +OPTIONAL_LOAD_LEVELS = [100] +MIN_SOAK_SECONDS = 2 * 60 * 60 +COMMAND_LOG_MANIFEST_ID = "bb_zyphra_rl_phase1_m12_command_log_manifest_v1" +REQUIRED_COMMAND_LOG_IDS = [ + "target_transfer_archive_verify", + "phase1_validation_suite", + "target_preflight", + "target_swe_probe", + "target_verl_export", + "target_ray_warm_pool", + "target_load_ladder", + "target_soak", +] +REQUIRED_COMMAND_LOG_COMMANDS = { + "target_transfer_archive_verify": ( + "python scripts/rl_phase1/verify_m12_transfer_archive.py " + "--manifest ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep/m12_transfer_archive_manifest.json " + "--output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_archive_verify/m12_archive_verify_report.json" + ), + "target_preflight": ( + "python scripts/rl_phase1/run_m12_preflight.py " + "--output-dir ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_target_preflight --require-pass" + ), + "phase1_validation_suite": "python -m pytest tests/test_rl_phase1_scorecard_schema.py tests/test_rl_phase1_claim_ledger.py tests/rl -q", + "target_swe_probe": ( + "python scripts/rl_phase1/run_swe_probe.py --package examples/rl_env_packages/swe_toy_patch/env_package.yaml " + "--output-dir ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_swe_probe --run-id m12_node_swe_probe --limit 10" + ), + "target_verl_export": ( + "python scripts/rl_phase1/export_verl_probe.py " + "--m6-summary ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_swe_probe/run_summary.json " + "--output-dir ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_verl_probe" + ), + "target_ray_warm_pool": ( + "python scripts/rl_phase1/run_ray_warm_pool_probe.py " + "--package examples/rl_env_packages/python_console_toy/env_package.yaml " + "--output-dir ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe --limit 20 --num-workers 20 --distributed" + ), + "target_load_ladder": ( + "python scripts/rl_phase1/run_m12_load_ladder.py --package examples/rl_env_packages/python_console_toy/env_package.yaml " + "--output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_load_ladder/load_ladder_report.json" + ), + "target_soak": ( + "python scripts/rl_phase1/run_m12_soak.py --package examples/rl_env_packages/python_console_toy/env_package.yaml " + "--output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_soak/soak_report.json" + ), +} +OPTIONAL_COMMAND_LOG_COMMANDS = { + "final_report": ( + "python scripts/rl_phase1/build_m12_final_report.py " + "--output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json " + "--archive-verify-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_archive_verify/m12_archive_verify_report.json " + "--preflight-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_target_preflight/m12_preflight_report.json " + "--swe-run-summary ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_swe_probe/run_summary.json " + "--verl-smoke-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_verl_probe/smoke_consumer_report.json " + "--ray-probe-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe/ray_probe_report.json " + "--warm-vs-cold-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe/warm_vs_cold_report.json " + "--load-ladder-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_load_ladder/load_ladder_report.json " + "--soak-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_soak/soak_report.json " + "--command-log-manifest ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json " + "--require-eligible" + ), + "promotion_audit": ( + "python scripts/rl_phase1/audit_m12_score_promotion.py " + "--output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json " + "--final-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json " + "--scorecard ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml " + "--claim-ledger ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md " + "--command-log-manifest ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json " + "--require-ready" + ), +} +TARGET_ARTIFACT_PATHS = { + "archive_verify_report": "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_archive_verify/m12_archive_verify_report.json", + "preflight_report": "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_target_preflight/m12_preflight_report.json", + "swe_run_summary": "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_swe_probe/run_summary.json", + "verl_smoke_report": "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_verl_probe/smoke_consumer_report.json", + "ray_probe_report": "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe/ray_probe_report.json", + "warm_vs_cold_report": "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe/warm_vs_cold_report.json", + "load_ladder_report": "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_load_ladder/load_ladder_report.json", + "soak_report": "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_soak/soak_report.json", + "command_log_manifest": "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json", +} +TARGET_COMMAND_LOG_MANIFEST_PATH = TARGET_ARTIFACT_PATHS["command_log_manifest"] +TARGET_SCORECARD_PATH = "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" +TARGET_CLAIM_LEDGER_PATH = "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" +TARGET_FINAL_REPORT_PATH = "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json" +TARGET_PROMOTION_AUDIT_PATH = "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json" +M12_FINAL_REPORT_GATE_NAMES = { + "artifact_paths_match_target_defaults", + "archive_verify_report_present", + "archive_verify_report_readable", + "archive_verify_report_valid", + "archive_verify_status_passed", + "preflight_report_readable", + "preflight_report_valid", + "swe_run_summary_readable", + "swe_run_summary_valid", + "verl_smoke_report_readable", + "verl_smoke_report_valid", + "ray_probe_report_readable", + "ray_probe_report_valid", + "warm_vs_cold_report_readable", + "warm_vs_cold_report_valid", + "preflight_passed", + "preflight_runtime_fingerprint_present", + "target_hardware", + "verl_available", + "ray_available", + "inference_engine_available", + "container_runtime_available", + "filesystem_cas_smoke_passed", + "swe_probe_ran_10_rows", + "swe_probe_has_accepted_rows", + "swe_probe_no_unknown_status", + "verl_jsonl_tensorizable", + "verl_parquet_tensorizable", + "verl_row_count_matches_swe", + "ray_probe_ran_10_rows", + "ray_probe_has_workers", + "ray_probe_distributed", + "warm_vs_cold_has_total_ms", + "load_ladder_report_present", + "load_ladder_report_valid", + "load_ladder_required_levels_passed", + "load_ladder_100_attempted_or_skipped", + "load_ladder_policy_integrity", + "load_ladder_no_queue_corruption", + "load_ladder_distributed", + "soak_report_present", + "soak_report_valid", + "soak_status_passed", + "soak_duration_at_least_2h", + "soak_no_runtime_failures", + "soak_distributed", + "command_log_manifest_present", + "command_log_manifest_id_valid", + "command_log_manifest_valid", + "command_log_required_ids_canonical", + "command_log_manifest_complete", + "command_log_hashes_present", + "command_log_hashes_verified", + "command_log_commands_passed", + "command_log_required_logs_archived_summary", + "command_log_required_commands_passed_summary", + "command_log_single_target_run_id", + "command_log_expected_commands_match", + "target_artifact_run_id_binding", +} +RUNTIME_FINGERPRINT_ID = "bb_zyphra_rl_phase1_m12_runtime_fingerprint_v1" +RUNTIME_FINGERPRINT_ENV_POLICY = "allowlist_only_redact_path_values_no_secret_keys_no_absolute_python_paths" +RUNTIME_FINGERPRINT_REDACTED_ABSOLUTE_PATH = "" +RUNTIME_FINGERPRINT_REDACTION_REASON_ABSOLUTE_PATH = "absolute_or_home_path" +RUNTIME_FINGERPRINT_ALLOWED_ENV_KEYS = { + "CUDA_VISIBLE_DEVICES", + "HIP_VISIBLE_DEVICES", + "ROCR_VISIBLE_DEVICES", + "HSA_VISIBLE_DEVICES", + "RAY_ADDRESS", + "CONDA_DEFAULT_ENV", +} +RUNTIME_FINGERPRINT_FORBIDDEN_ENV_KEY_PARTS = ("KEY", "TOKEN", "SECRET", "PASSWORD", "AUTH") +RUNTIME_FINGERPRINT_KEYS = { + "fingerprint_id", + "gpu_summary", + "platform", + "python_modules", + "ray_summary", + "sanitized_environment", + "sha256", + "tool_presence", +} +RUNTIME_FINGERPRINT_PLATFORM_KEYS = { + "machine", + "python", + "python_executable_name", + "python_implementation", + "release", + "system", +} +RUNTIME_FINGERPRINT_TOOL_PRESENCE_KEYS = { + "docker", + "firecracker", + "gvisor_runsc", + "rocm_smi", + "rocminfo", +} +RUNTIME_FINGERPRINT_PYTHON_MODULE_KEYS = {"ray", "sglang", "torch", "verl", "vllm"} +RUNTIME_FINGERPRINT_GPU_SUMMARY_KEYS = {"rocm_smi_output", "torch_probe"} +RUNTIME_FINGERPRINT_RAY_SUMMARY_KEYS = {"status_output"} +RUNTIME_FINGERPRINT_SANITIZED_ENVIRONMENT_KEYS = {"keys", "policy", "redactions", "values"} + + +def _read_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _read_json_object(path: Path) -> dict[str, Any]: + if not path.exists(): + return { + "present": False, + "path": str(path), + "read_error": "FileNotFoundError: JSON artifact is missing", + } + try: + payload = _read_json(path) + except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc: + return { + "present": True, + "path": str(path), + "read_error": f"{exc.__class__.__name__}: {exc}", + } + if not isinstance(payload, dict): + return { + "present": True, + "path": str(path), + "read_error": f"expected JSON object, got {type(payload).__name__}", + } + payload.setdefault("present", True) + payload.setdefault("path", str(path)) + return payload + + +def _read_optional_json(path: Path | None) -> dict[str, Any]: + if path is None: + return {"present": False, "path": None} + return _read_json_object(path) + + +def _read_error(payload: dict[str, Any]) -> str | None: + error = payload.get("read_error") + return str(error) if error else None + + +def _archive_verify_validation_errors(report: dict[str, Any]) -> list[str]: + if report.get("present") is not True: + return [] + if report.get("read_error"): + return [f"archive verify report unreadable: {report.get('read_error')}"] + errors: list[str] = [] + if report.get("report_id") != ARCHIVE_VERIFY_REPORT_ID: + errors.append(f"report_id must be {ARCHIVE_VERIFY_REPORT_ID}") + if report.get("claim_boundary") != ARCHIVE_VERIFY_CLAIM_BOUNDARY: + errors.append(f"claim_boundary must remain {ARCHIVE_VERIFY_CLAIM_BOUNDARY}") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if report.get("m12_points_awarded") is not False: + errors.append("m12_points_awarded must be false") + if report.get("status") not in {"passed", "failed"}: + errors.append("status must be passed or failed") + if report.get("archive_manifest_id") != "bb_zyphra_rl_phase1_m12_transfer_archive_manifest_v1": + errors.append("archive_manifest_id must be bb_zyphra_rl_phase1_m12_transfer_archive_manifest_v1") + if report.get("archive_claim_boundary") != "transfer_archive_only_not_m12_validation": + errors.append("archive_claim_boundary must remain transfer_archive_only_not_m12_validation") + if not str(report.get("archive_sha256") or "").startswith("sha256:"): + errors.append("archive_sha256 must start with sha256:") + if _int_or_zero(report.get("included_entry_count")) <= 0: + errors.append("included_entry_count must be positive") + for field in [ + "all_required_artifacts_present", + "all_transfer_requirements_covered", + "archive_contains_source_overlay", + "archive_deterministic", + "source_paths_portable", + ]: + if report.get(field) is not True: + errors.append(f"{field} must be true") + if report.get("manifest_read_error"): + errors.append("manifest_read_error must be empty") + report_errors = report.get("errors") + if not isinstance(report_errors, list): + errors.append("errors must be a list") + elif report.get("status") == "passed" and report_errors != []: + errors.append("passed report must have no errors") + return errors + + +def _artifact_paths_match_target_defaults(artifact_paths: dict[str, Any]) -> bool: + return { + key: str(value) + for key, value in artifact_paths.items() + if value is not None + } == TARGET_ARTIFACT_PATHS + + +def _gate_remediation(gate: str) -> dict[str, Any]: + archive_verify_gates = { + "archive_verify_report_present", + "archive_verify_report_readable", + "archive_verify_report_valid", + "archive_verify_status_passed", + } + preflight_gates = { + "preflight_report_readable", + "preflight_report_valid", + "preflight_passed", + "preflight_runtime_fingerprint_present", + "target_hardware", + "verl_available", + "ray_available", + "inference_engine_available", + "container_runtime_available", + "filesystem_cas_smoke_passed", + } + swe_gates = { + "swe_run_summary_readable", + "swe_run_summary_valid", + "swe_probe_ran_10_rows", + "swe_probe_has_accepted_rows", + "swe_probe_no_unknown_status", + } + verl_gates = { + "verl_smoke_report_readable", + "verl_smoke_report_valid", + "verl_jsonl_tensorizable", + "verl_parquet_tensorizable", + "verl_row_count_matches_swe", + } + ray_gates = { + "ray_probe_report_readable", + "ray_probe_report_valid", + "ray_probe_ran_10_rows", + "ray_probe_has_workers", + "ray_probe_distributed", + } + warm_gates = { + "warm_vs_cold_report_readable", + "warm_vs_cold_report_valid", + "warm_vs_cold_has_total_ms", + } + load_gates = { + "load_ladder_report_present", + "load_ladder_report_valid", + "load_ladder_required_levels_passed", + "load_ladder_100_attempted_or_skipped", + "load_ladder_policy_integrity", + "load_ladder_no_queue_corruption", + "load_ladder_distributed", + } + soak_gates = { + "soak_report_present", + "soak_report_valid", + "soak_status_passed", + "soak_duration_at_least_2h", + "soak_no_runtime_failures", + "soak_distributed", + } + + if gate == "artifact_paths_match_target_defaults": + return { + "gate": gate, + "blocking_stage": "final_report", + "target_action_id": "final_report", + "required_artifact_path": None, + "operator_action": ( + "Build the final report from the target-node default M12 artifact paths generated in " + "m12_test_commands.sh; local M6/M7/M8 preparation paths are intentionally non-promotable." + ), + } + if gate in archive_verify_gates: + return { + "gate": gate, + "blocking_stage": "target_transfer_archive_verify", + "target_action_id": "target_transfer_archive_verify", + "required_artifact_path": TARGET_ARTIFACT_PATHS["archive_verify_report"], + "operator_action": ( + "Run target_transfer_archive_verify through m12_test_commands.sh and preserve " + "m12_archive_verify/m12_archive_verify_report.json. The report must be readable, passed, " + "non-scoring, and consistent with the transfer archive manifest before target validation can proceed." + ), + } + if gate in preflight_gates: + return { + "gate": gate, + "blocking_stage": "target_preflight", + "target_action_id": "target_preflight", + "required_artifact_path": TARGET_ARTIFACT_PATHS["preflight_report"], + "operator_action": ( + "Run target_preflight through run_m12_logged_command.py on the 8xMI300X target and resolve " + "ROCm/GPU, Ray, VeRL, container, filesystem/CAS, or runtime-fingerprint blockers before continuing." + ), + } + if gate in swe_gates: + return { + "gate": gate, + "blocking_stage": "target_swe_probe", + "target_action_id": "target_swe_probe", + "required_artifact_path": TARGET_ARTIFACT_PATHS["swe_run_summary"], + "operator_action": ( + "Run the target SWE probe from m12_test_commands.sh and preserve the run summary with at least " + "10 rows, known row statuses, and at least one accepted hardened row." + ), + } + if gate in verl_gates: + return { + "gate": gate, + "blocking_stage": "target_verl_export", + "target_action_id": "target_verl_export", + "required_artifact_path": TARGET_ARTIFACT_PATHS["verl_smoke_report"], + "operator_action": ( + "Run the target VeRL export probe from the target SWE summary and fix JSONL/Parquet tensorization " + "or row-count drift before promotion review." + ), + } + if gate in ray_gates: + return { + "gate": gate, + "blocking_stage": "target_ray_warm_pool", + "target_action_id": "target_ray_warm_pool", + "required_artifact_path": TARGET_ARTIFACT_PATHS["ray_probe_report"], + "operator_action": ( + "Run the distributed Ray warm-worker probe from m12_test_commands.sh; local_mode output is a " + "rehearsal artifact and cannot satisfy M12." + ), + } + if gate in warm_gates: + return { + "gate": gate, + "blocking_stage": "target_ray_warm_pool", + "target_action_id": "target_ray_warm_pool", + "required_artifact_path": TARGET_ARTIFACT_PATHS["warm_vs_cold_report"], + "operator_action": ( + "Re-run or repair the Ray warm-worker comparison until the warm-vs-cold report is readable, valid, " + "and includes total_ms summaries." + ), + } + if gate in load_gates: + return { + "gate": gate, + "blocking_stage": "target_load_ladder", + "target_action_id": "target_load_ladder", + "required_artifact_path": TARGET_ARTIFACT_PATHS["load_ladder_report"], + "operator_action": ( + "Run the concrete target load ladder in distributed mode. Levels 5, 20, and 50 must pass; level 100 " + "must either pass or be resource-skipped with a reason, with policy and queue integrity preserved." + ), + } + if gate in soak_gates: + return { + "gate": gate, + "blocking_stage": "target_soak", + "target_action_id": "target_soak", + "required_artifact_path": TARGET_ARTIFACT_PATHS["soak_report"], + "operator_action": ( + "Run the concrete target soak in distributed mode for at least 7200 seconds and preserve a passed " + "report with zero runtime failures." + ), + } + if gate.startswith("command_log_"): + return { + "gate": gate, + "blocking_stage": "command_log_manifest", + "target_action_id": "m12_test_commands.sh", + "required_artifact_path": TARGET_ARTIFACT_PATHS["command_log_manifest"], + "operator_action": ( + "Run every target command through run_m12_logged_command.py from m12_test_commands.sh, preserve raw " + "logs and hashes, keep canonical required_command_ids, and use one target_run_id across required rows." + ), + } + if gate == "target_artifact_run_id_binding": + return { + "gate": gate, + "blocking_stage": "target_artifact_identity", + "target_action_id": "m12_test_commands.sh", + "required_artifact_path": TARGET_ARTIFACT_PATHS["command_log_manifest"], + "operator_action": ( + "Run all target-producing commands through m12_test_commands.sh so run_m12_logged_command.py exports " + "M12_TARGET_RUN_ID to child processes, then rebuild stale preflight/SWE/export/Ray/load/soak artifacts " + "until every component artifact records the same target_run_id as the required command logs." + ), + } + return { + "gate": gate, + "blocking_stage": "unknown", + "target_action_id": None, + "required_artifact_path": None, + "operator_action": "Unknown M12 gate; update final-report gate remediation mapping before relying on this report.", + } + + +def _missing_gate_remediations(missing_gates: list[str]) -> list[dict[str, Any]]: + return [_gate_remediation(gate) for gate in missing_gates] + + +def summarize_m12_final_report_remediations(report: dict[str, Any]) -> dict[str, Any]: + remediations = [ + remediation + for remediation in report.get("missing_gate_remediations") or [] + if isinstance(remediation, dict) + ] + by_target_action: dict[str, dict[str, Any]] = {} + for remediation in remediations: + action = str(remediation.get("target_action_id") or remediation.get("blocking_stage") or "unknown") + item = by_target_action.setdefault( + action, + { + "target_action_id": action, + "blocking_stages": [], + "gates": [], + "required_artifact_paths": [], + "operator_actions": [], + }, + ) + stage = str(remediation.get("blocking_stage") or "") + gate = str(remediation.get("gate") or "") + artifact_path = remediation.get("required_artifact_path") + operator_action = str(remediation.get("operator_action") or "") + if stage and stage not in item["blocking_stages"]: + item["blocking_stages"].append(stage) + if gate and gate not in item["gates"]: + item["gates"].append(gate) + if artifact_path and artifact_path not in item["required_artifact_paths"]: + item["required_artifact_paths"].append(artifact_path) + if operator_action and operator_action not in item["operator_actions"]: + item["operator_actions"].append(operator_action) + missing_gates = [str(gate) for gate in report.get("missing_gates") or []] + remediation_gates = [str(remediation.get("gate") or "") for remediation in remediations] + return { + "summary_id": "bb_zyphra_rl_phase1_m12_final_report_remediation_summary_v1", + "claim_boundary": "final_report_remediation_summary_not_scorecard_update", + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "m12_score_eligible": report.get("m12_score_eligible") is True, + "missing_gate_count": len(missing_gates), + "remediation_count": len(remediations), + "remediation_gates_match_missing_gates": remediation_gates == missing_gates, + "next_target_actions": list(by_target_action.values()), + "operator_next_step": ( + "If m12_score_eligible is false, run or repair the listed target actions and rebuild the final report. " + "This summary never updates the scorecard." + ), + } + + +def validate_m12_final_report_remediation_summary(summary: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if summary.get("summary_id") != "bb_zyphra_rl_phase1_m12_final_report_remediation_summary_v1": + errors.append("summary_id must be bb_zyphra_rl_phase1_m12_final_report_remediation_summary_v1") + if summary.get("claim_boundary") != "final_report_remediation_summary_not_scorecard_update": + errors.append("claim_boundary must remain final_report_remediation_summary_not_scorecard_update") + if summary.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if summary.get("m12_points_awarded") is not False: + errors.append("m12_points_awarded must be false") + if not isinstance(summary.get("m12_score_eligible"), bool): + errors.append("m12_score_eligible must be boolean") + if not isinstance(summary.get("missing_gate_count"), int) or summary.get("missing_gate_count", -1) < 0: + errors.append("missing_gate_count must be a non-negative integer") + if not isinstance(summary.get("remediation_count"), int) or summary.get("remediation_count", -1) < 0: + errors.append("remediation_count must be a non-negative integer") + if not isinstance(summary.get("remediation_gates_match_missing_gates"), bool): + errors.append("remediation_gates_match_missing_gates must be boolean") + actions = summary.get("next_target_actions") + if not isinstance(actions, list): + errors.append("next_target_actions must be a list") + actions = [] + gate_rows: list[str] = [] + for index, action in enumerate(actions, start=1): + if not isinstance(action, dict): + errors.append(f"next_target_actions row {index} must be an object") + continue + if not str(action.get("target_action_id") or "").strip(): + errors.append(f"next_target_actions row {index} requires target_action_id") + for field in ["blocking_stages", "gates", "required_artifact_paths", "operator_actions"]: + if not isinstance(action.get(field), list): + errors.append(f"next_target_actions row {index} {field} must be a list") + gates = action.get("gates") if isinstance(action.get("gates"), list) else [] + for gate in gates: + gate_name = str(gate) + if gate_name not in M12_FINAL_REPORT_GATE_NAMES: + errors.append(f"next_target_actions row {index} has unknown gate: {gate_name}") + gate_rows.append(gate_name) + artifact_paths = ( + action.get("required_artifact_paths") + if isinstance(action.get("required_artifact_paths"), list) + else [] + ) + for artifact_path in artifact_paths: + if str(artifact_path) not in TARGET_ARTIFACT_PATHS.values(): + errors.append(f"next_target_actions row {index} has non-target artifact path: {artifact_path}") + operator_actions = action.get("operator_actions") if isinstance(action.get("operator_actions"), list) else [] + for operator_action in operator_actions: + if not str(operator_action or "").strip(): + errors.append(f"next_target_actions row {index} has empty operator_action") + if len(gate_rows) != len(set(gate_rows)): + errors.append("next_target_actions gates must be unique across target actions") + if isinstance(summary.get("remediation_count"), int) and summary["remediation_count"] != len(gate_rows): + errors.append("remediation_count must match next_target_actions gate count") + if summary.get("remediation_gates_match_missing_gates") is True: + if isinstance(summary.get("missing_gate_count"), int) and summary["missing_gate_count"] != len(gate_rows): + errors.append("missing_gate_count must match next_target_actions gate count") + if summary.get("remediation_count") != summary.get("missing_gate_count"): + errors.append("remediation_count must match missing_gate_count when gates match") + if summary.get("m12_score_eligible") is True: + if summary.get("missing_gate_count") != 0: + errors.append("eligible remediation summary requires missing_gate_count=0") + if summary.get("remediation_count") != 0: + errors.append("eligible remediation summary requires remediation_count=0") + if gate_rows: + errors.append("eligible remediation summary requires no next_target_actions gates") + if summary.get("remediation_gates_match_missing_gates") is not True: + errors.append("eligible remediation summary requires remediation_gates_match_missing_gates=true") + if not str(summary.get("operator_next_step") or "").strip(): + errors.append("operator_next_step must be non-empty") + return errors + + +def _preflight_validation_errors(preflight: dict[str, Any]) -> list[str]: + if _read_error(preflight): + return [f"preflight report unreadable: {_read_error(preflight)}"] + from breadboard.rl.m12.preflight import validate_m12_preflight_report + + return validate_m12_preflight_report(preflight) + + +def _load_ladder_validation_errors(load_ladder: dict[str, Any]) -> list[str]: + if load_ladder.get("present") is not True: + return [] + if _read_error(load_ladder): + return [f"load ladder report unreadable: {_read_error(load_ladder)}"] + from breadboard.rl.m12.load_soak import validate_m12_load_ladder_report + + return validate_m12_load_ladder_report(load_ladder) + + +def _soak_validation_errors(soak: dict[str, Any]) -> list[str]: + if soak.get("present") is not True: + return [] + if _read_error(soak): + return [f"soak report unreadable: {_read_error(soak)}"] + from breadboard.rl.m12.load_soak import validate_m12_soak_report + + return validate_m12_soak_report(soak) + + +def _swe_summary_validation_errors(summary: dict[str, Any]) -> list[str]: + if _read_error(summary): + return [f"SWE run summary unreadable: {_read_error(summary)}"] + errors: list[str] = [] + for key in ["run_id", "package_id", "package_hash", "source_claim"]: + if not str(summary.get(key) or "").strip(): + errors.append(f"SWE run summary requires non-empty {key}") + rows = summary.get("rows") + if not isinstance(rows, list) or not rows: + errors.append("SWE run summary requires non-empty rows list") + return errors + task_ids: set[str] = set() + metric_keys = {"export_ms", "reset_ms", "step_ms", "total_ms", "verify_ms"} + allowed_statuses = {"accepted", "rejected", "quarantined"} + for index, row in enumerate(rows, start=1): + if not isinstance(row, dict): + errors.append(f"SWE row {index} must be an object") + continue + task_id = str(row.get("task_id") or "").strip() + if not task_id: + errors.append(f"SWE row {index} requires task_id") + elif task_id in task_ids: + errors.append(f"SWE row task_id must be unique: {task_id}") + task_ids.add(task_id) + status = str(row.get("row_status") or "") + if status not in allowed_statuses: + errors.append(f"SWE row {task_id or index} has invalid row_status") + if not isinstance(row.get("exportable_debug"), bool): + errors.append(f"SWE row {task_id or index} requires boolean exportable_debug") + if not isinstance(row.get("trainable"), bool): + errors.append(f"SWE row {task_id or index} requires boolean trainable") + if not str(row.get("projection_id") or "").strip(): + errors.append(f"SWE row {task_id or index} requires projection_id") + if not isinstance(row.get("blocked_reasons"), list): + errors.append(f"SWE row {task_id or index} requires blocked_reasons list") + if not isinstance(row.get("findings"), list): + errors.append(f"SWE row {task_id or index} requires findings list") + try: + float(row.get("reward")) + except (TypeError, ValueError): + errors.append(f"SWE row {task_id or index} requires numeric reward") + metrics = row.get("metrics_ms") + if not isinstance(metrics, dict) or set(metrics) != metric_keys: + errors.append(f"SWE row {task_id or index} requires exact metrics_ms keys") + else: + for metric_key, metric_value in metrics.items(): + try: + if float(metric_value) < 0: + errors.append(f"SWE row {task_id or index} metric {metric_key} must be non-negative") + except (TypeError, ValueError): + errors.append(f"SWE row {task_id or index} metric {metric_key} must be numeric") + if status == "accepted": + if row.get("hardening_status") != "passed": + errors.append(f"accepted SWE row {task_id or index} requires hardening_status=passed") + if row.get("replay_status") != "passed": + errors.append(f"accepted SWE row {task_id or index} requires replay_status=passed") + if row.get("exportable_debug") is not True: + errors.append(f"accepted SWE row {task_id or index} requires exportable_debug=true") + if status == "quarantined" and row.get("hardening_status") != "quarantined": + errors.append(f"quarantined SWE row {task_id or index} requires hardening_status=quarantined") + metrics_summary = summary.get("metrics_summary") + if not isinstance(metrics_summary, dict): + errors.append("SWE run summary requires metrics_summary object") + else: + for metric_key in metric_keys: + metric_summary = metrics_summary.get(metric_key) + if not isinstance(metric_summary, dict): + errors.append(f"SWE metrics_summary requires {metric_key}") + continue + for stat_key in ["p50", "p95"]: + try: + if float(metric_summary.get(stat_key)) < 0: + errors.append(f"SWE metrics_summary {metric_key}.{stat_key} must be non-negative") + except (TypeError, ValueError): + errors.append(f"SWE metrics_summary {metric_key}.{stat_key} must be numeric") + qc_report = summary.get("qc_report") + if not isinstance(qc_report, dict): + errors.append("SWE run summary requires qc_report object") + elif "Controlled SWE toy slice only" not in str(qc_report.get("operator_notes") or ""): + errors.append("SWE qc_report must preserve controlled-slice claim boundary note") + return errors + + +def _verl_smoke_validation_errors(smoke: dict[str, Any]) -> list[str]: + if _read_error(smoke): + return [f"VeRL smoke report unreadable: {_read_error(smoke)}"] + errors: list[str] = [] + row_count = _int_or(smoke.get("row_count"), -1) + trainable_count = _int_or(smoke.get("trainable_candidate_count"), -1) + if row_count < 1: + errors.append("VeRL smoke report requires positive row_count") + if trainable_count < 0 or trainable_count > max(row_count, 0): + errors.append("VeRL smoke report trainable_candidate_count must be between 0 and row_count") + if not isinstance(smoke.get("tensorizable"), bool): + errors.append("VeRL smoke report requires boolean tensorizable") + if "not DataProto or trainer execution" not in str(smoke.get("compatibility_target") or ""): + errors.append("VeRL smoke report must preserve non-trainer compatibility boundary") + if not str(smoke.get("projection_manifest_id") or "").strip(): + errors.append("VeRL smoke report requires projection_manifest_id") + errors_by_format = smoke.get("errors") + if not isinstance(errors_by_format, dict): + errors.append("VeRL smoke report requires errors object") + formats = smoke.get("formats") + if not isinstance(formats, dict): + errors.append("VeRL smoke report requires formats object") + return errors + for format_name in ["jsonl", "parquet"]: + report = formats.get(format_name) + if not isinstance(report, dict): + errors.append(f"VeRL smoke report missing {format_name} format report") + continue + if _int_or(report.get("row_count"), -1) != row_count: + errors.append(f"VeRL {format_name} row_count must match top-level row_count") + if _int_or(report.get("trainable_candidate_count"), -1) != trainable_count: + errors.append(f"VeRL {format_name} trainable_candidate_count must match top-level count") + if not isinstance(report.get("tensorizable"), bool): + errors.append(f"VeRL {format_name} requires boolean tensorizable") + report_errors = report.get("errors") + if not isinstance(report_errors, list): + errors.append(f"VeRL {format_name} errors must be a list") + elif report.get("tensorizable") is True and report_errors: + errors.append(f"VeRL {format_name} tensorizable=true requires empty errors") + return errors + + +def _ray_probe_validation_errors(ray_probe: dict[str, Any]) -> list[str]: + if _read_error(ray_probe): + return [f"Ray probe report unreadable: {_read_error(ray_probe)}"] + errors: list[str] = [] + if ray_probe.get("claim_boundary") != "local_ray_worker_probe_not_production_scale": + errors.append("Ray probe claim_boundary must remain local_ray_worker_probe_not_production_scale") + rows = ray_probe.get("rows") + if not isinstance(rows, list) or not rows: + errors.append("Ray probe requires non-empty rows list") + rows = [] + row_count = _int_or(ray_probe.get("row_count"), -1) + if row_count != len(rows): + errors.append("Ray probe row_count must equal rows length") + worker_count = _int_or(ray_probe.get("worker_count"), -1) + if worker_count < 1: + errors.append("Ray probe worker_count must be positive") + if not isinstance(ray_probe.get("ray_local_mode"), bool): + errors.append("Ray probe requires boolean ray_local_mode") + task_ids: set[str] = set() + worker_ids: set[str] = set() + metric_keys = {"reset_ms", "step_ms", "total_ms", "verify_ms"} + for index, row in enumerate(rows, start=1): + if not isinstance(row, dict): + errors.append(f"Ray probe row {index} must be an object") + continue + task_id = str(row.get("task_id") or "").strip() + if not task_id: + errors.append(f"Ray probe row {index} requires task_id") + elif task_id in task_ids: + errors.append(f"Ray probe task_id must be unique: {task_id}") + task_ids.add(task_id) + worker_id = str(row.get("worker_id") or "").strip() + if not worker_id: + errors.append(f"Ray probe row {task_id or index} requires worker_id") + worker_ids.add(worker_id) + if _int_or(row.get("event_count"), -1) < 1: + errors.append(f"Ray probe row {task_id or index} requires positive event_count") + if _int_or(row.get("run_count"), -1) < 1: + errors.append(f"Ray probe row {task_id or index} requires positive run_count") + try: + float(row.get("reward")) + except (TypeError, ValueError): + errors.append(f"Ray probe row {task_id or index} requires numeric reward") + metrics = row.get("metrics_ms") + if not isinstance(metrics, dict) or set(metrics) != metric_keys: + errors.append(f"Ray probe row {task_id or index} requires exact metrics_ms keys") + else: + for metric_key, metric_value in metrics.items(): + try: + if float(metric_value) < 0: + errors.append(f"Ray probe row {task_id or index} metric {metric_key} must be non-negative") + except (TypeError, ValueError): + errors.append(f"Ray probe row {task_id or index} metric {metric_key} must be numeric") + if len(worker_ids) > worker_count: + errors.append("Ray probe observed worker IDs cannot exceed worker_count") + return errors + + +def _warm_vs_cold_validation_errors(report: dict[str, Any]) -> list[str]: + if _read_error(report): + return [f"warm-vs-cold report unreadable: {_read_error(report)}"] + errors: list[str] = [] + if report.get("claim_boundary") != "local_ray_warm_pool_probe_not_production_scale": + errors.append("warm-vs-cold claim_boundary must remain local_ray_warm_pool_probe_not_production_scale") + warm = report.get("warm") + cold = report.get("cold") + if not isinstance(warm, dict) or not isinstance(cold, dict): + errors.append("warm-vs-cold report requires warm and cold objects") + return errors + if set(warm.keys()) != set(cold.keys()): + errors.append("warm-vs-cold warm and cold metric keys must match") + if "total_ms" not in warm or "total_ms" not in cold: + errors.append("warm-vs-cold requires total_ms metric") + for group_name, group in [("warm", warm), ("cold", cold)]: + for metric_name, summary in group.items(): + if not isinstance(summary, dict): + errors.append(f"warm-vs-cold {group_name}.{metric_name} must be an object") + continue + for stat_key in ["count", "p50", "p95"]: + try: + value = float(summary.get(stat_key)) + except (TypeError, ValueError): + errors.append(f"warm-vs-cold {group_name}.{metric_name}.{stat_key} must be numeric") + continue + if value < 0: + errors.append(f"warm-vs-cold {group_name}.{metric_name}.{stat_key} must be non-negative") + try: + if float(summary.get("p95")) < float(summary.get("p50")): + errors.append(f"warm-vs-cold {group_name}.{metric_name}.p95 must be >= p50") + except (TypeError, ValueError): + pass + return errors + + +def _status_counts(rows: list[dict[str, Any]]) -> dict[str, int]: + counts = {"accepted": 0, "rejected": 0, "quarantined": 0, "other": 0} + for row in rows: + status = str(row.get("row_status") or "") + if status in counts: + counts[status] += 1 + else: + counts["other"] += 1 + return counts + + +def _int_or(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _int_or_zero(value: Any) -> int: + return _int_or(value, 0) + + +def _command_log_entries(command_logs: dict[str, Any]) -> list[dict[str, Any]]: + entries = command_logs.get("commands") + if not isinstance(entries, list): + return [] + return [item for item in entries if isinstance(item, dict)] + + +def _command_log_ids(entries: list[dict[str, Any]]) -> set[str]: + return {str(item.get("command_id") or "") for item in entries if item.get("command_id")} + + +def _command_log_entry_has_hash(entry: dict[str, Any]) -> bool: + return str(entry.get("sha256") or "").startswith("sha256:") + + +def _command_log_entry_archived(entry: dict[str, Any]) -> bool: + return bool(entry.get("log_path")) and _command_log_entry_has_hash(entry) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def _resolve_command_log_path(manifest_path: Path | None, raw_log_path: Any) -> Path | None: + if not raw_log_path: + return None + path = Path(str(raw_log_path)) + if path.is_absolute(): + return path + if manifest_path is not None: + manifest_relative = manifest_path.parent / path + if manifest_relative.exists(): + return manifest_relative + return path + + +def _command_log_entry_hash_verified(entry: dict[str, Any], manifest_path: Path | None) -> bool: + if not _command_log_entry_archived(entry): + return False + path = _resolve_command_log_path(manifest_path, entry.get("log_path")) + if path is None or not path.is_file(): + return False + return _sha256_file(path) == entry.get("sha256") + + +def _required_command_target_run_ids(entries: list[dict[str, Any]], required_command_ids: set[str]) -> set[str]: + return { + str(entry.get("target_run_id") or "") + for entry in entries + if str(entry.get("command_id") or "") in required_command_ids + } + + +def _required_command_text_mismatches(entries: list[dict[str, Any]]) -> list[dict[str, str]]: + entries_by_id = {str(entry.get("command_id") or ""): entry for entry in entries} + mismatches: list[dict[str, str]] = [] + for command_id, expected in REQUIRED_COMMAND_LOG_COMMANDS.items(): + entry = entries_by_id.get(command_id) + if entry is None: + continue + observed = str(entry.get("command") or "") + if observed != expected: + mismatches.append({"command_id": command_id, "expected": expected, "observed": observed}) + return mismatches + + +def _target_artifact_run_ids( + *, + preflight: dict[str, Any], + swe_summary: dict[str, Any], + verl_smoke: dict[str, Any], + ray_probe: dict[str, Any], + warm_vs_cold: dict[str, Any], + load_ladder: dict[str, Any], + soak: dict[str, Any], +) -> dict[str, str | None]: + return { + "preflight_report": preflight.get("target_run_id"), + "swe_run_summary": swe_summary.get("target_run_id"), + "verl_smoke_report": verl_smoke.get("target_run_id"), + "ray_probe_report": ray_probe.get("target_run_id"), + "warm_vs_cold_report": warm_vs_cold.get("target_run_id"), + "load_ladder_report": load_ladder.get("target_run_id"), + "soak_report": soak.get("target_run_id"), + } + + +def _target_artifact_run_ids_from_final_report(report: dict[str, Any]) -> dict[str, str | None]: + return _target_artifact_run_ids( + preflight=report.get("preflight", {}) if isinstance(report.get("preflight"), dict) else {}, + swe_summary=report.get("swe_probe", {}) if isinstance(report.get("swe_probe"), dict) else {}, + verl_smoke=report.get("verl_export", {}) if isinstance(report.get("verl_export"), dict) else {}, + ray_probe=report.get("ray_probe", {}) if isinstance(report.get("ray_probe"), dict) else {}, + warm_vs_cold=report.get("warm_vs_cold", {}) if isinstance(report.get("warm_vs_cold"), dict) else {}, + load_ladder=report.get("load_ladder", {}) if isinstance(report.get("load_ladder"), dict) else {}, + soak=report.get("soak", {}) if isinstance(report.get("soak"), dict) else {}, + ) + + +def _target_run_identity_validation_errors(report: dict[str, Any]) -> list[str]: + errors: list[str] = [] + identity = report.get("target_run_identity") + if not isinstance(identity, dict): + return ["target_run_identity must be an object"] + expected_artifact_run_ids = _target_artifact_run_ids_from_final_report(report) + if identity.get("artifact_target_run_ids") != expected_artifact_run_ids: + errors.append("target_run_identity artifact_target_run_ids must match embedded artifact sections") + missing = sorted(name for name, target_run_id in expected_artifact_run_ids.items() if not target_run_id) + if identity.get("missing_artifact_target_run_ids") != missing: + errors.append("target_run_identity missing_artifact_target_run_ids is stale") + command_target_run_id = identity.get("single_command_log_target_run_id") + mismatched = { + name: {"expected": command_target_run_id, "observed": target_run_id} + for name, target_run_id in expected_artifact_run_ids.items() + if target_run_id and command_target_run_id and target_run_id != command_target_run_id + } + if identity.get("mismatched_artifact_target_run_ids") != mismatched: + errors.append("target_run_identity mismatched_artifact_target_run_ids is stale") + expected_match = bool(command_target_run_id) and not missing and not mismatched + if identity.get("run_ids_match_command_logs") is not expected_match: + errors.append("target_run_identity run_ids_match_command_logs is stale") + return errors + + +def _object_or_empty(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _missing_gate_names_from_final_report(report: dict[str, Any]) -> list[str]: + """Recompute final-report gates from the embedded final-report sections.""" + + artifact_paths = _object_or_empty(report.get("artifact_paths")) + archive_verify = _object_or_empty(report.get("archive_verify")) + preflight = _object_or_empty(report.get("preflight")) + swe_probe = _object_or_empty(report.get("swe_probe")) + verl_export = _object_or_empty(report.get("verl_export")) + ray_probe = _object_or_empty(report.get("ray_probe")) + warm_vs_cold = _object_or_empty(report.get("warm_vs_cold")) + load_ladder = _object_or_empty(report.get("load_ladder")) + soak = _object_or_empty(report.get("soak")) + command_logs = _object_or_empty(report.get("command_logs")) + + swe_row_count = _int_or_zero(swe_probe.get("row_count")) + swe_row_counts = _object_or_empty(swe_probe.get("row_status_counts")) + load_level_items = [ + item for item in load_ladder.get("concurrency_levels", []) if isinstance(item, dict) + ] + load_levels = { + _int_or(item.get("target_sessions"), -1): str(item.get("status") or "") + for item in load_level_items + } + resource_skipped_levels = { + _int_or(item.get("target_sessions"), -1) + for item in load_ladder.get("resource_skips", []) + if isinstance(item, dict) + } + + command_entries = _command_log_entries(command_logs) + required_command_ids = set(REQUIRED_COMMAND_LOG_IDS) + archived_command_ids = _command_log_ids( + [entry for entry in command_entries if _command_log_entry_archived(entry)] + ) + hash_verified_command_ids = { + str(command_id) + for command_id in command_logs.get("hash_verified_command_ids", []) + if str(command_id) + } + passed_command_ids = _command_log_ids( + [entry for entry in command_entries if str(entry.get("status") or "") == "passed"] + ) + missing_command_log_ids = sorted(required_command_ids - archived_command_ids) + required_target_run_ids = _required_command_target_run_ids(command_entries, required_command_ids) + required_target_run_ids_without_empty = {item for item in required_target_run_ids if item} + single_target_run_id = ( + next(iter(required_target_run_ids_without_empty)) + if len(required_target_run_ids_without_empty) == 1 and "" not in required_target_run_ids + else None + ) + required_command_text_mismatches = _required_command_text_mismatches(command_entries) + target_artifact_run_ids = _target_artifact_run_ids_from_final_report(report) + missing_target_artifact_run_ids = sorted( + name for name, target_run_id in target_artifact_run_ids.items() if not target_run_id + ) + mismatched_target_artifact_run_ids = { + name: {"expected": single_target_run_id, "observed": target_run_id} + for name, target_run_id in target_artifact_run_ids.items() + if target_run_id and single_target_run_id and target_run_id != single_target_run_id + } + + gates = { + "artifact_paths_match_target_defaults": _artifact_paths_match_target_defaults(artifact_paths), + "archive_verify_report_present": archive_verify.get("present") is True, + "archive_verify_report_readable": archive_verify.get("present") is True and _read_error(archive_verify) is None, + "archive_verify_report_valid": archive_verify.get("present") is True + and not _archive_verify_validation_errors(archive_verify), + "archive_verify_status_passed": archive_verify.get("status") == "passed", + "preflight_report_readable": _read_error(preflight) is None, + "preflight_report_valid": preflight.get("validation_errors") == [], + "swe_run_summary_readable": _read_error(swe_probe) is None, + "swe_run_summary_valid": swe_probe.get("validation_errors") == [], + "verl_smoke_report_readable": _read_error(verl_export) is None, + "verl_smoke_report_valid": verl_export.get("validation_errors") == [], + "ray_probe_report_readable": _read_error(ray_probe) is None, + "ray_probe_report_valid": ray_probe.get("validation_errors") == [], + "warm_vs_cold_report_readable": _read_error(warm_vs_cold) is None, + "warm_vs_cold_report_valid": warm_vs_cold.get("validation_errors") == [], + "preflight_passed": preflight.get("status") == "preflight_passed", + "preflight_runtime_fingerprint_present": _runtime_fingerprint_valid(preflight), + "target_hardware": ( + preflight.get("gpu", {}).get("required_accelerator_count") == 8 + and preflight.get("gpu", {}).get("required_accelerator_family") == "MI300X" + and preflight.get("gpu", {}).get("mi300x_product_evidence") is True + and _int_or_zero(preflight.get("gpu", {}).get("torch_probe", {}).get("device_count")) >= 8 + ), + "verl_available": preflight.get("python_modules", {}).get("verl", {}).get("available") is True, + "ray_available": preflight.get("python_modules", {}).get("ray", {}).get("available") is True, + "inference_engine_available": ( + preflight.get("inference_engine_feasibility", {}).get("decision") == "available" + and ( + preflight.get("inference_engine_feasibility", {}).get("vllm_available") is True + or preflight.get("inference_engine_feasibility", {}).get("sglang_available") is True + ) + ), + "container_runtime_available": any( + value is True for value in (preflight.get("container_runtimes") or {}).values() + ), + "filesystem_cas_smoke_passed": preflight.get("filesystem_cas_smoke", {}).get("status") == "passed", + "swe_probe_ran_10_rows": swe_row_count >= 10, + "swe_probe_has_accepted_rows": _int_or_zero(swe_row_counts.get("accepted")) > 0, + "swe_probe_no_unknown_status": _int_or(swe_row_counts.get("other"), 1) == 0, + "verl_jsonl_tensorizable": verl_export.get("formats", {}).get("jsonl", {}).get("tensorizable") is True, + "verl_parquet_tensorizable": verl_export.get("formats", {}).get("parquet", {}).get("tensorizable") is True, + "verl_row_count_matches_swe": _int_or(verl_export.get("row_count"), -1) == swe_row_count, + "ray_probe_ran_10_rows": _int_or_zero(ray_probe.get("row_count")) >= 10, + "ray_probe_has_workers": _int_or_zero(ray_probe.get("worker_count")) >= 2, + "ray_probe_distributed": ray_probe.get("ray_local_mode") is False, + "warm_vs_cold_has_total_ms": ( + "total_ms" in _object_or_empty(warm_vs_cold.get("warm")) + and "total_ms" in _object_or_empty(warm_vs_cold.get("cold")) + ), + "load_ladder_report_present": load_ladder.get("present") is True, + "load_ladder_report_valid": load_ladder.get("present") is not True or load_ladder.get("validation_errors") == [], + "load_ladder_required_levels_passed": all( + load_levels.get(level) == "passed" for level in REQUIRED_LOAD_LEVELS + ), + "load_ladder_100_attempted_or_skipped": all( + load_levels.get(level) == "passed" or level in resource_skipped_levels for level in OPTIONAL_LOAD_LEVELS + ), + "load_ladder_policy_integrity": load_ladder.get("policy_version_integrity") is True, + "load_ladder_no_queue_corruption": load_ladder.get("queue_backpressure_integrity") is True, + "load_ladder_distributed": all( + item.get("status") == "resource_skipped" or item.get("ray_local_mode") is False + for item in load_level_items + ) + and bool(load_level_items), + "soak_report_present": soak.get("present") is True, + "soak_report_valid": soak.get("present") is not True or soak.get("validation_errors") == [], + "soak_status_passed": soak.get("status") == "passed", + "soak_duration_at_least_2h": _int_or_zero(soak.get("duration_seconds")) >= MIN_SOAK_SECONDS, + "soak_no_runtime_failures": _int_or(soak.get("runtime_failure_count"), 1) == 0, + "soak_distributed": soak.get("ray_local_mode") is False, + "command_log_manifest_present": command_logs.get("present") is True, + "command_log_manifest_id_valid": command_logs.get("manifest_id") == COMMAND_LOG_MANIFEST_ID, + "command_log_manifest_valid": ( + command_logs.get("present") is True + and command_logs.get("manifest_validation_errors") == [] + ), + "command_log_required_ids_canonical": ( + command_logs.get("manifest_required_command_ids") == list(REQUIRED_COMMAND_LOG_IDS) + ), + "command_log_manifest_complete": not missing_command_log_ids, + "command_log_hashes_present": all( + _command_log_entry_has_hash(entry) + for entry in command_entries + if str(entry.get("command_id") or "") in required_command_ids + ) + and not missing_command_log_ids, + "command_log_hashes_verified": all(command_id in hash_verified_command_ids for command_id in required_command_ids), + "command_log_commands_passed": all(command_id in passed_command_ids for command_id in required_command_ids), + "command_log_required_logs_archived_summary": command_logs.get("all_required_logs_archived") is True, + "command_log_required_commands_passed_summary": command_logs.get("all_required_commands_passed") is True, + "command_log_single_target_run_id": not missing_command_log_ids and single_target_run_id is not None, + "command_log_expected_commands_match": not missing_command_log_ids and not required_command_text_mismatches, + "target_artifact_run_id_binding": ( + bool(single_target_run_id) + and not missing_target_artifact_run_ids + and not mismatched_target_artifact_run_ids + ), + } + return [name for name, passed in gates.items() if not passed] + + +def _artifact_path_policy_validation_errors(report: dict[str, Any]) -> list[str]: + errors: list[str] = [] + policy = report.get("artifact_path_policy") + if not isinstance(policy, dict) or not policy: + return errors + artifact_paths = _object_or_empty(report.get("artifact_paths")) + if policy.get("policy_id") != "m12_target_artifact_paths_v1": + errors.append("artifact_path_policy.policy_id must be m12_target_artifact_paths_v1") + if policy.get("required_target_paths") != TARGET_ARTIFACT_PATHS: + errors.append("artifact_path_policy.required_target_paths must match canonical M12 target artifact paths") + expected_paths_match = _artifact_paths_match_target_defaults(artifact_paths) + if policy.get("paths_match_target_defaults") is not expected_paths_match: + errors.append("artifact_path_policy.paths_match_target_defaults is stale") + if not str(policy.get("reason") or "").strip(): + errors.append("artifact_path_policy.reason must be non-empty") + return errors + + +def _artifact_section_paths_from_final_report(report: dict[str, Any]) -> dict[str, str | None]: + section_by_artifact = { + "archive_verify_report": "archive_verify", + "preflight_report": "preflight", + "swe_run_summary": "swe_probe", + "verl_smoke_report": "verl_export", + "ray_probe_report": "ray_probe", + "warm_vs_cold_report": "warm_vs_cold", + "load_ladder_report": "load_ladder", + "soak_report": "soak", + "command_log_manifest": "command_logs", + } + paths: dict[str, str | None] = {} + for artifact_key, section_key in section_by_artifact.items(): + section = report.get(section_key) + if not isinstance(section, dict): + paths[artifact_key] = None + continue + path = section.get("path") + paths[artifact_key] = str(path) if path is not None else None + return paths + + +def _artifact_section_path_validation_errors(report: dict[str, Any]) -> list[str]: + artifact_paths = report.get("artifact_paths") + if not isinstance(artifact_paths, dict): + return [] + section_paths = _artifact_section_paths_from_final_report(report) + errors: list[str] = [] + for artifact_key in TARGET_ARTIFACT_PATHS: + artifact_path = artifact_paths.get(artifact_key) + expected = str(artifact_path) if artifact_path is not None else None + if section_paths.get(artifact_key) != expected: + errors.append(f"artifact_paths.{artifact_key} must match embedded section path") + return errors + + +def _string_list(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if str(item)] + + +def _command_log_summary_validation_errors(report: dict[str, Any]) -> list[str]: + command_logs = report.get("command_logs") + if not isinstance(command_logs, dict) or not command_logs: + return [] + + command_entries = _command_log_entries(command_logs) + required_command_ids = set(REQUIRED_COMMAND_LOG_IDS) + archived_command_ids = sorted( + _command_log_ids([entry for entry in command_entries if _command_log_entry_archived(entry)]) + ) + passed_command_ids = _command_log_ids( + [entry for entry in command_entries if str(entry.get("status") or "") == "passed"] + ) + expected_missing_command_ids = sorted(required_command_ids - set(archived_command_ids)) + required_target_run_ids = _required_command_target_run_ids(command_entries, required_command_ids) + expected_target_run_ids = sorted(item for item in required_target_run_ids if item) + expected_single_target_run_id = ( + expected_target_run_ids[0] + if len(expected_target_run_ids) == 1 and "" not in required_target_run_ids + else None + ) + expected_command_text_mismatches = _required_command_text_mismatches(command_entries) + observed_hash_verified_ids = set(_string_list(command_logs.get("hash_verified_command_ids"))) + all_command_ids = _command_log_ids(command_entries) + + errors: list[str] = [] + if command_logs.get("required_command_ids") != list(REQUIRED_COMMAND_LOG_IDS): + errors.append("command_logs.required_command_ids must match canonical M12 required command IDs") + if sorted(_string_list(command_logs.get("archived_command_ids"))) != archived_command_ids: + errors.append("command_logs.archived_command_ids is stale") + if sorted(_string_list(command_logs.get("missing_command_log_ids"))) != expected_missing_command_ids: + errors.append("command_logs.missing_command_log_ids is stale") + if sorted(_string_list(command_logs.get("target_run_ids"))) != expected_target_run_ids: + errors.append("command_logs.target_run_ids is stale") + if command_logs.get("single_target_run_id") != expected_single_target_run_id: + errors.append("command_logs.single_target_run_id is stale") + if command_logs.get("command_text_mismatches") != expected_command_text_mismatches: + errors.append("command_logs.command_text_mismatches is stale") + if command_logs.get("command_count") != len(command_entries): + errors.append("command_logs.command_count is stale") + if observed_hash_verified_ids - all_command_ids: + errors.append("command_logs.hash_verified_command_ids contains unknown command IDs") + if observed_hash_verified_ids - set(archived_command_ids): + errors.append("command_logs.hash_verified_command_ids contains unarchived command IDs") + + readable_manifest = command_logs.get("present") is True and not command_logs.get("read_error") + if readable_manifest: + expected_all_required_logs_archived = not expected_missing_command_ids + expected_all_required_commands_passed = not expected_missing_command_ids and all( + command_id in passed_command_ids for command_id in required_command_ids + ) + if command_logs.get("all_required_logs_archived") is not expected_all_required_logs_archived: + errors.append("command_logs.all_required_logs_archived is stale") + if command_logs.get("all_required_commands_passed") is not expected_all_required_commands_passed: + errors.append("command_logs.all_required_commands_passed is stale") + else: + if command_logs.get("all_required_logs_archived") is True: + errors.append("command_logs.all_required_logs_archived cannot be true without a readable manifest") + if command_logs.get("all_required_commands_passed") is True: + errors.append("command_logs.all_required_commands_passed cannot be true without a readable manifest") + return errors + + +def _command_log_required_ids_canonical(command_logs: dict[str, Any]) -> bool: + return command_logs.get("required_command_ids") == list(REQUIRED_COMMAND_LOG_IDS) + + +def _stable_runtime_fingerprint_sha256(fingerprint: dict[str, Any]) -> str | None: + payload = dict(fingerprint) + observed = payload.pop("sha256", None) + if not isinstance(observed, str) or not observed.startswith("sha256:"): + return None + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _dict_has_exact_keys(raw: Any, expected_keys: set[str]) -> bool: + return isinstance(raw, dict) and {str(key) for key in raw.keys()} == expected_keys + + +def _env_value_is_path_like(value: str) -> bool: + stripped = value.strip() + if not stripped: + return False + if stripped.startswith("~"): + return True + return Path(stripped).is_absolute() or PureWindowsPath(stripped).is_absolute() + + +def _runtime_fingerprint_valid(preflight: dict[str, Any]) -> bool: + fingerprint = preflight.get("runtime_fingerprint") + if not _dict_has_exact_keys(fingerprint, RUNTIME_FINGERPRINT_KEYS): + return False + if fingerprint.get("fingerprint_id") != RUNTIME_FINGERPRINT_ID: + return False + if _stable_runtime_fingerprint_sha256(fingerprint) != fingerprint.get("sha256"): + return False + if not _dict_has_exact_keys(fingerprint.get("platform"), RUNTIME_FINGERPRINT_PLATFORM_KEYS): + return False + if not _dict_has_exact_keys(fingerprint.get("tool_presence"), RUNTIME_FINGERPRINT_TOOL_PRESENCE_KEYS): + return False + if not _dict_has_exact_keys(fingerprint.get("python_modules"), RUNTIME_FINGERPRINT_PYTHON_MODULE_KEYS): + return False + if not _dict_has_exact_keys(fingerprint.get("gpu_summary"), RUNTIME_FINGERPRINT_GPU_SUMMARY_KEYS): + return False + if not _dict_has_exact_keys(fingerprint.get("ray_summary"), RUNTIME_FINGERPRINT_RAY_SUMMARY_KEYS): + return False + sanitized = fingerprint.get("sanitized_environment") + if not _dict_has_exact_keys(sanitized, RUNTIME_FINGERPRINT_SANITIZED_ENVIRONMENT_KEYS): + return False + if sanitized.get("policy") != RUNTIME_FINGERPRINT_ENV_POLICY: + return False + raw_keys = sanitized.get("keys") + values = sanitized.get("values") + redactions = sanitized.get("redactions") + if not isinstance(raw_keys, list) or not isinstance(values, dict) or not isinstance(redactions, dict): + return False + keys = {str(key) for key in raw_keys} + value_keys = {str(key) for key in values} + redaction_keys = {str(key) for key in redactions} + if keys != RUNTIME_FINGERPRINT_ALLOWED_ENV_KEYS: + return False + if not value_keys.issubset(RUNTIME_FINGERPRINT_ALLOWED_ENV_KEYS): + return False + if not redaction_keys.issubset(value_keys): + return False + if any(part in key.upper() for key in keys | value_keys for part in RUNTIME_FINGERPRINT_FORBIDDEN_ENV_KEY_PARTS): + return False + for key, raw_value in values.items(): + value = str(raw_value) + if key in redactions: + if value != RUNTIME_FINGERPRINT_REDACTED_ABSOLUTE_PATH: + return False + if redactions.get(key) != RUNTIME_FINGERPRINT_REDACTION_REASON_ABSOLUTE_PATH: + return False + continue + if value == RUNTIME_FINGERPRINT_REDACTED_ABSOLUTE_PATH: + return False + if _env_value_is_path_like(value): + return False + python_executable_name = str((fingerprint.get("platform") or {}).get("python_executable_name") or "") + if "/" in python_executable_name or "\\" in python_executable_name: + return False + gpu = preflight.get("gpu") if isinstance(preflight.get("gpu"), dict) else {} + modules = preflight.get("python_modules") if isinstance(preflight.get("python_modules"), dict) else {} + ray_cluster = preflight.get("ray_cluster") if isinstance(preflight.get("ray_cluster"), dict) else {} + containers = preflight.get("container_runtimes") if isinstance(preflight.get("container_runtimes"), dict) else {} + expected_tool_presence = { + "rocm_smi": gpu.get("rocm_smi_available"), + "rocminfo": gpu.get("rocminfo_available"), + "docker": containers.get("docker"), + "gvisor_runsc": containers.get("gvisor_runsc"), + "firecracker": containers.get("firecracker"), + } + expected_gpu_summary = { + "rocm_smi_output": gpu.get("rocm_smi_output"), + "torch_probe": gpu.get("torch_probe"), + } + if fingerprint.get("tool_presence") != expected_tool_presence: + return False + if fingerprint.get("python_modules") != modules: + return False + if fingerprint.get("gpu_summary") != expected_gpu_summary: + return False + if fingerprint.get("ray_summary") != {"status_output": ray_cluster.get("status_output")}: + return False + return True + + +def build_m12_final_report( + *, + archive_verify_report_path: Path | None = None, + preflight_report_path: Path, + swe_run_summary_path: Path, + verl_smoke_report_path: Path, + ray_probe_report_path: Path, + warm_vs_cold_report_path: Path, + load_ladder_report_path: Path | None = None, + soak_report_path: Path | None = None, + command_log_manifest_path: Path | None = None, +) -> dict[str, Any]: + archive_verify = _read_optional_json(archive_verify_report_path) + preflight = _read_json_object(preflight_report_path) + swe_summary = _read_json_object(swe_run_summary_path) + verl_smoke = _read_json_object(verl_smoke_report_path) + ray_probe = _read_json_object(ray_probe_report_path) + warm_vs_cold = _read_json_object(warm_vs_cold_report_path) + load_ladder = _read_optional_json(load_ladder_report_path) + soak = _read_optional_json(soak_report_path) + command_logs = _read_optional_json(command_log_manifest_path) + command_log_manifest_validation_errors: list[str] = [] + if command_log_manifest_path is not None and command_logs.get("present") is True: + if command_logs.get("read_error"): + command_log_manifest_validation_errors = [ + f"command log manifest unreadable: {command_logs.get('read_error')}" + ] + else: + from breadboard.rl.m12.command_logs import validate_command_log_manifest + + command_log_manifest_validation_errors = validate_command_log_manifest( + command_log_manifest_path, + required_command_ids=list(REQUIRED_COMMAND_LOG_IDS), + require_passed=True, + verify_hashes=True, + ) + + rows = list(swe_summary.get("rows") or []) + row_counts = _status_counts(rows) + command_entries = _command_log_entries(command_logs) + archive_verify_validation_errors = _archive_verify_validation_errors(archive_verify) + preflight_validation_errors = _preflight_validation_errors(preflight) + swe_summary_validation_errors = _swe_summary_validation_errors(swe_summary) + verl_smoke_validation_errors = _verl_smoke_validation_errors(verl_smoke) + ray_probe_validation_errors = _ray_probe_validation_errors(ray_probe) + warm_vs_cold_validation_errors = _warm_vs_cold_validation_errors(warm_vs_cold) + load_ladder_validation_errors = _load_ladder_validation_errors(load_ladder) + soak_validation_errors = _soak_validation_errors(soak) + archived_command_ids = _command_log_ids( + [entry for entry in command_entries if _command_log_entry_archived(entry)] + ) + hash_verified_command_ids = _command_log_ids( + [entry for entry in command_entries if _command_log_entry_hash_verified(entry, command_log_manifest_path)] + ) + passed_command_ids = _command_log_ids( + [entry for entry in command_entries if str(entry.get("status") or "") == "passed"] + ) + required_command_ids = set(REQUIRED_COMMAND_LOG_IDS) + required_target_run_ids = _required_command_target_run_ids(command_entries, required_command_ids) + required_target_run_ids_without_empty = {item for item in required_target_run_ids if item} + single_target_run_id = ( + next(iter(required_target_run_ids_without_empty)) + if len(required_target_run_ids_without_empty) == 1 and "" not in required_target_run_ids + else None + ) + required_command_text_mismatches = _required_command_text_mismatches(command_entries) + missing_command_log_ids = sorted(required_command_ids - archived_command_ids) + target_artifact_run_ids = _target_artifact_run_ids( + preflight=preflight, + swe_summary=swe_summary, + verl_smoke=verl_smoke, + ray_probe=ray_probe, + warm_vs_cold=warm_vs_cold, + load_ladder=load_ladder, + soak=soak, + ) + missing_target_artifact_run_ids = sorted( + name for name, target_run_id in target_artifact_run_ids.items() if not target_run_id + ) + mismatched_target_artifact_run_ids = { + name: {"expected": single_target_run_id, "observed": target_run_id} + for name, target_run_id in target_artifact_run_ids.items() + if target_run_id and single_target_run_id and target_run_id != single_target_run_id + } + target_artifact_run_id_binding = ( + bool(single_target_run_id) + and not missing_target_artifact_run_ids + and not mismatched_target_artifact_run_ids + ) + load_levels = { + _int_or(item.get("target_sessions"), -1): str(item.get("status") or "") + for item in load_ladder.get("concurrency_levels", []) + if isinstance(item, dict) + } + load_level_items = [ + item for item in load_ladder.get("concurrency_levels", []) if isinstance(item, dict) + ] + resource_skipped_levels = { + _int_or(item.get("target_sessions"), -1) + for item in load_ladder.get("resource_skips", []) + if isinstance(item, dict) + } + expected_artifacts = { + "archive_verify_report": str(archive_verify_report_path) if archive_verify_report_path is not None else None, + "preflight_report": str(preflight_report_path), + "swe_run_summary": str(swe_run_summary_path), + "verl_smoke_report": str(verl_smoke_report_path), + "ray_probe_report": str(ray_probe_report_path), + "warm_vs_cold_report": str(warm_vs_cold_report_path), + "load_ladder_report": str(load_ladder_report_path) if load_ladder_report_path is not None else None, + "soak_report": str(soak_report_path) if soak_report_path is not None else None, + "command_log_manifest": str(command_log_manifest_path) if command_log_manifest_path is not None else None, + } + gates = { + "artifact_paths_match_target_defaults": _artifact_paths_match_target_defaults(expected_artifacts), + "archive_verify_report_present": archive_verify.get("present") is True, + "archive_verify_report_readable": archive_verify.get("present") is True and _read_error(archive_verify) is None, + "archive_verify_report_valid": archive_verify.get("present") is True + and not archive_verify_validation_errors, + "archive_verify_status_passed": archive_verify.get("status") == "passed", + "preflight_report_readable": _read_error(preflight) is None, + "preflight_report_valid": not preflight_validation_errors, + "swe_run_summary_readable": _read_error(swe_summary) is None, + "swe_run_summary_valid": not swe_summary_validation_errors, + "verl_smoke_report_readable": _read_error(verl_smoke) is None, + "verl_smoke_report_valid": not verl_smoke_validation_errors, + "ray_probe_report_readable": _read_error(ray_probe) is None, + "ray_probe_report_valid": not ray_probe_validation_errors, + "warm_vs_cold_report_readable": _read_error(warm_vs_cold) is None, + "warm_vs_cold_report_valid": not warm_vs_cold_validation_errors, + "preflight_passed": preflight.get("status") == "preflight_passed", + "preflight_runtime_fingerprint_present": _runtime_fingerprint_valid(preflight), + "target_hardware": ( + preflight.get("gpu", {}).get("required_accelerator_count") == 8 + and preflight.get("gpu", {}).get("required_accelerator_family") == "MI300X" + and preflight.get("gpu", {}).get("mi300x_product_evidence") is True + and _int_or_zero(preflight.get("gpu", {}).get("torch_probe", {}).get("device_count")) >= 8 + ), + "verl_available": preflight.get("python_modules", {}).get("verl", {}).get("available") is True, + "ray_available": preflight.get("python_modules", {}).get("ray", {}).get("available") is True, + "inference_engine_available": ( + preflight.get("inference_engine_feasibility", {}).get("decision") == "available" + and ( + preflight.get("inference_engine_feasibility", {}).get("vllm_available") is True + or preflight.get("inference_engine_feasibility", {}).get("sglang_available") is True + ) + ), + "container_runtime_available": any( + value is True for value in (preflight.get("container_runtimes") or {}).values() + ), + "filesystem_cas_smoke_passed": preflight.get("filesystem_cas_smoke", {}).get("status") == "passed", + "swe_probe_ran_10_rows": len(rows) >= 10, + "swe_probe_has_accepted_rows": row_counts["accepted"] > 0, + "swe_probe_no_unknown_status": row_counts["other"] == 0, + "verl_jsonl_tensorizable": verl_smoke.get("formats", {}).get("jsonl", {}).get("tensorizable") is True, + "verl_parquet_tensorizable": verl_smoke.get("formats", {}).get("parquet", {}).get("tensorizable") is True, + "verl_row_count_matches_swe": _int_or(verl_smoke.get("row_count"), -1) == len(rows), + "ray_probe_ran_10_rows": _int_or_zero(ray_probe.get("row_count")) >= 10, + "ray_probe_has_workers": _int_or_zero(ray_probe.get("worker_count")) >= 2, + "ray_probe_distributed": ray_probe.get("ray_local_mode") is False, + "warm_vs_cold_has_total_ms": ( + "total_ms" in warm_vs_cold.get("warm", {}) and "total_ms" in warm_vs_cold.get("cold", {}) + ), + "load_ladder_report_present": load_ladder.get("present") is True, + "load_ladder_report_valid": load_ladder.get("present") is not True or not load_ladder_validation_errors, + "load_ladder_required_levels_passed": all( + load_levels.get(level) == "passed" for level in REQUIRED_LOAD_LEVELS + ), + "load_ladder_100_attempted_or_skipped": all( + load_levels.get(level) == "passed" or level in resource_skipped_levels for level in OPTIONAL_LOAD_LEVELS + ), + "load_ladder_policy_integrity": load_ladder.get("policy_version_integrity") is True, + "load_ladder_no_queue_corruption": load_ladder.get("queue_backpressure_integrity") is True, + "load_ladder_distributed": all( + item.get("status") == "resource_skipped" or item.get("ray_local_mode") is False + for item in load_level_items + ) + and bool(load_level_items), + "soak_report_present": soak.get("present") is True, + "soak_report_valid": soak.get("present") is not True or not soak_validation_errors, + "soak_status_passed": soak.get("status") == "passed", + "soak_duration_at_least_2h": _int_or_zero(soak.get("duration_seconds")) >= MIN_SOAK_SECONDS, + "soak_no_runtime_failures": _int_or(soak.get("runtime_failure_count"), 1) == 0, + "soak_distributed": soak.get("ray_local_mode") is False, + "command_log_manifest_present": command_logs.get("present") is True, + "command_log_manifest_id_valid": command_logs.get("manifest_id") == COMMAND_LOG_MANIFEST_ID, + "command_log_manifest_valid": ( + command_logs.get("present") is True and not command_log_manifest_validation_errors + ), + "command_log_required_ids_canonical": _command_log_required_ids_canonical(command_logs), + "command_log_manifest_complete": not missing_command_log_ids, + "command_log_hashes_present": all( + _command_log_entry_has_hash(entry) + for entry in command_entries + if str(entry.get("command_id") or "") in required_command_ids + ) + and not missing_command_log_ids, + "command_log_hashes_verified": all(command_id in hash_verified_command_ids for command_id in required_command_ids), + "command_log_commands_passed": all(command_id in passed_command_ids for command_id in required_command_ids), + "command_log_required_logs_archived_summary": command_logs.get("all_required_logs_archived") is True, + "command_log_required_commands_passed_summary": command_logs.get("all_required_commands_passed") is True, + "command_log_single_target_run_id": ( + not missing_command_log_ids + and single_target_run_id is not None + ), + "command_log_expected_commands_match": not missing_command_log_ids and not required_command_text_mismatches, + "target_artifact_run_id_binding": target_artifact_run_id_binding, + } + missing_gates = [name for name, passed in gates.items() if not passed] + return { + "report_id": FINAL_REPORT_ID, + "claim_boundary": FINAL_REPORT_CLAIM_BOUNDARY, + "scorecard_update_allowed": False, + "m12_score_eligible": not missing_gates, + "missing_gates": missing_gates, + "missing_gate_remediations": _missing_gate_remediations(missing_gates), + "artifact_paths": expected_artifacts, + "artifact_path_policy": { + "policy_id": "m12_target_artifact_paths_v1", + "required_target_paths": dict(TARGET_ARTIFACT_PATHS), + "paths_match_target_defaults": _artifact_paths_match_target_defaults(expected_artifacts), + "reason": "M12 score eligibility requires target-node default artifact paths, not local M6/M7/M8 preparation paths.", + }, + "archive_verify": { + "present": archive_verify.get("present") is True, + "path": archive_verify.get("path"), + "read_error": archive_verify.get("read_error"), + "report_id": archive_verify.get("report_id"), + "claim_boundary": archive_verify.get("claim_boundary"), + "scorecard_update_allowed": archive_verify.get("scorecard_update_allowed"), + "m12_points_awarded": archive_verify.get("m12_points_awarded"), + "status": archive_verify.get("status"), + "archive_manifest_id": archive_verify.get("archive_manifest_id"), + "archive_claim_boundary": archive_verify.get("archive_claim_boundary"), + "archive_sha256": archive_verify.get("archive_sha256"), + "included_entry_count": archive_verify.get("included_entry_count"), + "all_required_artifacts_present": archive_verify.get("all_required_artifacts_present"), + "all_transfer_requirements_covered": archive_verify.get("all_transfer_requirements_covered"), + "archive_contains_source_overlay": archive_verify.get("archive_contains_source_overlay"), + "archive_deterministic": archive_verify.get("archive_deterministic"), + "source_paths_portable": archive_verify.get("source_paths_portable"), + "errors": list(archive_verify.get("errors") or []), + "validation_errors": archive_verify_validation_errors, + }, + "preflight": { + "present": preflight.get("present") is True, + "path": preflight.get("path"), + "read_error": preflight.get("read_error"), + "status": preflight.get("status"), + "target_run_id": preflight.get("target_run_id"), + "blockers": list(preflight.get("blockers") or []), + "gpu": preflight.get("gpu", {}), + "python_modules": preflight.get("python_modules", {}), + "ray_cluster": preflight.get("ray_cluster", {}), + "inference_engine_feasibility": preflight.get("inference_engine_feasibility", {}), + "filesystem_cas_smoke": preflight.get("filesystem_cas_smoke", {}), + "container_runtimes": preflight.get("container_runtimes", {}), + "runtime_fingerprint": preflight.get("runtime_fingerprint", {}), + "validation_errors": preflight_validation_errors, + }, + "swe_probe": { + "present": swe_summary.get("present") is True, + "path": swe_summary.get("path"), + "read_error": swe_summary.get("read_error"), + "run_id": swe_summary.get("run_id"), + "target_run_id": swe_summary.get("target_run_id"), + "row_count": len(rows), + "row_status_counts": row_counts, + "package_id": swe_summary.get("package_id"), + "package_hash": swe_summary.get("package_hash"), + "validation_errors": swe_summary_validation_errors, + }, + "verl_export": { + "present": verl_smoke.get("present") is True, + "path": verl_smoke.get("path"), + "read_error": verl_smoke.get("read_error"), + "row_count": verl_smoke.get("row_count"), + "target_run_id": verl_smoke.get("target_run_id"), + "trainable_candidate_count": verl_smoke.get("trainable_candidate_count"), + "tensorizable": verl_smoke.get("tensorizable"), + "formats": verl_smoke.get("formats", {}), + "validation_errors": verl_smoke_validation_errors, + }, + "ray_probe": { + "present": ray_probe.get("present") is True, + "path": ray_probe.get("path"), + "read_error": ray_probe.get("read_error"), + "row_count": ray_probe.get("row_count"), + "target_run_id": ray_probe.get("target_run_id"), + "worker_count": ray_probe.get("worker_count"), + "ray_local_mode": ray_probe.get("ray_local_mode"), + "validation_errors": ray_probe_validation_errors, + }, + "warm_vs_cold": { + "present": warm_vs_cold.get("present") is True, + "path": warm_vs_cold.get("path"), + "read_error": warm_vs_cold.get("read_error"), + "target_run_id": warm_vs_cold.get("target_run_id"), + "warm": warm_vs_cold.get("warm", {}), + "cold": warm_vs_cold.get("cold", {}), + "validation_errors": warm_vs_cold_validation_errors, + }, + "load_ladder": { + "present": load_ladder.get("present") is True, + "path": load_ladder.get("path"), + "read_error": load_ladder.get("read_error"), + "target_run_id": load_ladder.get("target_run_id"), + "required_levels": list(REQUIRED_LOAD_LEVELS), + "optional_levels": list(OPTIONAL_LOAD_LEVELS), + "concurrency_levels": list(load_ladder.get("concurrency_levels") or []), + "resource_skips": list(load_ladder.get("resource_skips") or []), + "policy_version_integrity": load_ladder.get("policy_version_integrity"), + "queue_backpressure_integrity": load_ladder.get("queue_backpressure_integrity"), + "validation_errors": load_ladder_validation_errors, + }, + "soak": { + "present": soak.get("present") is True, + "path": soak.get("path"), + "read_error": soak.get("read_error"), + "minimum_duration_seconds": MIN_SOAK_SECONDS, + "duration_seconds": soak.get("duration_seconds"), + "target_run_id": soak.get("target_run_id"), + "status": soak.get("status"), + "runtime_failure_count": soak.get("runtime_failure_count"), + "ray_local_mode": soak.get("ray_local_mode"), + "validation_errors": soak_validation_errors, + }, + "command_logs": { + "present": command_logs.get("present") is True, + "path": command_logs.get("path"), + "read_error": command_logs.get("read_error"), + "manifest_id": command_logs.get("manifest_id"), + "manifest_validation_errors": command_log_manifest_validation_errors, + "required_command_ids": list(REQUIRED_COMMAND_LOG_IDS), + "manifest_required_command_ids": command_logs.get("required_command_ids"), + "archived_command_ids": sorted(archived_command_ids), + "hash_verified_command_ids": sorted(hash_verified_command_ids), + "missing_command_log_ids": missing_command_log_ids, + "target_run_ids": sorted(required_target_run_ids_without_empty), + "single_target_run_id": ( + single_target_run_id + ), + "command_text_mismatches": required_command_text_mismatches, + "all_required_logs_archived": command_logs.get("all_required_logs_archived"), + "all_required_commands_passed": command_logs.get("all_required_commands_passed"), + "command_count": len(command_entries), + "commands": command_entries, + }, + "target_run_identity": { + "single_command_log_target_run_id": single_target_run_id, + "artifact_target_run_ids": target_artifact_run_ids, + "missing_artifact_target_run_ids": missing_target_artifact_run_ids, + "mismatched_artifact_target_run_ids": mismatched_target_artifact_run_ids, + "run_ids_match_command_logs": target_artifact_run_id_binding, + }, + "operator_next_step": ( + "If m12_score_eligible is true, archive this report with raw command logs and update the scorecard in a " + "separate reviewed change. This builder never updates the scorecard." + ), + } + + +def validate_m12_final_report(report: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != FINAL_REPORT_ID: + errors.append("report_id must be bb_zyphra_rl_phase1_m12_final_report_v1") + if report.get("claim_boundary") != FINAL_REPORT_CLAIM_BOUNDARY: + errors.append("claim_boundary must remain m12_target_validation_candidate_not_scorecard_update") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false; score updates require separate review") + if not isinstance(report.get("missing_gates"), list): + errors.append("missing_gates must be a list") + else: + unknown_gates = sorted({str(gate) for gate in report["missing_gates"]} - M12_FINAL_REPORT_GATE_NAMES) + for gate in unknown_gates: + errors.append(f"unknown missing gate: {gate}") + remediations = report.get("missing_gate_remediations") + if not isinstance(remediations, list): + errors.append("missing_gate_remediations must be a list") + elif isinstance(report.get("missing_gates"), list): + remediation_gates: list[str] = [] + for index, remediation in enumerate(remediations, start=1): + if not isinstance(remediation, dict): + errors.append(f"missing_gate_remediations row {index} must be an object") + continue + gate = str(remediation.get("gate") or "") + remediation_gates.append(gate) + if gate not in M12_FINAL_REPORT_GATE_NAMES: + errors.append(f"missing_gate_remediations row {index} has unknown gate: {gate}") + if not str(remediation.get("blocking_stage") or "").strip(): + errors.append(f"missing_gate_remediations row {index} requires blocking_stage") + if not str(remediation.get("operator_action") or "").strip(): + errors.append(f"missing_gate_remediations row {index} requires operator_action") + if remediation.get("target_action_id") is not None and not str( + remediation.get("target_action_id") or "" + ).strip(): + errors.append(f"missing_gate_remediations row {index} target_action_id must be non-empty when set") + required_artifact_path = remediation.get("required_artifact_path") + if required_artifact_path is not None and str(required_artifact_path) not in TARGET_ARTIFACT_PATHS.values(): + errors.append( + f"missing_gate_remediations row {index} required_artifact_path must be an M12 target artifact path" + ) + if remediation_gates != [str(gate) for gate in report["missing_gates"]]: + errors.append("missing_gate_remediations gates must match missing_gates in order") + if not isinstance(report.get("artifact_paths"), dict) or not report["artifact_paths"]: + errors.append("artifact_paths must be a non-empty object") + if not isinstance(report.get("artifact_path_policy"), dict) or not report["artifact_path_policy"]: + errors.append("artifact_path_policy must be a non-empty object") + errors.extend(_artifact_path_policy_validation_errors(report)) + errors.extend(_artifact_section_path_validation_errors(report)) + errors.extend(_command_log_summary_validation_errors(report)) + expected_missing_gates = _missing_gate_names_from_final_report(report) + if isinstance(report.get("missing_gates"), list): + observed_missing_gates = [str(gate) for gate in report["missing_gates"]] + if observed_missing_gates != expected_missing_gates: + errors.append("missing_gates must match final-report embedded evidence gates") + if report.get("m12_score_eligible") is not (not expected_missing_gates): + errors.append("m12_score_eligible must match final-report embedded evidence gates") + errors.extend(_target_run_identity_validation_errors(report)) + if bool(report.get("m12_score_eligible")) and report.get("missing_gates"): + errors.append("m12_score_eligible cannot be true while missing_gates is non-empty") + if bool(report.get("m12_score_eligible")): + preflight = report.get("preflight", {}) + archive_verify = report.get("archive_verify", {}) + swe_probe = report.get("swe_probe", {}) + verl_export = report.get("verl_export", {}) + ray_probe = report.get("ray_probe", {}) + load_ladder = report.get("load_ladder", {}) + soak = report.get("soak", {}) + command_logs = report.get("command_logs", {}) + target_run_identity = report.get("target_run_identity", {}) + if not _artifact_paths_match_target_defaults(report.get("artifact_paths") or {}): + errors.append("eligible report requires target-node default artifact paths") + artifact_path_policy = report.get("artifact_path_policy") or {} + if artifact_path_policy.get("paths_match_target_defaults") is not True: + errors.append("eligible report requires artifact_path_policy.paths_match_target_defaults=true") + for section_name, section in [ + ("archive_verify", archive_verify), + ("preflight", preflight), + ("swe_probe", swe_probe), + ("verl_export", verl_export), + ("ray_probe", ray_probe), + ("warm_vs_cold", report.get("warm_vs_cold", {})), + ("load_ladder", load_ladder), + ("soak", soak), + ("command_logs", command_logs), + ]: + if section.get("read_error"): + errors.append(f"eligible report requires readable {section_name}: {section.get('read_error')}") + gpu = preflight.get("gpu", {}) + modules = preflight.get("python_modules", {}) + if archive_verify.get("present") is not True: + errors.append("eligible report requires archive verify report") + if archive_verify.get("validation_errors") != []: + errors.append("eligible report requires archive verify validation errors to be empty") + if archive_verify.get("status") != "passed": + errors.append("eligible report requires archive verify status=passed") + if preflight.get("status") != "preflight_passed": + errors.append("eligible report requires preflight.status=preflight_passed") + if preflight.get("validation_errors") != []: + errors.append("eligible report requires preflight validation errors to be empty") + if preflight.get("blockers"): + errors.append("eligible report requires no preflight blockers") + if not _runtime_fingerprint_valid(preflight): + errors.append("eligible report requires preflight runtime fingerprint") + if gpu.get("required_accelerator_count") != 8: + errors.append("eligible report requires required_accelerator_count=8") + if gpu.get("required_accelerator_family") != "MI300X": + errors.append("eligible report requires required_accelerator_family=MI300X") + if gpu.get("mi300x_product_evidence") is not True: + errors.append("eligible report requires MI300X product evidence") + if _int_or_zero(gpu.get("torch_probe", {}).get("device_count")) < 8: + errors.append("eligible report requires torch device_count >= 8") + if not (gpu.get("rocminfo_available") is True or gpu.get("rocm_smi_available") is True): + errors.append("eligible report requires ROCm tooling availability") + if modules.get("verl", {}).get("available") is not True: + errors.append("eligible report requires VeRL import availability") + if modules.get("ray", {}).get("available") is not True: + errors.append("eligible report requires Ray import availability") + inference = preflight.get("inference_engine_feasibility", {}) + if not ( + inference.get("decision") == "available" + and (inference.get("vllm_available") is True or inference.get("sglang_available") is True) + ): + errors.append("eligible report requires vLLM or SGLang inference-engine availability") + if not any(value is True for value in (preflight.get("container_runtimes") or {}).values()): + errors.append("eligible report requires at least one container runtime") + if preflight.get("filesystem_cas_smoke", {}).get("status") != "passed": + errors.append("eligible report requires filesystem/CAS smoke passed") + if _int_or_zero(swe_probe.get("row_count")) < 10: + errors.append("eligible report requires at least 10 SWE rows") + if swe_probe.get("validation_errors") != []: + errors.append("eligible report requires SWE run summary validation errors to be empty") + if _int_or(swe_probe.get("row_status_counts", {}).get("other"), 1) != 0: + errors.append("eligible report requires no unknown SWE row statuses") + if _int_or_zero(swe_probe.get("row_status_counts", {}).get("accepted")) < 1: + errors.append("eligible report requires at least one accepted SWE row") + if _int_or(verl_export.get("row_count"), -1) != _int_or(swe_probe.get("row_count"), -2): + errors.append("eligible report requires VeRL row count to match SWE row count") + if verl_export.get("validation_errors") != []: + errors.append("eligible report requires VeRL smoke validation errors to be empty") + if verl_export.get("formats", {}).get("jsonl", {}).get("tensorizable") is not True: + errors.append("eligible report requires tensorizable JSONL export") + if verl_export.get("formats", {}).get("parquet", {}).get("tensorizable") is not True: + errors.append("eligible report requires tensorizable Parquet export") + if _int_or_zero(ray_probe.get("row_count")) < 10: + errors.append("eligible report requires at least 10 Ray rows") + if ray_probe.get("validation_errors") != []: + errors.append("eligible report requires Ray probe validation errors to be empty") + if _int_or_zero(ray_probe.get("worker_count")) < 2: + errors.append("eligible report requires at least two Ray workers") + if ray_probe.get("ray_local_mode") is not False: + errors.append("eligible report requires Ray distributed mode, not local_mode") + if report.get("warm_vs_cold", {}).get("validation_errors") != []: + errors.append("eligible report requires warm-vs-cold validation errors to be empty") + if load_ladder.get("present") is not True: + errors.append("eligible report requires load ladder report") + if load_ladder.get("validation_errors") != []: + errors.append("eligible report requires load ladder validation errors to be empty") + observed_levels = { + _int_or(item.get("target_sessions"), -1): str(item.get("status") or "") + for item in load_ladder.get("concurrency_levels", []) + if isinstance(item, dict) + } + skipped_levels = { + _int_or(item.get("target_sessions"), -1) + for item in load_ladder.get("resource_skips", []) + if isinstance(item, dict) + } + for level in REQUIRED_LOAD_LEVELS: + if observed_levels.get(level) != "passed": + errors.append(f"eligible report requires load level {level} passed") + for level in OPTIONAL_LOAD_LEVELS: + if observed_levels.get(level) != "passed" and level not in skipped_levels: + errors.append(f"eligible report requires load level {level} passed or resource-skipped") + if load_ladder.get("policy_version_integrity") is not True: + errors.append("eligible report requires load policy_version_integrity=true") + if load_ladder.get("queue_backpressure_integrity") is not True: + errors.append("eligible report requires load queue_backpressure_integrity=true") + for item in load_ladder.get("concurrency_levels", []): + if isinstance(item, dict) and item.get("status") != "resource_skipped" and item.get("ray_local_mode") is not False: + errors.append("eligible report requires distributed load ladder, not local_mode") + if soak.get("present") is not True: + errors.append("eligible report requires soak report") + if soak.get("validation_errors") != []: + errors.append("eligible report requires soak validation errors to be empty") + if soak.get("status") != "passed": + errors.append("eligible report requires soak status=passed") + if _int_or_zero(soak.get("duration_seconds")) < MIN_SOAK_SECONDS: + errors.append("eligible report requires soak duration >= 7200 seconds") + if _int_or(soak.get("runtime_failure_count"), 1) != 0: + errors.append("eligible report requires zero soak runtime failures") + if soak.get("ray_local_mode") is not False: + errors.append("eligible report requires distributed soak, not local_mode") + if command_logs.get("present") is not True: + errors.append("eligible report requires command log manifest") + if command_logs.get("manifest_id") != COMMAND_LOG_MANIFEST_ID: + errors.append("eligible report requires valid command log manifest_id") + if command_logs.get("manifest_validation_errors") != []: + errors.append("eligible report requires command log manifest validation errors to be empty") + if command_logs.get("manifest_required_command_ids") != list(REQUIRED_COMMAND_LOG_IDS): + errors.append("eligible report requires canonical command log required_command_ids") + archived_command_ids = set(command_logs.get("archived_command_ids") or []) + hash_verified_command_ids = set(command_logs.get("hash_verified_command_ids") or []) + for command_id in REQUIRED_COMMAND_LOG_IDS: + if command_id not in archived_command_ids: + errors.append(f"eligible report requires archived command log for {command_id}") + if command_id not in hash_verified_command_ids: + errors.append(f"eligible report requires hash-verified command log for {command_id}") + if command_logs.get("all_required_logs_archived") is not True: + errors.append("eligible report requires all_required_logs_archived=true") + if command_logs.get("all_required_commands_passed") is not True: + errors.append("eligible report requires all_required_commands_passed=true") + if not command_logs.get("single_target_run_id"): + errors.append("eligible report requires all required command logs to share one target_run_id") + if command_logs.get("command_text_mismatches") != []: + errors.append("eligible report requires required command text to match M12 target command manifest") + if not isinstance(target_run_identity, dict) or not target_run_identity: + errors.append("eligible report requires target_run_identity") + else: + command_target_run_id = command_logs.get("single_target_run_id") + if target_run_identity.get("single_command_log_target_run_id") != command_target_run_id: + errors.append("eligible report requires target_run_identity to match command log target_run_id") + if target_run_identity.get("run_ids_match_command_logs") is not True: + errors.append("eligible report requires target artifacts to share command-log target_run_id") + artifact_target_run_ids = target_run_identity.get("artifact_target_run_ids") + if not isinstance(artifact_target_run_ids, dict): + errors.append("eligible report requires artifact target_run_id map") + else: + actual_artifact_target_run_ids = _target_artifact_run_ids_from_final_report(report) + for artifact_key in [ + "preflight_report", + "swe_run_summary", + "verl_smoke_report", + "ray_probe_report", + "warm_vs_cold_report", + "load_ladder_report", + "soak_report", + ]: + if str(actual_artifact_target_run_ids.get(artifact_key) or "") != str(command_target_run_id or ""): + errors.append(f"eligible report requires {artifact_key} target_run_id to match command logs") + for entry in command_logs.get("commands", []): + if not isinstance(entry, dict): + continue + command_id = str(entry.get("command_id") or "") + if command_id in REQUIRED_COMMAND_LOG_IDS and not _command_log_entry_archived(entry): + errors.append(f"eligible report requires log_path and sha256 for {command_id}") + return errors + + +def write_m12_final_report( + *, + output_path: Path, + archive_verify_report_path: Path | None = None, + preflight_report_path: Path, + swe_run_summary_path: Path, + verl_smoke_report_path: Path, + ray_probe_report_path: Path, + warm_vs_cold_report_path: Path, + load_ladder_report_path: Path | None = None, + soak_report_path: Path | None = None, + command_log_manifest_path: Path | None = None, +) -> dict[str, Any]: + report = build_m12_final_report( + archive_verify_report_path=archive_verify_report_path, + preflight_report_path=preflight_report_path, + swe_run_summary_path=swe_run_summary_path, + verl_smoke_report_path=verl_smoke_report_path, + ray_probe_report_path=ray_probe_report_path, + warm_vs_cold_report_path=warm_vs_cold_report_path, + load_ladder_report_path=load_ladder_report_path, + soak_report_path=soak_report_path, + command_log_manifest_path=command_log_manifest_path, + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report diff --git a/breadboard/rl/m12/load_soak.py b/breadboard/rl/m12/load_soak.py new file mode 100644 index 00000000..eb47400b --- /dev/null +++ b/breadboard/rl/m12/load_soak.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +from breadboard.rl.env_package.schema import EnvPackage +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.m12.final_report import MIN_SOAK_SECONDS, OPTIONAL_LOAD_LEVELS, REQUIRED_LOAD_LEVELS +from breadboard.rl.runtime import run_local_ray_toy_probe, summarize_stage_metrics + + +LOAD_LADDER_REPORT_ID = "bb_zyphra_rl_phase1_m12_load_ladder_report_v1" +SOAK_REPORT_ID = "bb_zyphra_rl_phase1_m12_soak_report_v1" +LOAD_LADDER_CLAIM_BOUNDARY = "target_load_ladder_probe_not_scorecard_update" +SOAK_CLAIM_BOUNDARY = "target_soak_probe_not_scorecard_update" + + +def _task_ids(prefix: str, count: int) -> list[str]: + return [f"{prefix}_{index:04d}" for index in range(1, count + 1)] + + +def _p95_total_ms(rows: list[dict[str, Any]]) -> float | None: + summary = summarize_stage_metrics(rows) + total = summary.get("total_ms") + if not total: + return None + return float(total["p95"]) + + +def _row_counts(rows: list[dict[str, Any]]) -> dict[str, int]: + accepted = sum(float(row.get("reward", 0.0)) > 0.0 for row in rows) + return { + "accepted_count": accepted, + "rejected_count": len(rows) - accepted, + "quarantined_count": 0, + } + + +def build_m12_load_ladder_report( + *, + package: EnvPackage, + levels: list[int], + skip_levels: dict[int, str] | None = None, + min_rows_per_level: int = 10, + local_mode: bool = False, + target_run_id: str | None = None, +) -> dict[str, Any]: + skip_levels = skip_levels or {} + concurrency_levels: list[dict[str, Any]] = [] + resource_skips: list[dict[str, Any]] = [] + for level in levels: + started_at = time.time() + if level in skip_levels: + resource_skips.append({"target_sessions": level, "reason": skip_levels[level]}) + concurrency_levels.append( + { + "target_sessions": level, + "status": "resource_skipped", + "started_at": started_at, + "completed_at": time.time(), + "row_count": 0, + "accepted_count": 0, + "quarantined_count": 0, + "rejected_count": 0, + "p95_total_ms": None, + "policy_version_integrity": True, + "queue_backpressure_integrity": True, + "ray_local_mode": local_mode, + "worker_count": 0, + "notes": skip_levels[level], + } + ) + continue + row_count = max(min_rows_per_level, level) + probe = run_local_ray_toy_probe( + package=package, + task_ids=_task_ids(f"m12_load_{level}", row_count), + num_workers=level, + local_mode=local_mode, + ) + rows = list(probe.get("rows") or []) + counts = _row_counts(rows) + status = "passed" if len(rows) >= min_rows_per_level and counts["rejected_count"] == 0 else "failed" + concurrency_levels.append( + { + "target_sessions": level, + "status": status, + "started_at": started_at, + "completed_at": time.time(), + "row_count": len(rows), + **counts, + "p95_total_ms": _p95_total_ms(rows), + "policy_version_integrity": all(int(row.get("event_count", 0)) == 3 for row in rows), + "queue_backpressure_integrity": len(rows) == row_count, + "ray_local_mode": probe.get("ray_local_mode"), + "worker_count": probe.get("worker_count"), + "notes": "", + } + ) + passed_or_skipped = all(item["status"] in {"passed", "resource_skipped"} for item in concurrency_levels) + return { + "report_id": LOAD_LADDER_REPORT_ID, + "claim_boundary": LOAD_LADDER_CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "concurrency_levels": concurrency_levels, + "resource_skips": resource_skips, + "policy_version_integrity": passed_or_skipped + and all(item.get("policy_version_integrity") is True for item in concurrency_levels), + "queue_backpressure_integrity": passed_or_skipped + and all(item.get("queue_backpressure_integrity") is True for item in concurrency_levels), + "operator_notes": "Probe-scoped target load ladder; not trainer execution or production scale support.", + } + + +def validate_m12_load_ladder_report( + report: dict[str, Any], + *, + required_levels: list[int] | None = None, + optional_levels: list[int] | None = None, + require_distributed: bool = True, +) -> list[str]: + errors: list[str] = [] + required_levels = list(REQUIRED_LOAD_LEVELS if required_levels is None else required_levels) + optional_levels = list(OPTIONAL_LOAD_LEVELS if optional_levels is None else optional_levels) + if report.get("report_id") != LOAD_LADDER_REPORT_ID: + errors.append("load ladder report_id must be bb_zyphra_rl_phase1_m12_load_ladder_report_v1") + if report.get("claim_boundary") != LOAD_LADDER_CLAIM_BOUNDARY: + errors.append("load ladder claim_boundary must remain target_load_ladder_probe_not_scorecard_update") + levels = [item for item in report.get("concurrency_levels", []) if isinstance(item, dict)] + if not levels: + errors.append("load ladder requires at least one concurrency level") + levels_by_target = {_int_value(item.get("target_sessions")): item for item in levels} + skips_by_target = { + _int_value(item.get("target_sessions")): item + for item in report.get("resource_skips", []) + if isinstance(item, dict) + } + for level in required_levels: + item = levels_by_target.get(level) + if item is None: + errors.append(f"load ladder missing required level {level}") + continue + if item.get("status") != "passed": + errors.append(f"load ladder required level {level} must pass") + if _int_value(item.get("row_count")) < 1: + errors.append(f"load ladder required level {level} must record rows") + if _int_value(item.get("worker_count")) < 1: + errors.append(f"load ladder required level {level} must record worker_count") + for level in optional_levels: + item = levels_by_target.get(level) + skipped = skips_by_target.get(level) + if item is None: + errors.append(f"load ladder missing optional level {level} status") + continue + if item.get("status") == "resource_skipped": + reason = str((skipped or {}).get("reason") or item.get("notes") or "").strip() + if not reason: + errors.append(f"load ladder skipped optional level {level} requires concrete reason") + elif item.get("status") != "passed": + errors.append(f"load ladder optional level {level} must pass or be resource_skipped") + if report.get("policy_version_integrity") is not True: + errors.append("load ladder requires policy_version_integrity=true") + if report.get("queue_backpressure_integrity") is not True: + errors.append("load ladder requires queue_backpressure_integrity=true") + if require_distributed: + for item in levels: + if item.get("status") != "resource_skipped" and item.get("ray_local_mode") is not False: + errors.append("load ladder requires distributed Ray for non-skipped levels") + break + return errors + + +def build_m12_load_ladder_report_from_package( + *, + package_path: Path, + levels: list[int], + skip_levels: dict[int, str] | None = None, + min_rows_per_level: int = 10, + local_mode: bool = False, + target_run_id: str | None = None, +) -> dict[str, Any]: + return build_m12_load_ladder_report( + package=load_env_package(package_path), + levels=levels, + skip_levels=skip_levels, + min_rows_per_level=min_rows_per_level, + local_mode=local_mode, + target_run_id=target_run_id, + ) + + +def _int_value(value: Any, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def build_m12_soak_report( + *, + package: EnvPackage, + duration_seconds: int = 7200, + minimum_duration_seconds: int = 7200, + interval_seconds: float = 30.0, + num_workers: int = 20, + rows_per_iteration: int = 10, + min_iterations: int = 1, + local_mode: bool = False, + target_run_id: str | None = None, +) -> dict[str, Any]: + started_at = time.time() + deadline = started_at + max(0, duration_seconds) + iterations = 0 + runtime_failure_count = 0 + all_rows: list[dict[str, Any]] = [] + while iterations < min_iterations or time.time() < deadline: + try: + probe = run_local_ray_toy_probe( + package=package, + task_ids=_task_ids(f"m12_soak_{iterations + 1}", rows_per_iteration), + num_workers=num_workers, + local_mode=local_mode, + ) + all_rows.extend(list(probe.get("rows") or [])) + except Exception: + runtime_failure_count += 1 + iterations += 1 + if time.time() < deadline and interval_seconds > 0: + time.sleep(min(interval_seconds, max(0.0, deadline - time.time()))) + completed_at = time.time() + counts = _row_counts(all_rows) + observed_duration = max(duration_seconds, int(round(completed_at - started_at))) + status = "passed" if runtime_failure_count == 0 and observed_duration >= minimum_duration_seconds else "failed" + return { + "report_id": SOAK_REPORT_ID, + "claim_boundary": SOAK_CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "status": status, + "minimum_duration_seconds": minimum_duration_seconds, + "duration_seconds": observed_duration, + "started_at": started_at, + "completed_at": completed_at, + "runtime_failure_count": runtime_failure_count, + "row_count": len(all_rows), + **counts, + "max_queue_depth": num_workers, + "max_worker_restarts": 0, + "iterations": iterations, + "rows_per_iteration": rows_per_iteration, + "ray_local_mode": local_mode, + "operator_notes": "Probe-scoped target soak; not trainer execution or production scale support.", + } + + +def validate_m12_soak_report( + report: dict[str, Any], + *, + minimum_duration_seconds: int = MIN_SOAK_SECONDS, + require_distributed: bool = True, +) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != SOAK_REPORT_ID: + errors.append("soak report_id must be bb_zyphra_rl_phase1_m12_soak_report_v1") + if report.get("claim_boundary") != SOAK_CLAIM_BOUNDARY: + errors.append("soak claim_boundary must remain target_soak_probe_not_scorecard_update") + if report.get("status") != "passed": + errors.append("soak status must be passed") + if _int_value(report.get("duration_seconds")) < minimum_duration_seconds: + errors.append(f"soak duration_seconds must be >= {minimum_duration_seconds}") + if _int_value(report.get("runtime_failure_count"), default=1) != 0: + errors.append("soak runtime_failure_count must be zero") + if _int_value(report.get("row_count")) < 1: + errors.append("soak row_count must be positive") + if _int_value(report.get("accepted_count")) < 1: + errors.append("soak accepted_count must be positive") + if require_distributed and report.get("ray_local_mode") is not False: + errors.append("soak requires distributed Ray, not local_mode") + return errors + + +def build_m12_soak_report_from_package( + *, + package_path: Path, + duration_seconds: int = 7200, + minimum_duration_seconds: int = 7200, + interval_seconds: float = 30.0, + num_workers: int = 20, + rows_per_iteration: int = 10, + min_iterations: int = 1, + local_mode: bool = False, + target_run_id: str | None = None, +) -> dict[str, Any]: + return build_m12_soak_report( + package=load_env_package(package_path), + duration_seconds=duration_seconds, + minimum_duration_seconds=minimum_duration_seconds, + interval_seconds=interval_seconds, + num_workers=num_workers, + rows_per_iteration=rows_per_iteration, + min_iterations=min_iterations, + local_mode=local_mode, + target_run_id=target_run_id, + ) diff --git a/breadboard/rl/m12/preflight.py b/breadboard/rl/m12/preflight.py new file mode 100644 index 00000000..10d8f8c9 --- /dev/null +++ b/breadboard/rl/m12/preflight.py @@ -0,0 +1,591 @@ +from __future__ import annotations + +import importlib.util +import hashlib +import json +import os +import platform +import shutil +import subprocess +import sys +import time +from pathlib import Path, PureWindowsPath +from tempfile import TemporaryDirectory +from typing import Any + + +SAFE_ENV_KEYS = [ + "CUDA_VISIBLE_DEVICES", + "HIP_VISIBLE_DEVICES", + "ROCR_VISIBLE_DEVICES", + "HSA_VISIBLE_DEVICES", + "RAY_ADDRESS", + "CONDA_DEFAULT_ENV", +] +ENV_VALUE_REDACTED_ABSOLUTE_PATH = "" +ENV_REDACTION_REASON_ABSOLUTE_PATH = "absolute_or_home_path" +ENV_SNAPSHOT_POLICY = "allowlist_only_redact_path_values_no_secret_keys_no_absolute_python_paths" +RUNTIME_FINGERPRINT_ID = "bb_zyphra_rl_phase1_m12_runtime_fingerprint_v1" +RUNTIME_FINGERPRINT_KEYS = { + "fingerprint_id", + "gpu_summary", + "platform", + "python_modules", + "ray_summary", + "sanitized_environment", + "sha256", + "tool_presence", +} +RUNTIME_FINGERPRINT_PLATFORM_KEYS = { + "machine", + "python", + "python_executable_name", + "python_implementation", + "release", + "system", +} +RUNTIME_FINGERPRINT_TOOL_PRESENCE_KEYS = { + "docker", + "firecracker", + "gvisor_runsc", + "rocm_smi", + "rocminfo", +} +RUNTIME_FINGERPRINT_PYTHON_MODULE_KEYS = {"ray", "sglang", "torch", "verl", "vllm"} +RUNTIME_FINGERPRINT_GPU_SUMMARY_KEYS = {"rocm_smi_output", "torch_probe"} +RUNTIME_FINGERPRINT_RAY_SUMMARY_KEYS = {"status_output"} +RUNTIME_FINGERPRINT_SANITIZED_ENVIRONMENT_KEYS = {"keys", "policy", "redactions", "values"} +PYTHON_MODULE_KEYS = {"torch", "ray", "verl", "vllm", "sglang"} +CONTAINER_RUNTIME_KEYS = {"docker", "gvisor_runsc", "firecracker"} +PREFLIGHT_STATUS_VALUES = {"blocked", "preflight_passed"} + + +def _command_available(command: str) -> bool: + return shutil.which(command) is not None + + +def _command_output(command: list[str]) -> str: + try: + return subprocess.check_output(command, text=True, stderr=subprocess.STDOUT, timeout=15).strip() + except Exception as exc: + return f"unavailable: {type(exc).__name__}: {exc}" + + +def _env_value_is_path_like(value: str) -> bool: + stripped = value.strip() + if not stripped: + return False + if stripped.startswith("~"): + return True + return Path(stripped).is_absolute() or PureWindowsPath(stripped).is_absolute() + + +def _safe_environment_snapshot() -> dict[str, dict[str, str]]: + values: dict[str, str] = {} + redactions: dict[str, str] = {} + for key in SAFE_ENV_KEYS: + if key not in os.environ: + continue + raw_value = str(os.environ[key]) + if _env_value_is_path_like(raw_value): + values[key] = ENV_VALUE_REDACTED_ABSOLUTE_PATH + redactions[key] = ENV_REDACTION_REASON_ABSOLUTE_PATH + else: + values[key] = raw_value + return {"values": values, "redactions": redactions} + + +def _stable_sha256(payload: dict[str, Any]) -> str: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _dict_has_exact_keys(raw: Any, expected_keys: set[str]) -> bool: + return isinstance(raw, dict) and {str(key) for key in raw.keys()} == expected_keys + + +def _stable_runtime_fingerprint_sha256(fingerprint: dict[str, Any]) -> str | None: + payload = dict(fingerprint) + observed = payload.pop("sha256", None) + if not isinstance(observed, str) or not observed.startswith("sha256:"): + return None + return _stable_sha256(payload) + + +def _runtime_fingerprint_valid(fingerprint: Any) -> bool: + if not _dict_has_exact_keys(fingerprint, RUNTIME_FINGERPRINT_KEYS): + return False + if fingerprint.get("fingerprint_id") != RUNTIME_FINGERPRINT_ID: + return False + if _stable_runtime_fingerprint_sha256(fingerprint) != fingerprint.get("sha256"): + return False + if not _dict_has_exact_keys(fingerprint.get("platform"), RUNTIME_FINGERPRINT_PLATFORM_KEYS): + return False + if not _dict_has_exact_keys(fingerprint.get("tool_presence"), RUNTIME_FINGERPRINT_TOOL_PRESENCE_KEYS): + return False + if not _dict_has_exact_keys(fingerprint.get("python_modules"), RUNTIME_FINGERPRINT_PYTHON_MODULE_KEYS): + return False + if not _dict_has_exact_keys(fingerprint.get("gpu_summary"), RUNTIME_FINGERPRINT_GPU_SUMMARY_KEYS): + return False + if not _dict_has_exact_keys(fingerprint.get("ray_summary"), RUNTIME_FINGERPRINT_RAY_SUMMARY_KEYS): + return False + sanitized = fingerprint.get("sanitized_environment") + if not _dict_has_exact_keys(sanitized, RUNTIME_FINGERPRINT_SANITIZED_ENVIRONMENT_KEYS): + return False + if sanitized.get("policy") != ENV_SNAPSHOT_POLICY: + return False + raw_keys = sanitized.get("keys") + values = sanitized.get("values") + redactions = sanitized.get("redactions") + if not isinstance(raw_keys, list) or not isinstance(values, dict) or not isinstance(redactions, dict): + return False + keys = {str(key) for key in raw_keys} + value_keys = {str(key) for key in values} + redaction_keys = {str(key) for key in redactions} + if keys != set(SAFE_ENV_KEYS): + return False + if not value_keys.issubset(set(SAFE_ENV_KEYS)): + return False + if not redaction_keys.issubset(value_keys): + return False + forbidden_key_parts = ("KEY", "TOKEN", "SECRET", "PASSWORD", "AUTH") + if any(part in key.upper() for key in keys | value_keys for part in forbidden_key_parts): + return False + for key, raw_value in values.items(): + value = str(raw_value) + if key in redactions: + if value != ENV_VALUE_REDACTED_ABSOLUTE_PATH: + return False + if redactions.get(key) != ENV_REDACTION_REASON_ABSOLUTE_PATH: + return False + continue + if value == ENV_VALUE_REDACTED_ABSOLUTE_PATH: + return False + if _env_value_is_path_like(value): + return False + python_executable_name = str((fingerprint.get("platform") or {}).get("python_executable_name") or "") + if "/" in python_executable_name or "\\" in python_executable_name: + return False + return True + + +def _runtime_fingerprint( + *, + torch_info: dict[str, Any], + ray_info: dict[str, Any], + verl_info: dict[str, Any], + vllm_info: dict[str, Any], + sglang_info: dict[str, Any], + rocm_smi_available: bool, + rocminfo_available: bool, + docker_available: bool, + runsc_available: bool, + firecracker_available: bool, + rocm_smi_output: str, + torch_probe: dict[str, Any], + ray_status_output: str, +) -> dict[str, Any]: + env_snapshot = _safe_environment_snapshot() + payload: dict[str, Any] = { + "fingerprint_id": "bb_zyphra_rl_phase1_m12_runtime_fingerprint_v1", + "platform": { + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "python": platform.python_version(), + "python_implementation": platform.python_implementation(), + "python_executable_name": Path(sys.executable).name, + }, + "tool_presence": { + "rocm_smi": rocm_smi_available, + "rocminfo": rocminfo_available, + "docker": docker_available, + "gvisor_runsc": runsc_available, + "firecracker": firecracker_available, + }, + "python_modules": { + "torch": torch_info, + "ray": ray_info, + "verl": verl_info, + "vllm": vllm_info, + "sglang": sglang_info, + }, + "gpu_summary": { + "rocm_smi_output": rocm_smi_output, + "torch_probe": torch_probe, + }, + "ray_summary": { + "status_output": ray_status_output, + }, + "sanitized_environment": { + "policy": ENV_SNAPSHOT_POLICY, + "keys": list(SAFE_ENV_KEYS), + "values": env_snapshot["values"], + "redactions": env_snapshot["redactions"], + }, + } + return {**payload, "sha256": _stable_sha256(payload)} + + +def _module_info_valid(raw: Any) -> bool: + return isinstance(raw, dict) and isinstance(raw.get("available"), bool) and "version" in raw + + +def _module_available(modules: dict[str, Any], key: str) -> bool: + entry = modules.get(key) + return isinstance(entry, dict) and entry.get("available") is True + + +def _int_or_zero(raw: Any) -> int: + try: + return int(raw) + except (TypeError, ValueError): + return 0 + + +def _runtime_fingerprint_matches_report(report: dict[str, Any]) -> bool: + fingerprint = report.get("runtime_fingerprint") + if not isinstance(fingerprint, dict): + return False + gpu = report.get("gpu") + modules = report.get("python_modules") + ray_cluster = report.get("ray_cluster") + containers = report.get("container_runtimes") + if not isinstance(gpu, dict) or not isinstance(modules, dict) or not isinstance(ray_cluster, dict): + return False + if not isinstance(containers, dict): + return False + expected_tool_presence = { + "rocm_smi": gpu.get("rocm_smi_available"), + "rocminfo": gpu.get("rocminfo_available"), + "docker": containers.get("docker"), + "gvisor_runsc": containers.get("gvisor_runsc"), + "firecracker": containers.get("firecracker"), + } + expected_gpu_summary = { + "rocm_smi_output": gpu.get("rocm_smi_output"), + "torch_probe": gpu.get("torch_probe"), + } + expected_ray_summary = {"status_output": ray_cluster.get("status_output")} + return ( + fingerprint.get("tool_presence") == expected_tool_presence + and fingerprint.get("python_modules") == modules + and fingerprint.get("gpu_summary") == expected_gpu_summary + and fingerprint.get("ray_summary") == expected_ray_summary + ) + + +def validate_m12_preflight_report(report: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != "bb_zyphra_rl_phase1_m12_preflight_v1": + errors.append("report_id must be bb_zyphra_rl_phase1_m12_preflight_v1") + if report.get("claim_boundary") != "target_preflight_only_not_m12_validation": + errors.append("claim_boundary must remain target_preflight_only_not_m12_validation") + status = report.get("status") + blockers = report.get("blockers") + if status not in PREFLIGHT_STATUS_VALUES: + errors.append("status must be blocked or preflight_passed") + if not isinstance(blockers, list) or any(not isinstance(item, str) or not item for item in blockers): + errors.append("blockers must be a list of non-empty strings") + elif status == "preflight_passed" and blockers: + errors.append("preflight_passed requires blockers=[]") + elif status == "blocked" and not blockers: + errors.append("blocked preflight requires at least one blocker") + + system = report.get("system") + if not isinstance(system, dict) or not isinstance(system.get("platform"), str) or not isinstance(system.get("python"), str): + errors.append("system must include platform and python strings") + + gpu = report.get("gpu") + if not isinstance(gpu, dict): + errors.append("gpu must be an object") + else: + if gpu.get("required_accelerator_count") != 8: + errors.append("gpu.required_accelerator_count must be 8") + if gpu.get("required_accelerator_family") != "MI300X": + errors.append("gpu.required_accelerator_family must be MI300X") + for key in ["rocminfo_available", "rocm_smi_available", "mi300x_product_evidence"]: + if not isinstance(gpu.get(key), bool): + errors.append(f"gpu.{key} must be boolean") + if not isinstance(gpu.get("rocm_smi_output"), str): + errors.append("gpu.rocm_smi_output must be a string") + torch_probe = gpu.get("torch_probe") + if not isinstance(torch_probe, dict) or not isinstance(torch_probe.get("parse_error"), bool): + errors.append("gpu.torch_probe must include parse_error boolean") + + python_modules = report.get("python_modules") + if not isinstance(python_modules, dict) or {str(key) for key in python_modules.keys()} != PYTHON_MODULE_KEYS: + errors.append("python_modules must include exactly torch, ray, verl, vllm, and sglang") + elif any(not _module_info_valid(python_modules.get(key)) for key in PYTHON_MODULE_KEYS): + errors.append("python_modules entries must include available boolean and version") + + ray_cluster = report.get("ray_cluster") + if ( + not isinstance(ray_cluster, dict) + or not isinstance(ray_cluster.get("module_available"), bool) + or not isinstance(ray_cluster.get("status_output"), str) + ): + errors.append("ray_cluster must include module_available boolean and status_output string") + + inference = report.get("inference_engine_feasibility") + if not isinstance(inference, dict): + errors.append("inference_engine_feasibility must be an object") + else: + for key in ["vllm_available", "sglang_available"]: + if not isinstance(inference.get(key), bool): + errors.append(f"inference_engine_feasibility.{key} must be boolean") + if inference.get("decision") not in {"available", "blocked_no_vllm_or_sglang"}: + errors.append("inference_engine_feasibility.decision is invalid") + + filesystem = report.get("filesystem_cas_smoke") + if not isinstance(filesystem, dict): + errors.append("filesystem_cas_smoke must be an object") + else: + if filesystem.get("status") not in {"passed", "failed"}: + errors.append("filesystem_cas_smoke.status must be passed or failed") + if not str(filesystem.get("sha256") or "").startswith("sha256:"): + errors.append("filesystem_cas_smoke.sha256 must be recorded") + + container_runtimes = report.get("container_runtimes") + if not isinstance(container_runtimes, dict) or {str(key) for key in container_runtimes.keys()} != CONTAINER_RUNTIME_KEYS: + errors.append("container_runtimes must include exactly docker, gvisor_runsc, and firecracker") + elif any(not isinstance(container_runtimes.get(key), bool) for key in CONTAINER_RUNTIME_KEYS): + errors.append("container_runtimes entries must be boolean") + + if not _runtime_fingerprint_valid(report.get("runtime_fingerprint")): + errors.append("runtime_fingerprint must be exact-schema, self-hash-verified, path-redacted, and allowlisted") + elif not _runtime_fingerprint_matches_report(report): + errors.append("runtime_fingerprint must match top-level preflight evidence") + if status == "preflight_passed": + gpu = report.get("gpu") if isinstance(report.get("gpu"), dict) else {} + torch_probe = gpu.get("torch_probe") if isinstance(gpu.get("torch_probe"), dict) else {} + modules = report.get("python_modules") if isinstance(report.get("python_modules"), dict) else {} + ray_cluster = report.get("ray_cluster") if isinstance(report.get("ray_cluster"), dict) else {} + inference = report.get("inference_engine_feasibility") if isinstance(report.get("inference_engine_feasibility"), dict) else {} + filesystem = report.get("filesystem_cas_smoke") if isinstance(report.get("filesystem_cas_smoke"), dict) else {} + containers = report.get("container_runtimes") if isinstance(report.get("container_runtimes"), dict) else {} + if not (gpu.get("rocminfo_available") is True or gpu.get("rocm_smi_available") is True): + errors.append("preflight_passed requires ROCm tooling availability") + if not _module_available(modules, "torch"): + errors.append("preflight_passed requires torch availability") + if torch_probe.get("parse_error") is True: + errors.append("preflight_passed requires parseable torch probe") + if _int_or_zero(torch_probe.get("device_count")) < 8: + errors.append("preflight_passed requires torch device_count >= 8") + if gpu.get("mi300x_product_evidence") is not True: + errors.append("preflight_passed requires MI300X product evidence") + if not _module_available(modules, "ray") or ray_cluster.get("module_available") is not True: + errors.append("preflight_passed requires Ray availability") + if not _module_available(modules, "verl"): + errors.append("preflight_passed requires VeRL availability") + if not ( + inference.get("decision") == "available" + and (inference.get("vllm_available") is True or inference.get("sglang_available") is True) + ): + errors.append("preflight_passed requires vLLM or SGLang availability") + if not any(value is True for value in containers.values()): + errors.append("preflight_passed requires at least one container runtime") + if filesystem.get("status") != "passed": + errors.append("preflight_passed requires filesystem/CAS smoke passed") + if not isinstance(report.get("m12_completion_gate"), list) or not report["m12_completion_gate"]: + errors.append("m12_completion_gate must be a non-empty list") + if report.get("required_next_step") != "Run on target 8xMI300X node before awarding M12 points.": + errors.append("required_next_step must preserve the M12 target-node boundary") + return errors + + +def _module_version(module_name: str) -> dict[str, Any]: + spec = importlib.util.find_spec(module_name) + if spec is None: + return {"available": False, "version": None} + try: + module = __import__(module_name) + return {"available": True, "version": str(getattr(module, "__version__", "unknown"))} + except Exception as exc: + return {"available": False, "version": f"import_error:{type(exc).__name__}:{exc}"} + + +def _torch_probe() -> dict[str, Any]: + output = _command_output( + [ + sys.executable, + "-c", + ( + "import torch, json; " + "print(json.dumps({" + "'cuda_available': torch.cuda.is_available(), " + "'device_count': torch.cuda.device_count(), " + "'device_names': [torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]" + "}))" + ), + ] + ) + try: + parsed = json.loads(output) + except json.JSONDecodeError: + return {"raw": output, "parse_error": True} + return {"raw": output, "parse_error": False, **parsed} + + +def _rocm_names_indicate_mi300x(rocm_smi_output: str, torch_probe: dict[str, Any]) -> bool: + haystack = rocm_smi_output + "\n" + "\n".join(str(name) for name in torch_probe.get("device_names", [])) + normalized = haystack.lower() + return "mi300x" in normalized or "mi300" in normalized + + +def _filesystem_cas_smoke() -> dict[str, Any]: + payload = b"breadboard-m12-cas-smoke\n" * 4096 + try: + with TemporaryDirectory(prefix="bb_m12_cas_smoke_") as tmp: + path = Path(tmp) / "blob.bin" + start_write = time.perf_counter() + path.write_bytes(payload) + write_ms = (time.perf_counter() - start_write) * 1000 + + start_read = time.perf_counter() + observed = path.read_bytes() + read_ms = (time.perf_counter() - start_read) * 1000 + + start_hash = time.perf_counter() + digest = "sha256:" + hashlib.sha256(observed).hexdigest() + hash_ms = (time.perf_counter() - start_hash) * 1000 + return { + "status": "passed" if observed == payload else "failed", + "bytes": len(payload), + "sha256": digest, + "write_ms": round(write_ms, 3), + "read_ms": round(read_ms, 3), + "hash_ms": round(hash_ms, 3), + "failure_mode": None if observed == payload else "read_bytes_did_not_match_written_payload", + } + except Exception as exc: + return { + "status": "failed", + "bytes": len(payload), + "sha256": None, + "write_ms": None, + "read_ms": None, + "hash_ms": None, + "failure_mode": f"{type(exc).__name__}: {exc}", + } + + +def run_m12_preflight() -> dict[str, Any]: + torch_info = _module_version("torch") + ray_info = _module_version("ray") + verl_info = _module_version("verl") + vllm_info = _module_version("vllm") + sglang_info = _module_version("sglang") + rocm_smi_available = _command_available("rocm-smi") + rocminfo_available = _command_available("rocminfo") + docker_available = _command_available("docker") + runsc_available = _command_available("runsc") + firecracker_available = _command_available("firecracker") + filesystem_cas_smoke = _filesystem_cas_smoke() + + torch_probe = _torch_probe() if torch_info["available"] else {"raw": "torch unavailable", "parse_error": False} + rocm_smi_output = _command_output(["rocm-smi", "--showproductname"]) if rocm_smi_available else "unavailable" + ray_status_output = _command_output(["ray", "status"]) if _command_available("ray") else "ray CLI unavailable" + gpu_report = { + "required_accelerator_count": 8, + "required_accelerator_family": "MI300X", + "rocminfo_available": rocminfo_available, + "rocm_smi_available": rocm_smi_available, + "rocm_smi_output": rocm_smi_output, + "torch_probe": torch_probe, + "mi300x_product_evidence": _rocm_names_indicate_mi300x(rocm_smi_output, torch_probe), + } + + blockers: list[str] = [] + if not (rocminfo_available or rocm_smi_available): + blockers.append("rocm_tools_unavailable") + if not torch_info["available"]: + blockers.append("torch_unavailable") + elif torch_probe.get("parse_error"): + blockers.append("torch_probe_unparseable") + elif int(torch_probe.get("device_count", 0)) < 8: + blockers.append("torch_device_count_below_8") + if rocm_smi_available and not gpu_report["mi300x_product_evidence"]: + blockers.append("mi300x_product_not_detected") + if not ray_info["available"]: + blockers.append("ray_unavailable") + if not verl_info["available"]: + blockers.append("verl_unavailable") + if not (vllm_info["available"] or sglang_info["available"]): + blockers.append("no_vllm_or_sglang") + if not (docker_available or runsc_available or firecracker_available): + blockers.append("no_container_runtime_detected") + if filesystem_cas_smoke["status"] != "passed": + blockers.append("filesystem_cas_smoke_failed") + + return { + "report_id": "bb_zyphra_rl_phase1_m12_preflight_v1", + "claim_boundary": "target_preflight_only_not_m12_validation", + "target_run_id": os.environ.get("M12_TARGET_RUN_ID"), + "status": "blocked" if blockers else "preflight_passed", + "blockers": blockers, + "system": { + "platform": platform.platform(), + "python": platform.python_version(), + }, + "gpu": gpu_report, + "python_modules": { + "torch": torch_info, + "ray": ray_info, + "verl": verl_info, + "vllm": vllm_info, + "sglang": sglang_info, + }, + "ray_cluster": { + "module_available": ray_info["available"], + "status_output": ray_status_output, + }, + "inference_engine_feasibility": { + "vllm_available": vllm_info["available"], + "sglang_available": sglang_info["available"], + "decision": "available" if (vllm_info["available"] or sglang_info["available"]) else "blocked_no_vllm_or_sglang", + }, + "filesystem_cas_smoke": filesystem_cas_smoke, + "container_runtimes": { + "docker": docker_available, + "gvisor_runsc": runsc_available, + "firecracker": firecracker_available, + }, + "runtime_fingerprint": _runtime_fingerprint( + torch_info=torch_info, + ray_info=ray_info, + verl_info=verl_info, + vllm_info=vllm_info, + sglang_info=sglang_info, + rocm_smi_available=rocm_smi_available, + rocminfo_available=rocminfo_available, + docker_available=docker_available, + runsc_available=runsc_available, + firecracker_available=firecracker_available, + rocm_smi_output=rocm_smi_output, + torch_probe=torch_probe, + ray_status_output=ray_status_output, + ), + "m12_completion_gate": [ + "preflight_passed on target 8xMI300X node", + "full local test suite passes on target node", + "controlled SWE probe rerun succeeds on target node", + "VeRL-shaped export smoke consumer succeeds on target node", + "Ray warm-pool probe succeeds on target node", + "final M12 report records hardware/software versions, commands, outputs, and caveats", + ], + "required_next_step": "Run on target 8xMI300X node before awarding M12 points.", + } + + +def write_m12_preflight_report(output_dir: Path) -> dict[str, Any]: + report = run_m12_preflight() + errors = validate_m12_preflight_report(report) + if errors: + raise ValueError("invalid M12 preflight report: " + "; ".join(errors)) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "m12_preflight_report.json").write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return report diff --git a/breadboard/rl/m12/promotion_audit.py b/breadboard/rl/m12/promotion_audit.py new file mode 100644 index 00000000..16745f64 --- /dev/null +++ b/breadboard/rl/m12/promotion_audit.py @@ -0,0 +1,469 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import yaml + +from breadboard.rl.m12.command_logs import ( + resolve_manifest_log_path, + sha256_file, + validate_command_log_manifest, +) +from breadboard.rl.m12.final_report import ( + FINAL_REPORT_ID, + OPTIONAL_COMMAND_LOG_COMMANDS, + REQUIRED_COMMAND_LOG_COMMANDS, + REQUIRED_COMMAND_LOG_IDS, + TARGET_CLAIM_LEDGER_PATH, + TARGET_COMMAND_LOG_MANIFEST_PATH, + TARGET_FINAL_REPORT_PATH, + TARGET_PROMOTION_AUDIT_PATH, + TARGET_SCORECARD_PATH, + validate_m12_final_report, +) + + +PROMOTION_AUDIT_ID = "bb_zyphra_rl_phase1_m12_promotion_audit_v1" +PROMOTION_AUDIT_CLAIM_BOUNDARY = "promotion_review_only_not_scorecard_update" +FINAL_REPORT_COMMAND_ID = "final_report" + + +def _read_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _missing_read_error(kind: str) -> str: + return f"FileNotFoundError: {kind} is missing" + + +def _read_optional_json_object(path: Path) -> dict[str, Any]: + if not path.exists(): + return { + "present": False, + "path": str(path), + "read_error": _missing_read_error("JSON artifact"), + } + try: + payload = _read_json(path) + except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc: + return { + "present": True, + "path": str(path), + "read_error": f"{exc.__class__.__name__}: {exc}", + } + if not isinstance(payload, dict): + return { + "present": True, + "path": str(path), + "read_error": f"expected JSON object, got {type(payload).__name__}", + } + return payload + + +def _read_optional_yaml_object(path: Path) -> dict[str, Any]: + if not path.exists(): + return { + "present": False, + "path": str(path), + "read_error": _missing_read_error("YAML artifact"), + } + try: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError, UnicodeDecodeError) as exc: + return { + "present": True, + "path": str(path), + "read_error": f"{exc.__class__.__name__}: {exc}", + } + if not isinstance(payload, dict): + return { + "present": True, + "path": str(path), + "read_error": f"expected YAML object, got {type(payload).__name__}", + } + payload.setdefault("present", True) + payload.setdefault("path", str(path)) + return payload + + +def _read_optional_text(path: Path) -> dict[str, Any]: + if not path.exists(): + return { + "present": False, + "path": str(path), + "text": "", + "read_error": _missing_read_error("text artifact"), + } + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + return { + "present": True, + "path": str(path), + "text": "", + "read_error": f"{exc.__class__.__name__}: {exc}", + } + return { + "present": True, + "path": str(path), + "text": text, + "read_error": None, + } + + +def _sha_or_missing(path: Path) -> str | None: + if not path.is_file(): + return None + return sha256_file(path) + + +def _m12_milestone(scorecard: dict[str, Any]) -> dict[str, Any]: + for milestone in scorecard.get("milestones") or []: + if isinstance(milestone, dict) and milestone.get("id") == "M12": + return milestone + return {} + + +def _command_entries(manifest: dict[str, Any]) -> list[dict[str, Any]]: + return [item for item in manifest.get("commands") or [] if isinstance(item, dict)] + + +def _entry_by_id(manifest: dict[str, Any], command_id: str) -> dict[str, Any] | None: + return next((item for item in _command_entries(manifest) if item.get("command_id") == command_id), None) + + +def _required_command_text_mismatches(manifest: dict[str, Any]) -> list[dict[str, str]]: + entries_by_id = {str(entry.get("command_id") or ""): entry for entry in _command_entries(manifest)} + mismatches: list[dict[str, str]] = [] + for command_id, expected in REQUIRED_COMMAND_LOG_COMMANDS.items(): + entry = entries_by_id.get(command_id) + if entry is None: + continue + observed = str(entry.get("command") or "") + if observed != expected: + mismatches.append({"command_id": command_id, "expected": expected, "observed": observed}) + return mismatches + + +def _required_final_report_row_mismatches( + *, + final_report: dict[str, Any], + manifest: dict[str, Any], +) -> list[dict[str, str]]: + final_report_entries = { + str(entry.get("command_id") or ""): entry + for entry in (final_report.get("command_logs") or {}).get("commands") or [] + if isinstance(entry, dict) + } + manifest_entries = { + str(entry.get("command_id") or ""): entry + for entry in _command_entries(manifest) + } + mismatches: list[dict[str, str]] = [] + for command_id in REQUIRED_COMMAND_LOG_IDS: + final_report_entry = final_report_entries.get(command_id) + manifest_entry = manifest_entries.get(command_id) + if final_report_entry is None: + mismatches.append({"command_id": command_id, "reason": "missing_from_final_report"}) + continue + if manifest_entry is None: + mismatches.append({"command_id": command_id, "reason": "missing_from_manifest"}) + continue + if final_report_entry != manifest_entry: + mismatches.append({"command_id": command_id, "reason": "row_differs_from_manifest"}) + return mismatches + + +def _optional_command_hash_verified(manifest_path: Path, entry: dict[str, Any] | None) -> bool: + if not entry: + return False + raw_log_path = entry.get("log_path") + expected_sha = str(entry.get("sha256") or "") + if not raw_log_path or not expected_sha.startswith("sha256:"): + return False + resolved = resolve_manifest_log_path(manifest_path, str(raw_log_path)) + return resolved.is_file() and sha256_file(resolved) == expected_sha + + +def _path_matches_target(path: Path, target_path: str) -> bool: + normalized = str(path).replace("\\", "/") + suffixes = { + target_path, + target_path.removeprefix("../"), + } + return normalized in suffixes or any(normalized.endswith(f"/{suffix}") for suffix in suffixes) + + +def _bool_checks_missing(sections: dict[str, dict[str, bool]]) -> list[str]: + missing: list[str] = [] + for section, checks in sections.items(): + for name, passed in checks.items(): + if not passed: + missing.append(f"{section}.{name}") + return missing + + +def _refresh_audit_readiness(audit: dict[str, Any]) -> None: + checks = audit.get("checks") if isinstance(audit.get("checks"), dict) else {} + missing_requirements = _bool_checks_missing(checks) + audit["missing_requirements"] = missing_requirements + audit["promotion_review_ready"] = not missing_requirements + + +def _audit_check_summary(audit: dict[str, Any]) -> tuple[list[str], list[str]]: + checks = audit.get("checks") + if not isinstance(checks, dict) or not checks: + return [], [] + errors: list[str] = [] + missing: list[str] = [] + for section, section_checks in checks.items(): + section_name = str(section) + if not isinstance(section_checks, dict): + errors.append(f"checks.{section_name} must be an object") + missing.append(f"{section_name}.__section__") + continue + for name, passed in section_checks.items(): + check_name = str(name) + if passed is not True and passed is not False: + errors.append(f"checks.{section_name}.{check_name} must be boolean") + if passed is not True: + missing.append(f"{section_name}.{check_name}") + return errors, missing + + +def build_m12_promotion_audit( + *, + final_report_path: Path, + scorecard_path: Path, + claim_ledger_path: Path, + command_log_manifest_path: Path, + output_path: Path | None = None, +) -> dict[str, Any]: + final_report = _read_optional_json_object(final_report_path) + scorecard = _read_optional_yaml_object(scorecard_path) + claim_ledger = _read_optional_text(claim_ledger_path) + claim_ledger_text = str(claim_ledger.get("text") or "") + command_log_manifest = _read_optional_json_object(command_log_manifest_path) + + if final_report.get("read_error"): + final_report_errors = [f"final report unreadable: {final_report.get('read_error')}"] + else: + final_report_errors = validate_m12_final_report(final_report) + if command_log_manifest.get("read_error"): + command_log_errors = [ + f"command log manifest unreadable: {command_log_manifest.get('read_error')}" + ] + else: + command_log_errors = validate_command_log_manifest( + command_log_manifest_path, + required_command_ids=list(REQUIRED_COMMAND_LOG_IDS), + require_passed=True, + verify_hashes=True, + ) + m12 = _m12_milestone(scorecard) + final_report_command = _entry_by_id(command_log_manifest, FINAL_REPORT_COMMAND_ID) + final_report_command_hash_verified = _optional_command_hash_verified(command_log_manifest_path, final_report_command) + final_report_command_text_matches = bool( + final_report_command + and final_report_command.get("command") == OPTIONAL_COMMAND_LOG_COMMANDS[FINAL_REPORT_COMMAND_ID] + ) + final_report_target_run_id = (final_report.get("command_logs") or {}).get("single_target_run_id") + final_report_command_target_run_matches = bool( + final_report_target_run_id + and final_report_command + and final_report_command.get("target_run_id") == final_report_target_run_id + ) + final_report_sha = _sha_or_missing(final_report_path) + scorecard_sha = _sha_or_missing(scorecard_path) + claim_ledger_sha = _sha_or_missing(claim_ledger_path) + command_log_manifest_sha = _sha_or_missing(command_log_manifest_path) + current_verified_points = scorecard.get("current_verified_points") + milestone_verified_sum = sum( + int(milestone.get("verified_points", 0)) + for milestone in scorecard.get("milestones") or [] + if isinstance(milestone, dict) + ) + m12_pre_review_state = m12.get("verified_points") == 0 and m12.get("status") != "completed" + m12_post_review_state = ( + current_verified_points == 1000 + and m12.get("verified_points") == 80 + and m12.get("status") == "completed" + ) + required_ids = set(REQUIRED_COMMAND_LOG_IDS) + manifest_command_ids = {str(entry.get("command_id") or "") for entry in _command_entries(command_log_manifest)} + required_command_text_mismatches = _required_command_text_mismatches(command_log_manifest) + final_report_required_command_row_mismatches = _required_final_report_row_mismatches( + final_report=final_report, + manifest=command_log_manifest, + ) + required_command_text_matches = bool( + command_log_manifest_path.is_file() + and required_ids <= manifest_command_ids + and not required_command_text_mismatches + ) + command_log_section = final_report.get("command_logs") or {} + archived_ids = set(command_log_section.get("archived_command_ids") or []) + hash_verified_ids = set(command_log_section.get("hash_verified_command_ids") or []) + + sections = { + "final_report": { + "path_exists": final_report_path.is_file(), + "path_matches_target_default": _path_matches_target(final_report_path, TARGET_FINAL_REPORT_PATH), + "sha256_recorded": final_report_sha is not None, + "report_id_valid": final_report.get("report_id") == FINAL_REPORT_ID, + "schema_valid": not final_report_errors, + "score_eligible": final_report.get("m12_score_eligible") is True, + "missing_gates_empty": final_report.get("missing_gates") == [], + "scorecard_update_disallowed": final_report.get("scorecard_update_allowed") is False, + "required_command_logs_archived": required_ids <= archived_ids, + "required_command_logs_hash_verified": required_ids <= hash_verified_ids, + }, + "command_log_manifest": { + "path_exists": command_log_manifest_path.is_file(), + "path_matches_target_default": _path_matches_target( + command_log_manifest_path, + TARGET_COMMAND_LOG_MANIFEST_PATH, + ), + "sha256_recorded": command_log_manifest_sha is not None, + "required_manifest_valid": not command_log_errors, + "required_command_text_matches": required_command_text_matches, + "final_report_required_rows_match_manifest": not final_report_required_command_row_mismatches, + "final_report_command_logged": final_report_command is not None, + "final_report_command_text_matches": final_report_command_text_matches, + "final_report_command_passed": bool(final_report_command and final_report_command.get("status") == "passed"), + "final_report_command_hash_verified": final_report_command_hash_verified, + "final_report_command_target_run_matches": final_report_command_target_run_matches, + }, + "scorecard": { + "path_exists": scorecard_path.is_file(), + "path_matches_target_default": _path_matches_target(scorecard_path, TARGET_SCORECARD_PATH), + "sha256_recorded": scorecard_sha is not None, + "readable": not scorecard.get("read_error"), + "total_points_1000": scorecard.get("total_points") == 1000, + "current_points_match_milestones": current_verified_points == milestone_verified_sum, + "m12_milestone_present": bool(m12), + "m12_state_valid_for_review": m12_pre_review_state or m12_post_review_state, + "m12_post_review_if_awarded": (m12.get("verified_points") != 80) or m12_post_review_state, + "planning_prose_does_not_score": (scorecard.get("score_policy") or {}).get("planning_prose_scores") is False, + "points_require_evidence": (scorecard.get("score_policy") or {}).get("points_require_evidence") is True, + }, + "claim_ledger": { + "path_exists": claim_ledger_path.is_file(), + "path_matches_target_default": _path_matches_target(claim_ledger_path, TARGET_CLAIM_LEDGER_PATH), + "sha256_recorded": claim_ledger_sha is not None, + "readable": not claim_ledger.get("read_error"), + "future_claim_requires_eligible_report": "m12_score_eligible=true" in claim_ledger_text, + "future_claim_requires_raw_command_logs": "raw command logs" in claim_ledger_text, + "future_claim_requires_separate_review": "separately reviewed" in claim_ledger_text, + "m12_claim_boundary_recorded": ("M12 validation remains unawarded" in claim_ledger_text) + or ("BreadBoard passed 8xMI300X final validation" in claim_ledger_text), + }, + "promotion_audit_output": { + "path_recorded": output_path is not None, + "path_matches_target_default": ( + output_path is not None and _path_matches_target(output_path, TARGET_PROMOTION_AUDIT_PATH) + ), + }, + } + missing_requirements = _bool_checks_missing(sections) + promotion_review_ready = not missing_requirements + return { + "audit_id": PROMOTION_AUDIT_ID, + "claim_boundary": PROMOTION_AUDIT_CLAIM_BOUNDARY, + "scorecard_update_allowed": False, + "promotion_review_ready": promotion_review_ready, + "missing_requirements": missing_requirements, + "final_report_validation_errors": final_report_errors, + "final_report_read_error": final_report.get("read_error"), + "command_log_manifest_errors": command_log_errors, + "command_log_manifest_read_error": command_log_manifest.get("read_error"), + "scorecard_read_error": scorecard.get("read_error"), + "claim_ledger_read_error": claim_ledger.get("read_error"), + "required_command_text_mismatches": required_command_text_mismatches, + "final_report_required_command_row_mismatches": final_report_required_command_row_mismatches, + "checks": sections, + "inputs": { + "final_report": {"path": str(final_report_path), "sha256": final_report_sha}, + "scorecard": { + "path": str(scorecard_path), + "sha256": scorecard_sha, + "read_error": scorecard.get("read_error"), + }, + "claim_ledger": { + "path": str(claim_ledger_path), + "sha256": claim_ledger_sha, + "read_error": claim_ledger.get("read_error"), + }, + "command_log_manifest": {"path": str(command_log_manifest_path), "sha256": command_log_manifest_sha}, + }, + "outputs": { + "promotion_audit": {"path": str(output_path) if output_path is not None else None}, + }, + "scorecard_state": { + "current_verified_points": current_verified_points, + "milestone_verified_sum": milestone_verified_sum, + "m12_status": m12.get("status"), + "m12_verified_points": m12.get("verified_points"), + "m12_points": m12.get("points"), + }, + "operator_next_step": ( + "If promotion_review_ready is true, perform a separate human-reviewed scorecard and claim-ledger update. " + "This audit never edits the scorecard." + ), + } + + +def validate_m12_promotion_audit(audit: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if audit.get("audit_id") != PROMOTION_AUDIT_ID: + errors.append("audit_id must be bb_zyphra_rl_phase1_m12_promotion_audit_v1") + if audit.get("claim_boundary") != PROMOTION_AUDIT_CLAIM_BOUNDARY: + errors.append("claim_boundary must remain promotion_review_only_not_scorecard_update") + if audit.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if not isinstance(audit.get("missing_requirements"), list): + errors.append("missing_requirements must be a list") + if not isinstance(audit.get("checks"), dict) or not audit["checks"]: + errors.append("checks must be a non-empty object") + check_errors, expected_missing_requirements = _audit_check_summary(audit) + errors.extend(check_errors) + if isinstance(audit.get("missing_requirements"), list): + observed_missing_requirements = [str(item) for item in audit["missing_requirements"]] + if sorted(observed_missing_requirements) != sorted(expected_missing_requirements): + errors.append("missing_requirements must match promotion-audit embedded checks") + if audit.get("promotion_review_ready") is not (not expected_missing_requirements): + errors.append("promotion_review_ready must match promotion-audit embedded checks") + if bool(audit.get("promotion_review_ready")) and audit.get("missing_requirements"): + errors.append("promotion_review_ready cannot be true while missing_requirements is non-empty") + if bool(audit.get("promotion_review_ready")): + for section, checks in (audit.get("checks") or {}).items(): + if not isinstance(checks, dict): + errors.append(f"checks.{section} must be an object") + continue + for name, passed in checks.items(): + if passed is not True: + errors.append(f"ready audit requires checks.{section}.{name}=true") + return errors + + +def write_m12_promotion_audit( + *, + output_path: Path, + final_report_path: Path, + scorecard_path: Path, + claim_ledger_path: Path, + command_log_manifest_path: Path, +) -> dict[str, Any]: + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=scorecard_path, + claim_ledger_path=claim_ledger_path, + command_log_manifest_path=command_log_manifest_path, + output_path=output_path, + ) + _refresh_audit_readiness(audit) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return audit diff --git a/breadboard/rl/m12/transfer.py b/breadboard/rl/m12/transfer.py new file mode 100644 index 00000000..0e64b5aa --- /dev/null +++ b/breadboard/rl/m12/transfer.py @@ -0,0 +1,2442 @@ +from __future__ import annotations + +import gzip +import hashlib +import io +import json +import subprocess +import tarfile +from pathlib import Path +from typing import Any + +from breadboard.rl.m12.command_logs import COMMAND_LOG_ENTRY_TEMPLATES, COMMAND_LOG_MANIFEST_TEMPLATE +from breadboard.rl.m12.final_report import ( + OPTIONAL_COMMAND_LOG_COMMANDS, + TARGET_ARTIFACT_PATHS, + TARGET_CLAIM_LEDGER_PATH, + TARGET_COMMAND_LOG_MANIFEST_PATH, + TARGET_FINAL_REPORT_PATH, + TARGET_PROMOTION_AUDIT_PATH, + TARGET_SCORECARD_PATH, +) + + +REQUIRED_TRANSFER_ARTIFACTS = [ + "requirements.txt", + "breadboard/rl", + "scripts/rl_phase1", + "tests/__init__.py", + "tests/test_rl_phase1_scorecard_schema.py", + "tests/test_rl_phase1_claim_ledger.py", + "tests/rl", + "docs/rl_phase1", + "examples/rl_env_packages", + "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml", + "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md", + "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_M12_VALIDATION_REPORT.md", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/run_summary.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/run_ledger.jsonl", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/metrics_summary.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/qc_report.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/row_evidence", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/verl_probe_rows.jsonl", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/verl_probe_rows.parquet", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/projection_manifest.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/smoke_consumer_report.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m8_ray_warm_pool_probe/ray_probe_report.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m8_ray_warm_pool_probe/warm_vs_cold_report.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m9_adapter_probes/adapter_probe_summary.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m9_adapter_probes/benchflow.fixture.v1.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m9_adapter_probes/ors.fixture.v1.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m9_adapter_probes/verl.jsonl_probe.v1.json", + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m9_adapter_probes/prime_verifiers.fixture.v1.json", +] + +TRANSFER_PREP_FILES = [ + "m12_transfer_manifest.json", + "m12_test_commands.sh", + "m12_apply_overlay.py", + "m12_target_bootstrap.sh", + "m12_rollback_plan.md", + "m12_readiness_summary.json", + "m12_load_ladder_report_template.json", + "m12_soak_report_template.json", + "m12_command_log_manifest_template.json", + "m12_transfer_summary.json", +] + +M12_LOGGED_COMMAND_IDS = [ + "target_transfer_archive_verify", + "phase1_validation_suite", + "target_preflight", + "target_swe_probe", + "target_verl_export", + "target_ray_warm_pool", + "target_load_ladder", + "target_soak", +] +M12_FINAL_REPORT_COMMAND = OPTIONAL_COMMAND_LOG_COMMANDS["final_report"] +M12_PROMOTION_AUDIT_COMMAND = OPTIONAL_COMMAND_LOG_COMMANDS["promotion_audit"] +_DEFAULT_COMMAND_LOG_MANIFEST_ARG = "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json" +_FINAL_REPORT_OUTPUT_PATH = "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json" +_FINAL_REPORT_TARGET_ARGS = ( + ("--output", _FINAL_REPORT_OUTPUT_PATH), + ("--archive-verify-report", TARGET_ARTIFACT_PATHS["archive_verify_report"]), + ("--preflight-report", TARGET_ARTIFACT_PATHS["preflight_report"]), + ("--swe-run-summary", TARGET_ARTIFACT_PATHS["swe_run_summary"]), + ("--verl-smoke-report", TARGET_ARTIFACT_PATHS["verl_smoke_report"]), + ("--ray-probe-report", TARGET_ARTIFACT_PATHS["ray_probe_report"]), + ("--warm-vs-cold-report", TARGET_ARTIFACT_PATHS["warm_vs_cold_report"]), + ("--load-ladder-report", TARGET_ARTIFACT_PATHS["load_ladder_report"]), + ("--soak-report", TARGET_ARTIFACT_PATHS["soak_report"]), +) +_FINAL_REPORT_MANIFEST_ARGS = ( + *_FINAL_REPORT_TARGET_ARGS, + ("--command-log-manifest", TARGET_ARTIFACT_PATHS["command_log_manifest"]), +) +_FINAL_REPORT_SCRIPT_ARGS = ( + *_FINAL_REPORT_TARGET_ARGS, + ("--command-log-manifest", '"$COMMAND_LOG_MANIFEST"'), +) +_PROMOTION_AUDIT_TARGET_ARGS = ( + ("--output", TARGET_PROMOTION_AUDIT_PATH), + ("--final-report", TARGET_FINAL_REPORT_PATH), + ("--scorecard", TARGET_SCORECARD_PATH), + ("--claim-ledger", TARGET_CLAIM_LEDGER_PATH), + ("--command-log-manifest", TARGET_COMMAND_LOG_MANIFEST_PATH), +) +_PROMOTION_AUDIT_SCRIPT_ARGS = ( + ("--output", TARGET_PROMOTION_AUDIT_PATH), + ("--final-report", TARGET_FINAL_REPORT_PATH), + ("--scorecard", TARGET_SCORECARD_PATH), + ("--claim-ledger", TARGET_CLAIM_LEDGER_PATH), + ("--command-log-manifest", '"$COMMAND_LOG_MANIFEST"'), +) +_CLOSEOUT_ARTIFACT_REUSE_GUARD_TEXT = "Existing M12 close-out artifact would make target evidence ambiguous" + + +def _command_has_flag_value(command: str, flag: str, value: str) -> bool: + return f"{flag} {value}" in command + + +def _final_command_has_explicit_target_args(command: str) -> bool: + return all(_command_has_flag_value(command, flag, value) for flag, value in _FINAL_REPORT_MANIFEST_ARGS) + + +def _promotion_command_has_explicit_target_args(command: str) -> bool: + return all(_command_has_flag_value(command, flag, value) for flag, value in _PROMOTION_AUDIT_TARGET_ARGS) + + +def _build_logged_command_rows() -> list[tuple[str, str]]: + entries_by_id = {str(entry.get("command_id")): str(entry["command"]) for entry in COMMAND_LOG_ENTRY_TEMPLATES} + rows: list[tuple[str, str]] = [] + missing_ids: list[str] = [] + for command_id in M12_LOGGED_COMMAND_IDS: + command = entries_by_id.get(command_id) + if command is None: + missing_ids.append(command_id) + continue + rows.append((command_id, command)) + if missing_ids: + raise ValueError(f"M12 logged command template missing ids: {', '.join(missing_ids)}") + if len(rows) != len(M12_LOGGED_COMMAND_IDS): + raise ValueError("M12 logged command rows must match M12_LOGGED_COMMAND_IDS") + return rows + + +M12_LOGGED_COMMAND_ROWS = _build_logged_command_rows() +M12_LOGGED_COMMANDS = [command for _, command in M12_LOGGED_COMMAND_ROWS] +M12_TEST_COMMAND_ROWS = [ + *M12_LOGGED_COMMAND_ROWS, + ("final_report", M12_FINAL_REPORT_COMMAND), + ("promotion_audit", M12_PROMOTION_AUDIT_COMMAND), +] +M12_TEST_COMMANDS = list(M12_LOGGED_COMMANDS) + [M12_FINAL_REPORT_COMMAND, M12_PROMOTION_AUDIT_COMMAND] + +EXPECTED_OUTPUTS = [ + "m12_archive_verify/m12_archive_verify_report.json", + "m12_target_preflight/m12_preflight_report.json", + "m12_node_swe_probe/run_summary.json", + "m12_node_swe_probe/run_ledger.jsonl", + "m12_node_verl_probe/verl_probe_rows.jsonl", + "m12_node_verl_probe/verl_probe_rows.parquet", + "m12_node_verl_probe/projection_manifest.json", + "m12_node_verl_probe/smoke_consumer_report.json", + "m12_node_ray_probe/ray_probe_report.json", + "m12_node_ray_probe/warm_vs_cold_report.json", + "m12_node_load_ladder/load_ladder_report.json", + "m12_node_soak/soak_report.json", + "m12_command_logs/command_log_manifest.json", + "m12_final_report/m12_final_report.json", + "m12_promotion_audit/m12_promotion_audit.json", +] + +LOAD_LADDER_REPORT_TEMPLATE = { + "report_id": "bb_zyphra_rl_phase1_m12_load_ladder_report_v1", + "claim_boundary": "target_load_ladder_probe_not_scorecard_update", + "concurrency_levels": [ + { + "target_sessions": 5, + "status": "pending", + "started_at": None, + "completed_at": None, + "row_count": None, + "accepted_count": None, + "quarantined_count": None, + "rejected_count": None, + "p95_total_ms": None, + "policy_version_integrity": None, + "queue_backpressure_integrity": None, + "notes": "", + }, + { + "target_sessions": 20, + "status": "pending", + "started_at": None, + "completed_at": None, + "row_count": None, + "accepted_count": None, + "quarantined_count": None, + "rejected_count": None, + "p95_total_ms": None, + "policy_version_integrity": None, + "queue_backpressure_integrity": None, + "notes": "", + }, + { + "target_sessions": 50, + "status": "pending", + "started_at": None, + "completed_at": None, + "row_count": None, + "accepted_count": None, + "quarantined_count": None, + "rejected_count": None, + "p95_total_ms": None, + "policy_version_integrity": None, + "queue_backpressure_integrity": None, + "notes": "", + }, + { + "target_sessions": 100, + "status": "pending_or_resource_skipped", + "started_at": None, + "completed_at": None, + "row_count": None, + "accepted_count": None, + "quarantined_count": None, + "rejected_count": None, + "p95_total_ms": None, + "policy_version_integrity": None, + "queue_backpressure_integrity": None, + "notes": "If not run, add a resource_skips entry with target_sessions=100 and a concrete reason.", + }, + ], + "resource_skips": [], + "policy_version_integrity": None, + "queue_backpressure_integrity": None, + "operator_notes": "", +} + +SOAK_REPORT_TEMPLATE = { + "report_id": "bb_zyphra_rl_phase1_m12_soak_report_v1", + "claim_boundary": "target_soak_probe_not_scorecard_update", + "status": "pending", + "minimum_duration_seconds": 7200, + "duration_seconds": None, + "started_at": None, + "completed_at": None, + "runtime_failure_count": None, + "row_count": None, + "accepted_count": None, + "quarantined_count": None, + "rejected_count": None, + "max_queue_depth": None, + "max_worker_restarts": None, + "operator_notes": "", +} + +TRANSFER_REQUIREMENT_COVERAGE = [ + { + "requirement": "Repo snapshot or commit SHA", + "covered_by": ["repo.head", "repo.branch", "repo.dirty_status_short"], + }, + { + "requirement": "Python environment lock", + "covered_by": ["requirements.txt"], + }, + { + "requirement": "RL Phase 1 source/test/doc overlay", + "covered_by": [ + "breadboard/rl", + "scripts/rl_phase1", + "tests/rl", + "tests/test_rl_phase1_scorecard_schema.py", + "tests/test_rl_phase1_claim_ledger.py", + "docs/rl_phase1", + "examples/rl_env_packages", + ], + }, + { + "requirement": "ROCm/PyTorch/VeRL/Ray versions", + "covered_by": ["m12_target_preflight/m12_preflight_report.json"], + }, + { + "requirement": "EnvPackage set", + "covered_by": [ + "examples/rl_env_packages/python_console_toy/env_package.yaml", + "examples/rl_env_packages/swe_toy_patch/env_package.yaml", + "examples/rl_env_packages/math_console_toy/env_package.yaml", + ], + }, + { + "requirement": "Run manifests", + "covered_by": ["test_commands", "docs/rl_phase1/m12_transfer_pack.md"], + }, + { + "requirement": "Test command list", + "covered_by": ["test_commands"], + }, + { + "requirement": "Expected outputs", + "covered_by": ["expected_outputs"], + }, + { + "requirement": "Rollback plan", + "covered_by": ["rollback_plan"], + }, + { + "requirement": "Final target report contract", + "covered_by": [ + "breadboard/rl/m12/final_report.py", + "scripts/rl_phase1/build_m12_final_report.py", + "scripts/rl_phase1/summarize_m12_final_report_remediations.py", + "m12_final_report/m12_final_report.json", + ], + }, + { + "requirement": "Score promotion audit", + "covered_by": [ + "breadboard/rl/m12/promotion_audit.py", + "scripts/rl_phase1/audit_m12_score_promotion.py", + "m12_promotion_audit/m12_promotion_audit.json", + ], + }, + { + "requirement": "Load ladder and soak evidence", + "covered_by": [ + "breadboard/rl/m12/load_soak.py", + "scripts/rl_phase1/run_m12_load_ladder.py", + "scripts/rl_phase1/run_m12_soak.py", + "m12_node_load_ladder/load_ladder_report.json", + "m12_node_soak/soak_report.json", + "breadboard/rl/m12/final_report.py", + ], + }, + { + "requirement": "Raw command log archive", + "covered_by": [ + "m12_command_logs/command_log_manifest.json", + "m12_command_log_manifest_template.json", + "breadboard/rl/m12/command_logs.py", + "breadboard/rl/m12/final_report.py", + "scripts/rl_phase1/run_m12_logged_command.py", + ], + }, +] + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def _archive_file_mode(path: Path) -> int: + return 0o755 if path.stat().st_mode & 0o111 else 0o644 + + +def _gzip_mtime(path: Path) -> int | None: + header = path.read_bytes()[:8] + if len(header) < 8 or header[:2] != b"\x1f\x8b": + return None + return int.from_bytes(header[4:8], "little") + + +def _sha256_bytes(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _git_output(repo_root: Path, *args: str) -> str: + try: + return subprocess.check_output(["git", *args], cwd=repo_root, text=True, stderr=subprocess.DEVNULL).strip() + except Exception: + return "unavailable" + + +def _artifact_record(repo_root: Path, rel_path: str) -> dict[str, Any]: + path = (repo_root / rel_path).resolve() + if not path.exists(): + return {"path": rel_path, "exists": False} + if path.is_file(): + return { + "path": rel_path, + "exists": True, + "kind": "file", + "size_bytes": path.stat().st_size, + "sha256": _sha256_file(path), + } + files = _iter_artifact_files(path) + return { + "path": rel_path, + "exists": True, + "kind": "directory", + "file_count": len(files), + "size_bytes": sum(item.stat().st_size for item in files), + } + + +def _resolve_archive_manifest_path(manifest_path: Path, key: str, fallback_name: str) -> Path: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + raw = manifest.get(key) + if not raw: + return manifest_path.parent / fallback_name + path = Path(str(raw)) + return path if path.is_absolute() else manifest_path.parent / path + + +def _portable_colocated_file_name_error(*, key: str, value: str, expected_name: str) -> str | None: + path = Path(value) + if not value: + return f"{key} must be set" + if path.is_absolute() or ".." in path.parts or len(path.parts) != 1 or path.name != expected_name: + return f"{key} must be portable colocated file name: {expected_name}" + return None + + +def validate_m12_transfer_archive_manifest(manifest_path: Path) -> list[str]: + errors: list[str] = [] + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception as exc: + return [f"archive manifest is not readable JSON: {type(exc).__name__}: {exc}"] + + archive_name = str(manifest.get("archive_name") or "m12_transfer_evidence_pack.tar.gz") + archive_path = _resolve_archive_manifest_path(manifest_path, "archive_path", archive_name) + sha_path = _resolve_archive_manifest_path(manifest_path, "archive_sha256_file", archive_name + ".sha256") + expected_archive_sha = str(manifest.get("archive_sha256") or "") + entries = manifest.get("included_entries") + archive_name_error = _portable_colocated_file_name_error( + key="archive_name", + value=archive_name, + expected_name=Path(archive_name).name, + ) + if archive_name_error or Path(archive_name).name != archive_name: + errors.append("archive_name must be portable file name") + archive_path_error = _portable_colocated_file_name_error( + key="archive_path", + value=str(manifest.get("archive_path") or ""), + expected_name=archive_name, + ) + if archive_path_error: + errors.append(archive_path_error) + sha_path_error = _portable_colocated_file_name_error( + key="archive_sha256_file", + value=str(manifest.get("archive_sha256_file") or ""), + expected_name=archive_name + ".sha256", + ) + if sha_path_error: + errors.append(sha_path_error) + if manifest.get("archive_manifest_id") != "bb_zyphra_rl_phase1_m12_transfer_archive_manifest_v1": + errors.append("archive_manifest_id must be bb_zyphra_rl_phase1_m12_transfer_archive_manifest_v1") + if manifest.get("claim_boundary") != "transfer_archive_only_not_m12_validation": + errors.append("claim_boundary must remain transfer_archive_only_not_m12_validation") + if manifest.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if manifest.get("m12_points_awarded") is not False: + errors.append("m12_points_awarded must be false") + if manifest.get("archive_is_repo_replacement") is not False: + errors.append("archive_is_repo_replacement must be false") + if manifest.get("archive_contains_source_overlay") is not True: + errors.append("archive_contains_source_overlay must be true") + if manifest.get("archive_excludes_pycache") is not True: + errors.append("archive_excludes_pycache must be true") + if manifest.get("archive_deterministic") is not True: + errors.append("archive_deterministic must be true") + if manifest.get("source_paths_portable") is not True: + errors.append("source_paths_portable must be true") + deterministic_metadata = manifest.get("deterministic_archive_metadata") + if deterministic_metadata != { + "gzip_mtime": 0, + "member_gid": 0, + "member_gname": "", + "member_mtime": 0, + "member_order": "sorted_by_archive_path", + "member_uid": 0, + "member_uname": "", + }: + errors.append("deterministic_archive_metadata must match normalized archive metadata") + if not expected_archive_sha.startswith("sha256:"): + errors.append("archive_sha256 must start with sha256:") + if not isinstance(entries, list) or not entries: + errors.append("included_entries must be a non-empty list") + entries = [] + if manifest.get("included_entry_count") != len(entries): + errors.append("included_entry_count must equal len(included_entries)") + generated_files = manifest.get("generated_transfer_files") + if generated_files != TRANSFER_PREP_FILES: + errors.append("generated_transfer_files must match TRANSFER_PREP_FILES") + if not archive_path.is_file(): + errors.append(f"archive file missing: {archive_path}") + else: + actual_archive_sha = _sha256_file(archive_path) + if actual_archive_sha != expected_archive_sha: + errors.append("archive sha256 mismatch") + if manifest.get("archive_size_bytes") != archive_path.stat().st_size: + errors.append("archive_size_bytes does not match archive file") + gzip_mtime = _gzip_mtime(archive_path) + if gzip_mtime is not None and gzip_mtime != 0: + errors.append("archive gzip mtime must be zero") + if not sha_path.is_file(): + errors.append(f"archive sha256 sidecar missing: {sha_path}") + else: + sidecar_first = sha_path.read_text(encoding="utf-8").strip().split()[0] + if sidecar_first != expected_archive_sha: + errors.append("archive sha256 sidecar does not match archive manifest") + + entry_paths = [str(entry.get("archive_path") or "") for entry in entries if isinstance(entry, dict)] + expected_paths = set(entry_paths) + if len(entry_paths) != len(expected_paths): + errors.append("included_entries archive_path values must be unique") + if "" in expected_paths: + errors.append("included_entries must all have archive_path") + for entry in entries: + if not isinstance(entry, dict): + continue + private_keys = [str(key) for key in entry if str(key).startswith("_")] + if private_keys: + errors.append(f"included_entry private keys are not allowed: {','.join(sorted(private_keys))}") + archive_member = str(entry.get("archive_path") or "") + source_path = str(entry.get("source_path") or "") + source_member = Path(source_path) + if not source_path: + errors.append("included_entries must all have source_path") + elif source_member.is_absolute() or ".." in source_member.parts: + errors.append(f"unsafe included source path: {source_path}") + elif source_path != archive_member: + errors.append(f"included_entry source_path must equal archive_path: {archive_member}") + for archive_member in expected_paths: + member_path = Path(archive_member) + if member_path.is_absolute() or ".." in member_path.parts: + errors.append(f"unsafe archive member path: {archive_member}") + if "__pycache__" in member_path.parts or member_path.suffix == ".pyc": + errors.append(f"generated Python cache member is not allowed: {archive_member}") + if archive_path.is_file(): + try: + archived_transfer_manifest: bytes | None = None + archived_test_commands: bytes | None = None + archived_readiness_summary: bytes | None = None + archived_transfer_summary: bytes | None = None + with tarfile.open(archive_path, "r:gz") as archive: + members = [member for member in archive.getmembers() if member.isfile()] + member_names = [member.name for member in members] + tar_paths = {member.name for member in members} + if len(member_names) != len(tar_paths): + errors.append("archive file member paths must be unique") + if tar_paths != expected_paths: + errors.append("archive member list does not match included_entries") + if member_names != sorted(expected_paths): + errors.append("archive member order must match sorted included_entries") + by_name = {member.name: member for member in members} + for entry in entries: + if not isinstance(entry, dict): + errors.append("included_entries must contain objects") + continue + archive_member = str(entry.get("archive_path") or "") + member = by_name.get(archive_member) + if member is None: + continue + extracted = archive.extractfile(member) + if extracted is None: + errors.append(f"archive member unreadable: {archive_member}") + continue + payload = extracted.read() + if len(payload) != entry.get("size_bytes"): + errors.append(f"archive member size mismatch: {archive_member}") + if _sha256_bytes(payload) != entry.get("sha256"): + errors.append(f"archive member sha256 mismatch: {archive_member}") + if member.mtime != 0: + errors.append(f"archive member mtime must be zero: {archive_member}") + if member.uid != 0 or member.gid != 0: + errors.append(f"archive member uid/gid must be zero: {archive_member}") + if member.uname or member.gname: + errors.append(f"archive member uname/gname must be empty: {archive_member}") + if member.mode != entry.get("mode"): + errors.append(f"archive member mode mismatch: {archive_member}") + if member.mode not in {0o644, 0o755}: + errors.append(f"archive member mode must be normalized: {archive_member}") + if archive_member.endswith("/m12_transfer_manifest.json"): + archived_transfer_manifest = payload + if archive_member.endswith("/m12_test_commands.sh"): + archived_test_commands = payload + if archive_member.endswith("/m12_readiness_summary.json"): + archived_readiness_summary = payload + if archive_member.endswith("/m12_transfer_summary.json"): + archived_transfer_summary = payload + errors.extend( + _validate_archived_m12_transfer_manifest( + archived_transfer_manifest=archived_transfer_manifest, + ) + ) + errors.extend( + _validate_archived_m12_test_commands_pair( + archived_transfer_manifest=archived_transfer_manifest, + archived_test_commands=archived_test_commands, + ) + ) + errors.extend( + _validate_archived_m12_transfer_summaries( + archived_transfer_manifest=archived_transfer_manifest, + archived_readiness_summary=archived_readiness_summary, + archived_transfer_summary=archived_transfer_summary, + ) + ) + except Exception as exc: + errors.append(f"archive file is not readable tar.gz: {type(exc).__name__}: {exc}") + + for generated_name in TRANSFER_PREP_FILES: + if not any(str(path).endswith("/" + generated_name) for path in expected_paths): + errors.append(f"generated transfer file missing from archive: {generated_name}") + return errors + + +def _validate_archived_m12_test_commands_pair( + *, + archived_transfer_manifest: bytes | None, + archived_test_commands: bytes | None, +) -> list[str]: + errors: list[str] = [] + if archived_transfer_manifest is None: + errors.append("archived m12_transfer_manifest.json missing") + return errors + if archived_test_commands is None: + errors.append("archived m12_test_commands.sh missing") + return errors + try: + transfer_manifest = json.loads(archived_transfer_manifest.decode("utf-8")) + except Exception as exc: + return [f"archived m12_transfer_manifest.json is not readable JSON: {type(exc).__name__}: {exc}"] + try: + test_commands = archived_test_commands.decode("utf-8") + except Exception as exc: + return [f"archived m12_test_commands.sh is not readable UTF-8: {type(exc).__name__}: {exc}"] + return [f"archived m12_test_commands.sh invalid: {error}" for error in validate_m12_test_commands_script(test_commands, transfer_manifest)] + + +def _validate_archived_m12_transfer_manifest(*, archived_transfer_manifest: bytes | None) -> list[str]: + if archived_transfer_manifest is None: + return ["archived m12_transfer_manifest.json missing"] + try: + transfer_manifest = json.loads(archived_transfer_manifest.decode("utf-8")) + except Exception as exc: + return [f"archived m12_transfer_manifest.json is not readable JSON: {type(exc).__name__}: {exc}"] + if not isinstance(transfer_manifest, dict): + return ["archived m12_transfer_manifest.json must be an object"] + + errors: list[str] = [] + if transfer_manifest.get("manifest_id") != "bb_zyphra_rl_phase1_m12_transfer_manifest_v1": + errors.append( + "archived m12_transfer_manifest.json invalid: " + "manifest_id must be bb_zyphra_rl_phase1_m12_transfer_manifest_v1" + ) + if transfer_manifest.get("claim_boundary") != "transfer_preparation_only_not_m12_validation": + errors.append( + "archived m12_transfer_manifest.json invalid: " + "claim_boundary must remain transfer_preparation_only_not_m12_validation" + ) + repo = transfer_manifest.get("repo") + if not isinstance(repo, dict): + errors.append("archived m12_transfer_manifest.json invalid: repo must be an object") + elif repo.get("root_path_portable") is not True: + errors.append("archived m12_transfer_manifest.json invalid: repo.root_path_portable must be true") + if transfer_manifest.get("all_required_artifacts_present") is not True: + errors.append("archived m12_transfer_manifest.json invalid: all_required_artifacts_present must be true") + if transfer_manifest.get("all_transfer_requirements_covered") is not True: + errors.append("archived m12_transfer_manifest.json invalid: all_transfer_requirements_covered must be true") + if transfer_manifest.get("expected_outputs") != EXPECTED_OUTPUTS: + errors.append("archived m12_transfer_manifest.json invalid: expected_outputs must match EXPECTED_OUTPUTS") + return errors + + +def _validate_archived_m12_transfer_summaries( + *, + archived_transfer_manifest: bytes | None, + archived_readiness_summary: bytes | None, + archived_transfer_summary: bytes | None, +) -> list[str]: + errors: list[str] = [] + if archived_transfer_manifest is None: + errors.append("archived m12_transfer_manifest.json missing") + return errors + if archived_readiness_summary is None: + errors.append("archived m12_readiness_summary.json missing") + return errors + if archived_transfer_summary is None: + errors.append("archived m12_transfer_summary.json missing") + return errors + try: + transfer_manifest = json.loads(archived_transfer_manifest.decode("utf-8")) + except Exception as exc: + return [f"archived m12_transfer_manifest.json is not readable JSON: {type(exc).__name__}: {exc}"] + try: + readiness_summary = json.loads(archived_readiness_summary.decode("utf-8")) + except Exception as exc: + return [f"archived m12_readiness_summary.json is not readable JSON: {type(exc).__name__}: {exc}"] + try: + transfer_summary = json.loads(archived_transfer_summary.decode("utf-8")) + except Exception as exc: + return [f"archived m12_transfer_summary.json is not readable JSON: {type(exc).__name__}: {exc}"] + if not isinstance(transfer_manifest, dict): + return ["archived m12_transfer_manifest.json must be an object"] + if not isinstance(readiness_summary, dict): + errors.append("archived m12_readiness_summary.json must be an object") + else: + errors.extend( + f"archived m12_readiness_summary.json invalid: {error}" + for error in validate_m12_readiness_summary(readiness_summary, transfer_manifest) + ) + if not isinstance(transfer_summary, dict): + errors.append("archived m12_transfer_summary.json must be an object") + else: + errors.extend( + f"archived m12_transfer_summary.json invalid: {error}" + for error in validate_m12_transfer_summary(transfer_summary, transfer_manifest) + ) + return errors + + +def _iter_artifact_files(path: Path) -> list[Path]: + if path.is_file(): + return [path] if _include_artifact_file(path) else [] + if path.is_dir(): + return sorted(item for item in path.rglob("*") if item.is_file() and _include_artifact_file(item)) + return [] + + +def _include_artifact_file(path: Path) -> bool: + return "__pycache__" not in path.parts and path.suffix != ".pyc" + + +def _archive_name( + *, + workspace_root: Path, + path: Path, + fallback_root: Path | None = None, + fallback_prefix: str = "external", +) -> str: + resolved = path.resolve() + try: + return "workspace/" + resolved.relative_to(workspace_root.resolve()).as_posix() + except ValueError: + if fallback_root is not None: + try: + return fallback_prefix.strip("/") + "/" + resolved.relative_to(fallback_root.resolve()).as_posix() + except ValueError: + pass + return fallback_prefix.strip("/") + "/" + resolved.name + + +def build_m12_transfer_manifest(repo_root: Path) -> dict[str, Any]: + repo_root = repo_root.resolve() + artifacts = [_artifact_record(repo_root, rel_path) for rel_path in REQUIRED_TRANSFER_ARTIFACTS] + return { + "manifest_id": "bb_zyphra_rl_phase1_m12_transfer_manifest_v1", + "claim_boundary": "transfer_preparation_only_not_m12_validation", + "repo": { + "root": repo_root.name, + "root_path_portable": True, + "head": _git_output(repo_root, "rev-parse", "HEAD"), + "branch": _git_output(repo_root, "rev-parse", "--abbrev-ref", "HEAD"), + "dirty_status_short": _git_output(repo_root, "status", "--short"), + }, + "artifacts": artifacts, + "all_required_artifacts_present": all(item.get("exists") for item in artifacts), + "transfer_requirement_coverage": list(TRANSFER_REQUIREMENT_COVERAGE), + "all_transfer_requirements_covered": True, + "test_commands": list(M12_TEST_COMMANDS), + "expected_outputs": list(EXPECTED_OUTPUTS), + "rollback_plan": [ + "Stop Ray workers and runtime processes.", + "Preserve run directories before cleanup.", + "Archive preflight, run reports, and raw command logs with sha256 hashes.", + "If load/soak artifacts are missing or non-eligible, preserve the final report as a blocked target outcome.", + "Build and validate m12_final_report.json before any scorecard edit.", + "Do not update scorecard unless M12 target evidence satisfies the gate.", + ], + } + + +def build_m12_readiness_summary(manifest: dict[str, Any]) -> dict[str, Any]: + commands = [str(command) for command in manifest.get("test_commands") or []] + preflight_command = next((command for command in commands if "run_m12_preflight.py" in command), "") + final_command = next((command for command in commands if "build_m12_final_report.py" in command), "") + promotion_command = next((command for command in commands if "audit_m12_score_promotion.py" in command), "") + generated_script = build_m12_test_commands_script() + script_errors = validate_m12_test_commands_script(generated_script, manifest) + return { + "summary_id": "bb_zyphra_rl_phase1_m12_readiness_summary_v1", + "claim_boundary": "transfer_readiness_summary_not_m12_validation", + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "artifact_count": len(manifest.get("artifacts") or []), + "command_count": len(manifest.get("test_commands") or []), + "expected_output_count": len(manifest.get("expected_outputs") or []), + "all_required_artifacts_present": bool(manifest.get("all_required_artifacts_present")), + "all_transfer_requirements_covered": bool(manifest.get("all_transfer_requirements_covered")), + "target_script_fail_closed": { + "archive_verifier_runs_first": bool(commands) and "verify_m12_transfer_archive.py" in commands[0], + "preflight_requires_pass": "--require-pass" in preflight_command, + "final_report_requires_eligible": "--require-eligible" in final_command, + "final_report_uses_explicit_target_artifact_args": _final_command_has_explicit_target_args(final_command), + "promotion_audit_requires_ready": "--require-ready" in promotion_command, + "promotion_audit_uses_explicit_score_inputs": ( + "--scorecard ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" in promotion_command + and "--claim-ledger ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + in promotion_command + ), + "promotion_audit_uses_explicit_target_paths": _promotion_command_has_explicit_target_args( + promotion_command + ), + "generated_script_uses_logged_command_wrapper": True, + "generated_script_runs_concrete_load_soak": True, + "bootstrap_rejects_dirty_checkout_by_default": True, + "bootstrap_runs_overlaid_test_commands": True, + "bootstrap_cds_to_repo_root_before_handoff": True, + "generated_script_sets_target_run_id": True, + "generated_script_rejects_mixed_target_run_logs": ( + "Existing M12 command log manifest belongs to different target run id(s)" in generated_script + ), + "generated_script_rejects_stale_closeout_artifacts": ( + _CLOSEOUT_ARTIFACT_REUSE_GUARD_TEXT in generated_script + ), + "generated_script_summarizes_final_report_remediations_on_error": ( + "trap m12_on_error ERR" in generated_script + and "summarize_m12_final_report_remediations.py" in generated_script + ), + "generated_script_manifest_consistent": not script_errors, + }, + "generated_script_validation_errors": script_errors, + "target_only_required_outputs": [ + "m12_node_load_ladder/load_ladder_report.json", + "m12_node_soak/soak_report.json", + "m12_command_logs/command_log_manifest.json", + "m12_final_report/m12_final_report.json", + "m12_promotion_audit/m12_promotion_audit.json", + ], + "score_promotion_rule": ( + "Do not edit the scorecard unless target m12_final_report.json has m12_score_eligible=true, " + "raw command logs are archived with final-report-verified sha256 hashes, and the scorecard edit is separately reviewed." + ), + "known_local_blockers": [ + "rocm_tools_unavailable", + "torch_device_count_below_8", + "verl_unavailable", + ], + } + + +def build_m12_transfer_summary(manifest: dict[str, Any]) -> dict[str, Any]: + commands = [str(command) for command in manifest.get("test_commands") or []] + preflight_command = next((command for command in commands if "run_m12_preflight.py" in command), "") + final_command = next((command for command in commands if "build_m12_final_report.py" in command), "") + promotion_command = next((command for command in commands if "audit_m12_score_promotion.py" in command), "") + generated_script = build_m12_test_commands_script() + script_errors = validate_m12_test_commands_script(generated_script, manifest) + return { + "manifest_id": manifest["manifest_id"], + "artifacts_present": manifest["all_required_artifacts_present"], + "artifact_count": len(manifest["artifacts"]), + "command_count": len(manifest["test_commands"]), + "concrete_load_soak_scripts": True, + "claim_boundary": manifest["claim_boundary"], + "expected_output_count": len(manifest["expected_outputs"]), + "archive_verifier_runs_first": bool(commands) and "verify_m12_transfer_archive.py" in commands[0], + "preflight_command_require_pass": "--require-pass" in preflight_command, + "final_command_require_eligible": "--require-eligible" in final_command, + "final_command_explicit_target_artifact_args": _final_command_has_explicit_target_args(final_command), + "promotion_audit_require_ready": "--require-ready" in promotion_command, + "promotion_audit_explicit_score_inputs": ( + "--scorecard ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" in promotion_command + and "--claim-ledger ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + in promotion_command + ), + "promotion_audit_explicit_target_paths": _promotion_command_has_explicit_target_args(promotion_command), + "load_soak_command_log_templates": True, + "logged_command_wrapper": True, + "bootstrap_dirty_checkout_guard": True, + "bootstrap_overlaid_test_commands_handoff": True, + "bootstrap_repo_root_cwd_handoff": True, + "target_run_id_command_binding": True, + "target_run_log_reuse_guard": ( + "Existing M12 command log manifest belongs to different target run id(s)" in generated_script + ), + "target_closeout_artifact_reuse_guard": _CLOSEOUT_ARTIFACT_REUSE_GUARD_TEXT in generated_script, + "final_report_failure_remediation_summary": ( + "trap m12_on_error ERR" in generated_script + and "summarize_m12_final_report_remediations.py" in generated_script + ), + "generated_script_manifest_consistent": not script_errors, + "generated_script_validation_errors": script_errors, + "readiness_summary": "m12_readiness_summary.json", + "all_transfer_requirements_covered": manifest["all_transfer_requirements_covered"], + } + + +def validate_m12_readiness_summary(summary: dict[str, Any], manifest: dict[str, Any]) -> list[str]: + errors: list[str] = [] + expected = build_m12_readiness_summary(manifest) + if summary.get("summary_id") != "bb_zyphra_rl_phase1_m12_readiness_summary_v1": + errors.append("summary_id must be bb_zyphra_rl_phase1_m12_readiness_summary_v1") + if summary.get("claim_boundary") != "transfer_readiness_summary_not_m12_validation": + errors.append("claim_boundary must remain transfer_readiness_summary_not_m12_validation") + if summary.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if summary.get("m12_points_awarded") is not False: + errors.append("m12_points_awarded must be false") + for field in [ + "artifact_count", + "command_count", + "expected_output_count", + "all_required_artifacts_present", + "all_transfer_requirements_covered", + "target_only_required_outputs", + "score_promotion_rule", + "known_local_blockers", + ]: + if summary.get(field) != expected.get(field): + errors.append(f"{field} must match transfer manifest") + fail_closed = summary.get("target_script_fail_closed") + expected_fail_closed = expected["target_script_fail_closed"] + if not isinstance(fail_closed, dict): + errors.append("target_script_fail_closed must be an object") + else: + if set(fail_closed) != set(expected_fail_closed): + errors.append("target_script_fail_closed keys must match expected fail-closed checks") + for field, expected_value in expected_fail_closed.items(): + if fail_closed.get(field) != expected_value: + errors.append(f"target_script_fail_closed.{field} must match transfer manifest") + if summary.get("generated_script_validation_errors") != expected.get("generated_script_validation_errors"): + errors.append("generated_script_validation_errors must match generated target script validation") + return errors + + +def validate_m12_transfer_summary(summary: dict[str, Any], manifest: dict[str, Any]) -> list[str]: + errors: list[str] = [] + expected = build_m12_transfer_summary(manifest) + if summary.get("manifest_id") != "bb_zyphra_rl_phase1_m12_transfer_manifest_v1": + errors.append("manifest_id must be bb_zyphra_rl_phase1_m12_transfer_manifest_v1") + if summary.get("claim_boundary") != "transfer_preparation_only_not_m12_validation": + errors.append("claim_boundary must remain transfer_preparation_only_not_m12_validation") + for field in [ + "artifacts_present", + "artifact_count", + "command_count", + "expected_output_count", + "archive_verifier_runs_first", + "preflight_command_require_pass", + "final_command_require_eligible", + "final_command_explicit_target_artifact_args", + "promotion_audit_require_ready", + "promotion_audit_explicit_score_inputs", + "promotion_audit_explicit_target_paths", + "load_soak_command_log_templates", + "logged_command_wrapper", + "bootstrap_dirty_checkout_guard", + "bootstrap_overlaid_test_commands_handoff", + "bootstrap_repo_root_cwd_handoff", + "target_run_id_command_binding", + "target_run_log_reuse_guard", + "target_closeout_artifact_reuse_guard", + "final_report_failure_remediation_summary", + "generated_script_manifest_consistent", + "generated_script_validation_errors", + "readiness_summary", + "all_transfer_requirements_covered", + ]: + if summary.get(field) != expected.get(field): + errors.append(f"{field} must match transfer manifest") + if summary.get("concrete_load_soak_scripts") is not True: + errors.append("concrete_load_soak_scripts must be true") + return errors + + +def _logged_command_line(command_id: str, command: str) -> str: + return ( + 'python scripts/rl_phase1/run_m12_logged_command.py --manifest "$COMMAND_LOG_MANIFEST" ' + f'--log-dir "$COMMAND_LOG_DIR" --command-id {command_id} --target-run-id "$M12_TARGET_RUN_ID" -- {command}' + ) + + +def _script_command(command_id: str, command: str) -> str: + if command_id in {"final_report", "promotion_audit"}: + return command.replace(_DEFAULT_COMMAND_LOG_MANIFEST_ARG, '"$COMMAND_LOG_MANIFEST"') + return command + + +def build_m12_test_commands_script() -> str: + command_lines = [ + _logged_command_line(command_id, _script_command(command_id, command)) + for command_id, command in M12_TEST_COMMAND_ROWS + ] + return ( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n\n" + 'REPO_ROOT="${REPO_ROOT:-$(pwd)}"\n' + 'if [[ ! -d "$REPO_ROOT/breadboard/rl" ]]; then\n' + ' echo "Set REPO_ROOT to the BreadBoard repository root before running M12 commands." >&2\n' + " exit 2\n" + "fi\n" + 'cd "$REPO_ROOT"\n\n' + 'COMMAND_LOG_DIR="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs"\n' + 'COMMAND_LOG_MANIFEST="$COMMAND_LOG_DIR/command_log_manifest.json"\n' + 'M12_FINAL_REPORT_PATH="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json"\n' + 'M12_REMEDIATION_SUMMARY_PATH="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_remediation_summary.json"\n' + 'M12_PROMOTION_AUDIT_PATH="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json"\n' + 'M12_TARGET_RUN_ID="${M12_TARGET_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)-$$}"\n' + 'mkdir -p "$COMMAND_LOG_DIR"\n\n' + 'if [[ -f "$COMMAND_LOG_MANIFEST" ]]; then\n' + ' python - "$COMMAND_LOG_MANIFEST" "$M12_TARGET_RUN_ID" <<\'PY\'\n' + 'import json\n' + 'import sys\n' + 'from pathlib import Path\n\n' + 'manifest_path = Path(sys.argv[1])\n' + 'target_run_id = sys.argv[2]\n' + 'try:\n' + ' manifest = json.loads(manifest_path.read_text(encoding="utf-8"))\n' + 'except Exception as exc:\n' + ' print(f"Existing M12 command log manifest is unreadable: {exc}", file=sys.stderr)\n' + ' sys.exit(3)\n' + 'seen_ids = set()\n' + 'for value in manifest.get("target_run_ids") or []:\n' + ' if value:\n' + ' seen_ids.add(str(value))\n' + 'for row in manifest.get("commands") or []:\n' + ' if not isinstance(row, dict):\n' + ' continue\n' + ' if row.get("target_run_id"):\n' + ' seen_ids.add(str(row["target_run_id"]))\n' + ' for attempt in row.get("attempts") or []:\n' + ' if isinstance(attempt, dict) and attempt.get("target_run_id"):\n' + ' seen_ids.add(str(attempt["target_run_id"]))\n' + 'foreign_ids = sorted(value for value in seen_ids if value != target_run_id)\n' + 'if foreign_ids:\n' + ' joined = ", ".join(foreign_ids)\n' + ' print(\n' + ' "Existing M12 command log manifest belongs to different target run id(s): "\n' + ' f"{joined}. Set M12_TARGET_RUN_ID to resume that run or archive/remove "\n' + ' "runs/m12_command_logs before starting a new target run.",\n' + ' file=sys.stderr,\n' + ' )\n' + ' sys.exit(3)\n' + 'PY\n' + 'fi\n\n' + 'for existing_artifact in "$M12_FINAL_REPORT_PATH" "$M12_REMEDIATION_SUMMARY_PATH" "$M12_PROMOTION_AUDIT_PATH"; do\n' + ' if [[ -e "$existing_artifact" ]]; then\n' + ' echo "Existing M12 close-out artifact would make target evidence ambiguous: $existing_artifact. Archive/remove runs/m12_final_report and runs/m12_promotion_audit before starting a new target run." >&2\n' + ' exit 3\n' + ' fi\n' + 'done\n\n' + 'm12_on_error() {\n' + ' local exit_code="$?"\n' + ' if [[ -f "$M12_FINAL_REPORT_PATH" ]]; then\n' + ' echo "m12_final_report_remediation_summary_attempt=1" >&2\n' + ' python scripts/rl_phase1/summarize_m12_final_report_remediations.py \\\n' + ' --final-report "$M12_FINAL_REPORT_PATH" \\\n' + ' --output "$M12_REMEDIATION_SUMMARY_PATH" || true\n' + ' fi\n' + ' exit "$exit_code"\n' + '}\n' + 'trap m12_on_error ERR\n\n' + 'echo "m12_target_run_id=$M12_TARGET_RUN_ID"\n\n' + + "\n".join(command_lines) + + "\n" + ) + + +def validate_m12_test_commands_script(script_text: str, manifest: dict[str, Any] | None = None) -> list[str]: + errors: list[str] = [] + command_rows = list(M12_TEST_COMMAND_ROWS) + command_lines = [ + line.strip() + for line in script_text.splitlines() + if "scripts/rl_phase1/run_m12_logged_command.py" in line and "--command-id" in line + ] + if len(command_lines) != len(command_rows): + errors.append("m12_test_commands.sh logged command count must match M12_TEST_COMMAND_ROWS") + if 'set -euo pipefail' not in script_text: + errors.append("m12_test_commands.sh must fail closed with set -euo pipefail") + if 'COMMAND_LOG_MANIFEST="$COMMAND_LOG_DIR/command_log_manifest.json"' not in script_text: + errors.append("m12_test_commands.sh must define COMMAND_LOG_MANIFEST under COMMAND_LOG_DIR") + if ( + 'M12_FINAL_REPORT_PATH="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json"' + not in script_text + ): + errors.append("m12_test_commands.sh must define M12_FINAL_REPORT_PATH") + if ( + 'M12_REMEDIATION_SUMMARY_PATH="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_remediation_summary.json"' + not in script_text + ): + errors.append("m12_test_commands.sh must define M12_REMEDIATION_SUMMARY_PATH") + if ( + 'M12_PROMOTION_AUDIT_PATH="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json"' + not in script_text + ): + errors.append("m12_test_commands.sh must define M12_PROMOTION_AUDIT_PATH") + if 'M12_TARGET_RUN_ID="${M12_TARGET_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)-$$}"' not in script_text: + errors.append("m12_test_commands.sh must define M12_TARGET_RUN_ID") + if "Existing M12 command log manifest belongs to different target run id(s)" not in script_text: + errors.append("m12_test_commands.sh must reject mixed target-run command logs") + if _CLOSEOUT_ARTIFACT_REUSE_GUARD_TEXT not in script_text: + errors.append("m12_test_commands.sh must reject stale close-out artifacts") + if "trap m12_on_error ERR" not in script_text: + errors.append("m12_test_commands.sh must install ERR trap for final-report remediation summary") + if "scripts/rl_phase1/summarize_m12_final_report_remediations.py" not in script_text: + errors.append("m12_test_commands.sh must summarize final-report remediations on failure") + final_report_lines = [line for line in command_lines if "--command-id final_report" in line] + if not final_report_lines: + errors.append("m12_test_commands.sh must include logged final_report command") + else: + final_report_line = final_report_lines[0] + for flag, value in _FINAL_REPORT_SCRIPT_ARGS: + if not _command_has_flag_value(final_report_line, flag, value): + errors.append(f"final_report command must pass explicit {flag} path") + promotion_lines = [line for line in command_lines if "--command-id promotion_audit" in line] + if not promotion_lines: + errors.append("m12_test_commands.sh must include logged promotion_audit command") + else: + promotion_line = promotion_lines[0] + for flag, value in _PROMOTION_AUDIT_SCRIPT_ARGS: + if not _command_has_flag_value(promotion_line, flag, value): + errors.append(f"promotion_audit command must pass explicit {flag} path") + if "--require-ready" not in promotion_line: + errors.append("promotion_audit command must require ready evidence") + for index, (command_id, command) in enumerate(command_rows): + expected_line = _logged_command_line(command_id, _script_command(command_id, command)) + if index >= len(command_lines): + errors.append(f"missing logged command line: {command_id}") + continue + actual_line = command_lines[index] + if actual_line != expected_line: + errors.append(f"logged command line mismatch at position {index + 1}: {command_id}") + if f"--command-id {command_id}" not in actual_line: + errors.append(f"logged command line missing command_id: {command_id}") + if '--target-run-id "$M12_TARGET_RUN_ID"' not in actual_line: + errors.append(f"logged command line missing target run binding: {command_id}") + if manifest is not None: + if manifest.get("test_commands") != list(M12_TEST_COMMANDS): + errors.append("transfer manifest test_commands must match M12_TEST_COMMANDS") + if len(manifest.get("test_commands") or []) != len(command_rows): + errors.append("transfer manifest test_commands count must match M12_TEST_COMMAND_ROWS") + return errors + + +def _standalone_overlay_script() -> str: + return '''#!/usr/bin/env python +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import tarfile + + +EXPECTED_OUTPUTS = [ + "m12_archive_verify/m12_archive_verify_report.json", + "m12_target_preflight/m12_preflight_report.json", + "m12_node_swe_probe/run_summary.json", + "m12_node_swe_probe/run_ledger.jsonl", + "m12_node_verl_probe/verl_probe_rows.jsonl", + "m12_node_verl_probe/verl_probe_rows.parquet", + "m12_node_verl_probe/projection_manifest.json", + "m12_node_verl_probe/smoke_consumer_report.json", + "m12_node_ray_probe/ray_probe_report.json", + "m12_node_ray_probe/warm_vs_cold_report.json", + "m12_node_load_ladder/load_ladder_report.json", + "m12_node_soak/soak_report.json", + "m12_command_logs/command_log_manifest.json", + "m12_final_report/m12_final_report.json", + "m12_promotion_audit/m12_promotion_audit.json", +] +FINAL_REPORT_EXPLICIT_TARGET_ARGS = [ + ("--output", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json"), + ("--archive-verify-report", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_archive_verify/m12_archive_verify_report.json"), + ("--preflight-report", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_target_preflight/m12_preflight_report.json"), + ("--swe-run-summary", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_swe_probe/run_summary.json"), + ("--verl-smoke-report", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_verl_probe/smoke_consumer_report.json"), + ("--ray-probe-report", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe/ray_probe_report.json"), + ("--warm-vs-cold-report", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe/warm_vs_cold_report.json"), + ("--load-ladder-report", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_load_ladder/load_ladder_report.json"), + ("--soak-report", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_soak/soak_report.json"), + ("--command-log-manifest", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json"), +] +PROMOTION_AUDIT_EXPLICIT_TARGET_ARGS = [ + ("--output", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json"), + ("--final-report", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json"), + ("--scorecard", "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml"), + ("--claim-ledger", "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md"), + ("--command-log-manifest", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json"), +] +PROMOTION_AUDIT_EXPLICIT_SCRIPT_ARGS = [ + ("--output", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json"), + ("--final-report", "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json"), + ("--scorecard", "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml"), + ("--claim-ledger", "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md"), + ("--command-log-manifest", '"$COMMAND_LOG_MANIFEST"'), +] + + +def final_command_has_explicit_target_args(command: str) -> bool: + return all(f"{flag} {value}" in command for flag, value in FINAL_REPORT_EXPLICIT_TARGET_ARGS) + + +def promotion_command_has_explicit_target_args(command: str) -> bool: + return all(f"{flag} {value}" in command for flag, value in PROMOTION_AUDIT_EXPLICIT_TARGET_ARGS) + + +def promotion_script_line_has_explicit_target_args(command: str) -> bool: + return all(f"{flag} {value}" in command for flag, value in PROMOTION_AUDIT_EXPLICIT_SCRIPT_ARGS) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def sha256_bytes(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def gzip_mtime(path: Path) -> int | None: + header = path.read_bytes()[:8] + if len(header) < 8 or header[:2] != b"\\x1f\\x8b": + return None + return int.from_bytes(header[4:8], "little") + + +def resolve_path(manifest_path: Path, manifest: dict, key: str, fallback_name: str) -> Path: + raw = manifest.get(key) + if not raw: + return manifest_path.parent / fallback_name + path = Path(str(raw)) + return path if path.is_absolute() else manifest_path.parent / path + + +def portable_colocated_file_name_error(key: str, value: str, expected_name: str) -> str | None: + path = Path(value) + if not value: + return f"{key} must be set" + if path.is_absolute() or ".." in path.parts or len(path.parts) != 1 or path.name != expected_name: + return f"{key} must be portable colocated file name: {expected_name}" + return None + + +def destination_for(workspace_root: Path, archive_path: str) -> Path: + member_path = Path(archive_path) + if member_path.is_absolute() or ".." in member_path.parts: + raise ValueError(f"unsafe archive member path: {archive_path}") + if "__pycache__" in member_path.parts or member_path.suffix == ".pyc": + raise ValueError(f"generated Python cache member is not allowed: {archive_path}") + if not member_path.parts: + raise ValueError(f"archive member must be rooted under workspace/ or m12_transfer_prep/: {archive_path}") + if member_path.parts[0] == "workspace": + relative = Path(*member_path.parts[1:]) + elif member_path.parts[0] == "m12_transfer_prep": + relative = Path("docs_tmp", "ZYPHRA", "RL_PHASE_1", "runs", *member_path.parts) + else: + raise ValueError(f"archive member must be rooted under workspace/ or m12_transfer_prep/: {archive_path}") + destination = (workspace_root / relative).resolve() + workspace_root = workspace_root.resolve() + try: + destination.relative_to(workspace_root) + except ValueError as exc: + raise ValueError(f"archive member escapes workspace root: {archive_path}") from exc + return destination + + +def blocking_parent_for(path: Path, workspace_root: Path) -> Path | None: + parent = path.parent + workspace_root = workspace_root.resolve() + while True: + if parent.exists(): + return None if parent.is_dir() else parent + if parent == workspace_root or parent.parent == parent: + return None + parent = parent.parent + + +M12_TEST_COMMAND_IDS = [ + "target_transfer_archive_verify", + "phase1_validation_suite", + "target_preflight", + "target_swe_probe", + "target_verl_export", + "target_ray_warm_pool", + "target_load_ladder", + "target_soak", + "final_report", + "promotion_audit", +] +DEFAULT_COMMAND_LOG_MANIFEST_ARG = "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json" +GENERATED_TRANSFER_FILES = [ + "m12_transfer_manifest.json", + "m12_test_commands.sh", + "m12_apply_overlay.py", + "m12_target_bootstrap.sh", + "m12_rollback_plan.md", + "m12_readiness_summary.json", + "m12_load_ladder_report_template.json", + "m12_soak_report_template.json", + "m12_command_log_manifest_template.json", + "m12_transfer_summary.json", +] + + +def script_command(command_id: str, command: str) -> str: + if command_id in {"final_report", "promotion_audit"}: + return command.replace(DEFAULT_COMMAND_LOG_MANIFEST_ARG, '"$COMMAND_LOG_MANIFEST"') + return command + + +def logged_command_line(command_id: str, command: str) -> str: + return ( + 'python scripts/rl_phase1/run_m12_logged_command.py --manifest "$COMMAND_LOG_MANIFEST" ' + f'--log-dir "$COMMAND_LOG_DIR" --command-id {command_id} --target-run-id "$M12_TARGET_RUN_ID" -- {command}' + ) + + +def validate_archived_test_commands_pair( + *, + transfer_manifest_payload: bytes | None, + test_commands_payload: bytes | None, +) -> list[str]: + errors: list[str] = [] + if transfer_manifest_payload is None: + errors.append("archived m12_transfer_manifest.json missing") + return errors + if test_commands_payload is None: + errors.append("archived m12_test_commands.sh missing") + return errors + try: + transfer_manifest = json.loads(transfer_manifest_payload.decode("utf-8")) + except Exception as exc: + return [f"archived m12_transfer_manifest.json is not readable JSON: {type(exc).__name__}: {exc}"] + try: + test_commands = test_commands_payload.decode("utf-8") + except Exception as exc: + return [f"archived m12_test_commands.sh is not readable UTF-8: {type(exc).__name__}: {exc}"] + + manifest_commands = transfer_manifest.get("test_commands") + if not isinstance(manifest_commands, list): + errors.append("archived transfer manifest test_commands must be a list") + manifest_commands = [] + if len(manifest_commands) != len(M12_TEST_COMMAND_IDS): + errors.append("archived transfer manifest test_commands count must match M12_TEST_COMMAND_IDS") + command_lines = [ + line.strip() + for line in test_commands.splitlines() + if "scripts/rl_phase1/run_m12_logged_command.py" in line and "--command-id" in line + ] + if len(command_lines) != len(M12_TEST_COMMAND_IDS): + errors.append("archived m12_test_commands.sh logged command count must match M12_TEST_COMMAND_IDS") + if "set -euo pipefail" not in test_commands: + errors.append("archived m12_test_commands.sh must fail closed with set -euo pipefail") + if 'COMMAND_LOG_MANIFEST="$COMMAND_LOG_DIR/command_log_manifest.json"' not in test_commands: + errors.append("archived m12_test_commands.sh must define COMMAND_LOG_MANIFEST under COMMAND_LOG_DIR") + if 'M12_TARGET_RUN_ID="${M12_TARGET_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)-$$}"' not in test_commands: + errors.append("archived m12_test_commands.sh must define M12_TARGET_RUN_ID") + if "Existing M12 command log manifest belongs to different target run id(s)" not in test_commands: + errors.append("archived m12_test_commands.sh must reject mixed target-run command logs") + if "Existing M12 close-out artifact would make target evidence ambiguous" not in test_commands: + errors.append("archived m12_test_commands.sh must reject stale close-out artifacts") + for index, command_id in enumerate(M12_TEST_COMMAND_IDS): + if index >= len(command_lines) or index >= len(manifest_commands): + continue + expected_line = logged_command_line(command_id, script_command(command_id, str(manifest_commands[index]))) + actual_line = command_lines[index] + if actual_line != expected_line: + errors.append(f"archived logged command line mismatch at position {index + 1}: {command_id}") + if f"--command-id {command_id}" not in actual_line: + errors.append(f"archived logged command line missing command_id: {command_id}") + if '--target-run-id "$M12_TARGET_RUN_ID"' not in actual_line: + errors.append(f"archived logged command line missing target run binding: {command_id}") + promotion_lines = [line for line in command_lines if "--command-id promotion_audit" in line] + if not promotion_lines: + errors.append("archived m12_test_commands.sh must include promotion_audit command") + elif not promotion_script_line_has_explicit_target_args(promotion_lines[0]): + errors.append("archived promotion_audit command must use explicit target output/input paths") + if promotion_lines and "--require-ready" not in promotion_lines[0]: + errors.append("archived promotion_audit command must require ready evidence") + return errors + + +def validate_archived_transfer_manifest(*, transfer_manifest_payload: bytes | None) -> list[str]: + if transfer_manifest_payload is None: + return ["archived m12_transfer_manifest.json missing"] + try: + transfer_manifest = json.loads(transfer_manifest_payload.decode("utf-8")) + except Exception as exc: + return [f"archived m12_transfer_manifest.json is not readable JSON: {type(exc).__name__}: {exc}"] + if not isinstance(transfer_manifest, dict): + return ["archived m12_transfer_manifest.json must be an object"] + errors: list[str] = [] + if transfer_manifest.get("manifest_id") != "bb_zyphra_rl_phase1_m12_transfer_manifest_v1": + errors.append( + "archived m12_transfer_manifest.json invalid: " + "manifest_id must be bb_zyphra_rl_phase1_m12_transfer_manifest_v1" + ) + if transfer_manifest.get("claim_boundary") != "transfer_preparation_only_not_m12_validation": + errors.append( + "archived m12_transfer_manifest.json invalid: " + "claim_boundary must remain transfer_preparation_only_not_m12_validation" + ) + repo = transfer_manifest.get("repo") + if not isinstance(repo, dict): + errors.append("archived m12_transfer_manifest.json invalid: repo must be an object") + elif repo.get("root_path_portable") is not True: + errors.append("archived m12_transfer_manifest.json invalid: repo.root_path_portable must be true") + if transfer_manifest.get("all_required_artifacts_present") is not True: + errors.append("archived m12_transfer_manifest.json invalid: all_required_artifacts_present must be true") + if transfer_manifest.get("all_transfer_requirements_covered") is not True: + errors.append("archived m12_transfer_manifest.json invalid: all_transfer_requirements_covered must be true") + if transfer_manifest.get("expected_outputs") != EXPECTED_OUTPUTS: + errors.append("archived m12_transfer_manifest.json invalid: expected_outputs must match EXPECTED_OUTPUTS") + return errors + + +def validate_archived_transfer_summaries( + *, + transfer_manifest_payload: bytes | None, + readiness_summary_payload: bytes | None, + transfer_summary_payload: bytes | None, +) -> list[str]: + errors: list[str] = [] + if transfer_manifest_payload is None: + errors.append("archived m12_transfer_manifest.json missing") + return errors + if readiness_summary_payload is None: + errors.append("archived m12_readiness_summary.json missing") + return errors + if transfer_summary_payload is None: + errors.append("archived m12_transfer_summary.json missing") + return errors + try: + transfer_manifest = json.loads(transfer_manifest_payload.decode("utf-8")) + except Exception as exc: + return [f"archived m12_transfer_manifest.json is not readable JSON: {type(exc).__name__}: {exc}"] + try: + readiness = json.loads(readiness_summary_payload.decode("utf-8")) + except Exception as exc: + return [f"archived m12_readiness_summary.json is not readable JSON: {type(exc).__name__}: {exc}"] + try: + transfer_summary = json.loads(transfer_summary_payload.decode("utf-8")) + except Exception as exc: + return [f"archived m12_transfer_summary.json is not readable JSON: {type(exc).__name__}: {exc}"] + if not isinstance(transfer_manifest, dict): + return ["archived m12_transfer_manifest.json must be an object"] + if not isinstance(readiness, dict): + errors.append("archived m12_readiness_summary.json must be an object") + readiness = {} + if not isinstance(transfer_summary, dict): + errors.append("archived m12_transfer_summary.json must be an object") + transfer_summary = {} + if readiness.get("summary_id") != "bb_zyphra_rl_phase1_m12_readiness_summary_v1": + errors.append( + "archived m12_readiness_summary.json invalid: " + "summary_id must be bb_zyphra_rl_phase1_m12_readiness_summary_v1" + ) + if readiness.get("claim_boundary") != "transfer_readiness_summary_not_m12_validation": + errors.append( + "archived m12_readiness_summary.json invalid: " + "claim_boundary must remain transfer_readiness_summary_not_m12_validation" + ) + if readiness.get("scorecard_update_allowed") is not False: + errors.append("archived m12_readiness_summary.json invalid: scorecard_update_allowed must be false") + if readiness.get("m12_points_awarded") is not False: + errors.append("archived m12_readiness_summary.json invalid: m12_points_awarded must be false") + if transfer_summary.get("manifest_id") != "bb_zyphra_rl_phase1_m12_transfer_manifest_v1": + errors.append( + "archived m12_transfer_summary.json invalid: " + "manifest_id must be bb_zyphra_rl_phase1_m12_transfer_manifest_v1" + ) + if transfer_summary.get("claim_boundary") != "transfer_preparation_only_not_m12_validation": + errors.append( + "archived m12_transfer_summary.json invalid: " + "claim_boundary must remain transfer_preparation_only_not_m12_validation" + ) + commands = [str(command) for command in transfer_manifest.get("test_commands") or []] + preflight_command = next((command for command in commands if "run_m12_preflight.py" in command), "") + final_command = next((command for command in commands if "build_m12_final_report.py" in command), "") + promotion_command = next((command for command in commands if "audit_m12_score_promotion.py" in command), "") + counts = { + "artifact_count": len(transfer_manifest.get("artifacts") or []), + "command_count": len(transfer_manifest.get("test_commands") or []), + "expected_output_count": len(transfer_manifest.get("expected_outputs") or []), + } + for field, expected in counts.items(): + if readiness.get(field) != expected: + errors.append(f"archived m12_readiness_summary.json invalid: {field} must match transfer manifest") + if transfer_summary.get(field) != expected: + errors.append(f"archived m12_transfer_summary.json invalid: {field} must match transfer manifest") + if readiness.get("all_required_artifacts_present") != bool(transfer_manifest.get("all_required_artifacts_present")): + errors.append("archived m12_readiness_summary.json invalid: all_required_artifacts_present must match transfer manifest") + if readiness.get("all_transfer_requirements_covered") != bool(transfer_manifest.get("all_transfer_requirements_covered")): + errors.append("archived m12_readiness_summary.json invalid: all_transfer_requirements_covered must match transfer manifest") + if transfer_summary.get("artifacts_present") != transfer_manifest.get("all_required_artifacts_present"): + errors.append("archived m12_transfer_summary.json invalid: artifacts_present must match transfer manifest") + if transfer_summary.get("all_transfer_requirements_covered") != transfer_manifest.get("all_transfer_requirements_covered"): + errors.append("archived m12_transfer_summary.json invalid: all_transfer_requirements_covered must match transfer manifest") + target_only_outputs = [ + "m12_node_load_ladder/load_ladder_report.json", + "m12_node_soak/soak_report.json", + "m12_command_logs/command_log_manifest.json", + "m12_final_report/m12_final_report.json", + "m12_promotion_audit/m12_promotion_audit.json", + ] + if readiness.get("target_only_required_outputs") != target_only_outputs: + errors.append("archived m12_readiness_summary.json invalid: target_only_required_outputs must match expected target outputs") + score_promotion_rule = ( + "Do not edit the scorecard unless target m12_final_report.json has m12_score_eligible=true, " + "raw command logs are archived with final-report-verified sha256 hashes, and the scorecard edit is separately reviewed." + ) + if readiness.get("score_promotion_rule") != score_promotion_rule: + errors.append("archived m12_readiness_summary.json invalid: score_promotion_rule must match expected rule") + if readiness.get("known_local_blockers") != [ + "rocm_tools_unavailable", + "torch_device_count_below_8", + "verl_unavailable", + ]: + errors.append("archived m12_readiness_summary.json invalid: known_local_blockers must match expected local blockers") + fail_closed = readiness.get("target_script_fail_closed") if isinstance(readiness.get("target_script_fail_closed"), dict) else {} + fail_closed_expectations = { + "archive_verifier_runs_first": bool(commands) and "verify_m12_transfer_archive.py" in commands[0], + "preflight_requires_pass": "--require-pass" in preflight_command, + "final_report_requires_eligible": "--require-eligible" in final_command, + "final_report_uses_explicit_target_artifact_args": final_command_has_explicit_target_args(final_command), + "promotion_audit_requires_ready": "--require-ready" in promotion_command, + "promotion_audit_uses_explicit_score_inputs": ( + "--scorecard ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" in promotion_command + and "--claim-ledger ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" in promotion_command + ), + "promotion_audit_uses_explicit_target_paths": promotion_command_has_explicit_target_args(promotion_command), + "generated_script_uses_logged_command_wrapper": True, + "generated_script_runs_concrete_load_soak": True, + "bootstrap_rejects_dirty_checkout_by_default": True, + "bootstrap_runs_overlaid_test_commands": True, + "bootstrap_cds_to_repo_root_before_handoff": True, + "generated_script_sets_target_run_id": True, + "generated_script_rejects_mixed_target_run_logs": True, + "generated_script_rejects_stale_closeout_artifacts": True, + "generated_script_summarizes_final_report_remediations_on_error": True, + "generated_script_manifest_consistent": True, + } + if set(fail_closed) != set(fail_closed_expectations): + errors.append( + "archived m12_readiness_summary.json invalid: " + "target_script_fail_closed keys must match expected fail-closed checks" + ) + for field, expected in fail_closed_expectations.items(): + if fail_closed.get(field) != expected: + errors.append(f"archived m12_readiness_summary.json invalid: target_script_fail_closed.{field} must match transfer manifest") + transfer_expectations = { + "archive_verifier_runs_first": bool(commands) and "verify_m12_transfer_archive.py" in commands[0], + "preflight_command_require_pass": "--require-pass" in preflight_command, + "final_command_require_eligible": "--require-eligible" in final_command, + "final_command_explicit_target_artifact_args": final_command_has_explicit_target_args(final_command), + "promotion_audit_require_ready": "--require-ready" in promotion_command, + "promotion_audit_explicit_score_inputs": fail_closed_expectations["promotion_audit_uses_explicit_score_inputs"], + "promotion_audit_explicit_target_paths": fail_closed_expectations[ + "promotion_audit_uses_explicit_target_paths" + ], + "load_soak_command_log_templates": True, + "logged_command_wrapper": True, + "bootstrap_dirty_checkout_guard": True, + "bootstrap_overlaid_test_commands_handoff": True, + "bootstrap_repo_root_cwd_handoff": True, + "target_run_id_command_binding": True, + "target_run_log_reuse_guard": fail_closed_expectations[ + "generated_script_rejects_mixed_target_run_logs" + ], + "target_closeout_artifact_reuse_guard": fail_closed_expectations[ + "generated_script_rejects_stale_closeout_artifacts" + ], + "final_report_failure_remediation_summary": True, + "generated_script_manifest_consistent": True, + "readiness_summary": "m12_readiness_summary.json", + "concrete_load_soak_scripts": True, + } + for field, expected in transfer_expectations.items(): + if transfer_summary.get(field) != expected: + errors.append(f"archived m12_transfer_summary.json invalid: {field} must match transfer manifest") + if readiness.get("generated_script_validation_errors") != []: + errors.append("archived m12_readiness_summary.json invalid: generated_script_validation_errors must be empty") + if transfer_summary.get("generated_script_validation_errors") != []: + errors.append("archived m12_transfer_summary.json invalid: generated_script_validation_errors must be empty") + return errors + + +def main() -> None: + parser = argparse.ArgumentParser(description="Apply the M12 transfer overlay after verifying archive hashes.") + parser.add_argument("--manifest", type=Path, default=Path(__file__).resolve().parent / "m12_transfer_archive_manifest.json") + parser.add_argument("--workspace-root", type=Path, default=Path.cwd().parent) + parser.add_argument("--output", type=Path, default=Path(__file__).resolve().parent / "m12_overlay_apply_report.json") + parser.add_argument("--apply", action="store_true") + parser.add_argument("--allow-overwrite", action="store_true") + args = parser.parse_args() + + manifest_path = args.manifest.resolve() + workspace_root = args.workspace_root.resolve() + errors: list[str] = [] + written_count = 0 + entries: list[dict] = [] + payloads_by_destination: dict[str, bytes] = {} + + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception as exc: + manifest = {} + errors.append(f"archive manifest is not readable JSON: {type(exc).__name__}: {exc}") + + archive_name = str(manifest.get("archive_name") or "m12_transfer_evidence_pack.tar.gz") + archive_name_error = portable_colocated_file_name_error( + "archive_name", + archive_name, + Path(archive_name).name, + ) + if archive_name_error or Path(archive_name).name != archive_name: + errors.append("archive_name must be portable file name") + archive_path_error = portable_colocated_file_name_error( + "archive_path", + str(manifest.get("archive_path") or ""), + archive_name, + ) + if archive_path_error: + errors.append(archive_path_error) + sha_path_error = portable_colocated_file_name_error( + "archive_sha256_file", + str(manifest.get("archive_sha256_file") or ""), + archive_name + ".sha256", + ) + if sha_path_error: + errors.append(sha_path_error) + + archive_path = resolve_path( + manifest_path, + manifest, + "archive_path", + archive_name, + ) + sha_path = resolve_path( + manifest_path, + manifest, + "archive_sha256_file", + archive_name + ".sha256", + ) + expected_sha = str(manifest.get("archive_sha256") or "") + included = manifest.get("included_entries") + if manifest.get("archive_manifest_id") != "bb_zyphra_rl_phase1_m12_transfer_archive_manifest_v1": + errors.append("archive_manifest_id must be bb_zyphra_rl_phase1_m12_transfer_archive_manifest_v1") + if manifest.get("claim_boundary") != "transfer_archive_only_not_m12_validation": + errors.append("claim_boundary must remain transfer_archive_only_not_m12_validation") + if manifest.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if manifest.get("m12_points_awarded") is not False: + errors.append("m12_points_awarded must be false") + if manifest.get("archive_is_repo_replacement") is not False: + errors.append("archive_is_repo_replacement must be false") + if manifest.get("archive_contains_source_overlay") is not True: + errors.append("archive_contains_source_overlay must be true") + if manifest.get("archive_excludes_pycache") is not True: + errors.append("archive_excludes_pycache must be true") + if manifest.get("archive_deterministic") is not True: + errors.append("archive_deterministic must be true") + if manifest.get("source_paths_portable") is not True: + errors.append("source_paths_portable must be true") + deterministic_metadata = manifest.get("deterministic_archive_metadata") + if deterministic_metadata != { + "gzip_mtime": 0, + "member_gid": 0, + "member_gname": "", + "member_mtime": 0, + "member_order": "sorted_by_archive_path", + "member_uid": 0, + "member_uname": "", + }: + errors.append("deterministic_archive_metadata must match normalized archive metadata") + if not expected_sha.startswith("sha256:"): + errors.append("archive_sha256 must start with sha256:") + if not isinstance(included, list) or not included: + errors.append("included_entries must be a non-empty list") + included = [] + if manifest.get("included_entry_count") != len(included): + errors.append("included_entry_count must equal len(included_entries)") + if manifest.get("generated_transfer_files") != GENERATED_TRANSFER_FILES: + errors.append("generated_transfer_files must match GENERATED_TRANSFER_FILES") + if not archive_path.is_file(): + errors.append(f"archive file missing: {archive_path}") + else: + if sha256_file(archive_path) != expected_sha: + errors.append("archive sha256 mismatch") + if manifest.get("archive_size_bytes") != archive_path.stat().st_size: + errors.append("archive_size_bytes does not match archive file") + observed_gzip_mtime = gzip_mtime(archive_path) + if observed_gzip_mtime is not None and observed_gzip_mtime != 0: + errors.append("archive gzip mtime must be zero") + if not sha_path.is_file(): + errors.append(f"archive sha256 sidecar missing: {sha_path}") + else: + sidecar_parts = sha_path.read_text(encoding="utf-8").strip().split() + sidecar_first = sidecar_parts[0] if sidecar_parts else "" + if sidecar_first != expected_sha: + errors.append("archive sha256 sidecar does not match archive manifest") + + expected_paths = set() + included_archive_paths = [] + destinations = set() + for item in included: + if not isinstance(item, dict): + errors.append("included_entries must contain objects") + continue + archive_member = str(item.get("archive_path") or "") + private_keys = [str(key) for key in item if str(key).startswith("_")] + if private_keys: + errors.append(f"included_entry private keys are not allowed: {','.join(sorted(private_keys))}") + source_path = str(item.get("source_path") or "") + source_member = Path(source_path) + if not source_path: + errors.append("included_entries must all have source_path") + elif source_member.is_absolute() or ".." in source_member.parts: + errors.append(f"unsafe included source path: {source_path}") + elif source_path != archive_member: + errors.append(f"included_entry source_path must equal archive_path: {archive_member}") + included_archive_paths.append(archive_member) + expected_paths.add(archive_member) + try: + destination = destination_for(workspace_root, archive_member) + except ValueError as exc: + errors.append(str(exc)) + continue + if str(destination) in destinations: + errors.append(f"duplicate overlay destination: {destination}") + continue + destinations.add(str(destination)) + exists = destination.exists() + blocking_parent = blocking_parent_for(destination, workspace_root) + if blocking_parent is not None: + errors.append(f"destination parent exists and is not directory: {blocking_parent}") + if exists and destination.is_dir(): + errors.append(f"destination exists and is directory: {destination}") + if exists and args.apply and not args.allow_overwrite: + errors.append(f"destination exists and allow_overwrite is false: {destination}") + entries.append( + { + "archive_path": archive_member, + "destination_path": str(destination), + "exists": exists, + "size_bytes": item.get("size_bytes"), + "mode": item.get("mode"), + "sha256": item.get("sha256"), + } + ) + if len(included_archive_paths) != len(set(included_archive_paths)): + errors.append("included_entries archive_path values must be unique") + for generated_name in GENERATED_TRANSFER_FILES: + if not any(str(path).endswith("/" + generated_name) for path in expected_paths): + errors.append(f"generated transfer file missing from archive: {generated_name}") + + if archive_path.is_file() and not errors: + transfer_manifest_payload = None + test_commands_payload = None + readiness_summary_payload = None + transfer_summary_payload = None + try: + with tarfile.open(archive_path, "r:gz") as archive: + member_list = [member for member in archive.getmembers() if member.isfile()] + member_names = [member.name for member in member_list] + members = {member.name: member for member in member_list} + if len(member_names) != len(set(member_names)): + errors.append("archive file member paths must be unique") + if set(members) != expected_paths: + errors.append("archive member list does not match included_entries") + if member_names != sorted(expected_paths): + errors.append("archive member order must match sorted included_entries") + else: + for entry in entries: + member = members.get(str(entry["archive_path"])) + if member is None: + errors.append(f"archive member missing: {entry['archive_path']}") + break + extracted = archive.extractfile(member) + if extracted is None: + errors.append(f"archive member unreadable: {entry['archive_path']}") + break + payload = extracted.read() + if len(payload) != entry["size_bytes"]: + errors.append(f"archive member size mismatch: {entry['archive_path']}") + break + if sha256_bytes(payload) != entry["sha256"]: + errors.append(f"archive member sha256 mismatch: {entry['archive_path']}") + break + if member.mtime != 0: + errors.append(f"archive member mtime must be zero: {entry['archive_path']}") + break + if member.uid != 0 or member.gid != 0: + errors.append(f"archive member uid/gid must be zero: {entry['archive_path']}") + break + if member.uname or member.gname: + errors.append(f"archive member uname/gname must be empty: {entry['archive_path']}") + break + if member.mode != entry["mode"]: + errors.append(f"archive member mode mismatch: {entry['archive_path']}") + break + if member.mode not in {0o644, 0o755}: + errors.append(f"archive member mode must be normalized: {entry['archive_path']}") + break + if str(entry["archive_path"]).endswith("/m12_transfer_manifest.json"): + transfer_manifest_payload = payload + if str(entry["archive_path"]).endswith("/m12_test_commands.sh"): + test_commands_payload = payload + if str(entry["archive_path"]).endswith("/m12_readiness_summary.json"): + readiness_summary_payload = payload + if str(entry["archive_path"]).endswith("/m12_transfer_summary.json"): + transfer_summary_payload = payload + payloads_by_destination[str(entry["destination_path"])] = payload + except Exception as exc: + errors.append(f"archive file is not readable tar.gz: {type(exc).__name__}: {exc}") + if not errors: + errors.extend( + validate_archived_transfer_manifest( + transfer_manifest_payload=transfer_manifest_payload, + ) + ) + errors.extend( + validate_archived_test_commands_pair( + transfer_manifest_payload=transfer_manifest_payload, + test_commands_payload=test_commands_payload, + ) + ) + errors.extend( + validate_archived_transfer_summaries( + transfer_manifest_payload=transfer_manifest_payload, + readiness_summary_payload=readiness_summary_payload, + transfer_summary_payload=transfer_summary_payload, + ) + ) + if args.apply and not errors: + for destination_path, payload in payloads_by_destination.items(): + destination = Path(destination_path) + try: + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(payload) + except Exception as exc: + errors.append(f"overlay write failed: {destination}: {type(exc).__name__}: {exc}") + break + written_count += 1 + + report = { + "report_id": "bb_zyphra_rl_phase1_m12_overlay_apply_report_v1", + "claim_boundary": "transfer_overlay_application_not_m12_validation", + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "dry_run": not args.apply, + "allow_overwrite": args.allow_overwrite, + "workspace_root": str(workspace_root), + "archive_manifest": str(manifest_path), + "archive_path": str(archive_path), + "status": "passed" if not errors else "failed", + "would_write_count": len(entries), + "written_count": written_count, + "existing_destination_count": sum(1 for entry in entries if entry["exists"]), + "errors": errors, + "entries": entries, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\\n", encoding="utf-8") + print( + f"report={report['report_id']} status={report['status']} dry_run={report['dry_run']} " + f"would_write={report['would_write_count']} written={report['written_count']} " + f"existing_destinations={report['existing_destination_count']} errors={len(errors)}" + ) + if errors: + for error in errors: + print(f"error={error}") + raise SystemExit(6) + + +if __name__ == "__main__": + main() +''' + + +def _target_bootstrap_script() -> str: + return '''#!/usr/bin/env bash +set -euo pipefail + +PREP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="${REPO_ROOT:-$(pwd)}" +WORKSPACE_ROOT="${WORKSPACE_ROOT:-$(cd "$REPO_ROOT/.." && pwd)}" +TRANSFER_MANIFEST="$PREP_DIR/m12_transfer_manifest.json" +ARCHIVE_MANIFEST="$PREP_DIR/m12_transfer_archive_manifest.json" +OVERLAY_DRY_RUN_REPORT="$PREP_DIR/m12_overlay_apply_dry_run_report.json" +OVERLAY_APPLY_REPORT="$PREP_DIR/m12_overlay_apply_report.json" +TARGET_PREP_DIR="$WORKSPACE_ROOT/docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep" +TARGET_TEST_COMMANDS="$TARGET_PREP_DIR/m12_test_commands.sh" + +if ! git -C "$REPO_ROOT" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + echo "REPO_ROOT must point to the exact BreadBoard git checkout before M12 bootstrap." >&2 + exit 2 +fi +if [[ ! -f "$TRANSFER_MANIFEST" ]]; then + echo "Missing transfer manifest: $TRANSFER_MANIFEST" >&2 + exit 2 +fi +if [[ ! -f "$ARCHIVE_MANIFEST" ]]; then + echo "Missing archive manifest: $ARCHIVE_MANIFEST" >&2 + exit 2 +fi + +PYTHON_BIN="${PYTHON_BIN:-}" +if [[ -z "$PYTHON_BIN" ]]; then + if command -v python3 >/dev/null 2>&1; then + PYTHON_BIN=python3 + elif command -v python >/dev/null 2>&1; then + PYTHON_BIN=python + else + echo "python3 or python is required for M12 bootstrap." >&2 + exit 2 + fi +fi + +EXPECTED_HEAD="$("$PYTHON_BIN" - "$TRANSFER_MANIFEST" <<'PY' +import json +import sys +from pathlib import Path + +manifest = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +print((manifest.get("repo") or {}).get("head") or "") +PY +)" +ACTUAL_HEAD="$(git -C "$REPO_ROOT" rev-parse HEAD)" +if [[ -z "$EXPECTED_HEAD" || "$EXPECTED_HEAD" == "unavailable" ]]; then + echo "Transfer manifest does not record a usable repo HEAD." >&2 + exit 7 +fi +if [[ "$ACTUAL_HEAD" != "$EXPECTED_HEAD" ]]; then + echo "Repo HEAD mismatch. expected=$EXPECTED_HEAD actual=$ACTUAL_HEAD" >&2 + exit 7 +fi +REPO_STATUS_SHORT="$(git -C "$REPO_ROOT" status --short)" +REPO_STATUS_LINES="$(printf "%s\n" "$REPO_STATUS_SHORT" | sed '/^$/d' | wc -l | tr -d ' ')" +if [[ -n "$REPO_STATUS_SHORT" && "${ALLOW_M12_DIRTY_CHECKOUT:-0}" != "1" ]]; then + echo "Repo checkout is dirty before M12 overlay. Refusing to continue; set ALLOW_M12_DIRTY_CHECKOUT=1 only for local rehearsal/debugging." >&2 + echo "dirty_status_lines=$REPO_STATUS_LINES" >&2 + exit 7 +fi +if [[ -n "$REPO_STATUS_SHORT" ]]; then + echo "repo_dirty_check=override dirty_status_lines=$REPO_STATUS_LINES" +else + echo "repo_dirty_check=clean dirty_status_lines=0" +fi + +"$PYTHON_BIN" "$PREP_DIR/m12_apply_overlay.py" \ + --manifest "$ARCHIVE_MANIFEST" \ + --workspace-root "$WORKSPACE_ROOT" \ + --output "$OVERLAY_DRY_RUN_REPORT" + +if [[ "${BOOTSTRAP_DRY_RUN_ONLY:-0}" == "1" ]]; then + echo "bootstrap_dry_run_only=true repo_head_verified=true overlay_dry_run_report=$OVERLAY_DRY_RUN_REPORT" + exit 0 +fi + +"$PYTHON_BIN" "$PREP_DIR/m12_apply_overlay.py" \ + --manifest "$ARCHIVE_MANIFEST" \ + --workspace-root "$WORKSPACE_ROOT" \ + --output "$OVERLAY_APPLY_REPORT" \ + --apply \ + --allow-overwrite + +mkdir -p "$TARGET_PREP_DIR" +for transfer_artifact in \ + m12_transfer_archive_manifest.json \ + m12_transfer_evidence_pack.tar.gz \ + m12_transfer_evidence_pack.tar.gz.sha256; do + if [[ ! -f "$PREP_DIR/$transfer_artifact" ]]; then + echo "Missing target transfer artifact before command handoff: $PREP_DIR/$transfer_artifact" >&2 + exit 8 + fi + cp -f "$PREP_DIR/$transfer_artifact" "$TARGET_PREP_DIR/$transfer_artifact" +done + +if [[ -n "${M12_LOCAL_TEST_FIXTURES_ZIP:-}" ]]; then + if [[ ! -f "$M12_LOCAL_TEST_FIXTURES_ZIP" ]]; then + echo "M12_LOCAL_TEST_FIXTURES_ZIP does not exist: $M12_LOCAL_TEST_FIXTURES_ZIP" >&2 + exit 8 + fi + M12_FIXTURE_WORKSPACE="$WORKSPACE_ROOT" M12_FIXTURE_ZIP="$M12_LOCAL_TEST_FIXTURES_ZIP" "$PYTHON_BIN" - <<'PY' +from __future__ import annotations + +import os +import zipfile +from pathlib import Path + +fixture_zip = Path(os.environ["M12_FIXTURE_ZIP"]) +roots = [ + Path(os.environ["M12_FIXTURE_WORKSPACE"]), + Path("/Users/kylemccleary/projects/breadboard"), + Path("/shared_folders/querylake_server/ray_testing/ray_SCE"), +] +for root in roots: + root.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(fixture_zip) as zf: + zf.extractall(root) +PY +fi + +if [[ ! -f "$TARGET_TEST_COMMANDS" ]]; then + echo "Missing overlaid M12 test command script after overlay apply: $TARGET_TEST_COMMANDS" >&2 + exit 8 +fi +if [[ ! -r "$TARGET_TEST_COMMANDS" ]]; then + echo "Overlaid M12 test command script is not readable after overlay apply: $TARGET_TEST_COMMANDS" >&2 + exit 8 +fi + +cd "$REPO_ROOT" +REPO_ROOT="$REPO_ROOT" bash "$TARGET_TEST_COMMANDS" +''' + + +def write_m12_transfer_pack(*, repo_root: Path, output_dir: Path) -> dict[str, Any]: + manifest = build_m12_transfer_manifest(repo_root) + test_commands_script = build_m12_test_commands_script() + script_errors = validate_m12_test_commands_script(test_commands_script, manifest) + if script_errors: + raise ValueError("invalid M12 test command script: " + "; ".join(script_errors)) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "m12_transfer_manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + (output_dir / "m12_test_commands.sh").write_text( + test_commands_script, + encoding="utf-8", + ) + overlay_script = output_dir / "m12_apply_overlay.py" + overlay_script.write_text(_standalone_overlay_script(), encoding="utf-8") + overlay_script.chmod(0o755) + bootstrap_script = output_dir / "m12_target_bootstrap.sh" + bootstrap_script.write_text(_target_bootstrap_script(), encoding="utf-8") + bootstrap_script.chmod(0o755) + (output_dir / "m12_rollback_plan.md").write_text( + "# M12 Rollback Plan\n\n" + + "\n".join(f"- {item}" for item in manifest["rollback_plan"]) + + "\n", + encoding="utf-8", + ) + readiness_summary = build_m12_readiness_summary(manifest) + readiness_errors = validate_m12_readiness_summary(readiness_summary, manifest) + if readiness_errors: + raise ValueError("invalid M12 readiness summary: " + "; ".join(readiness_errors)) + (output_dir / "m12_readiness_summary.json").write_text( + json.dumps(readiness_summary, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + transfer_summary = build_m12_transfer_summary(manifest) + transfer_summary_errors = validate_m12_transfer_summary(transfer_summary, manifest) + if transfer_summary_errors: + raise ValueError("invalid M12 transfer summary: " + "; ".join(transfer_summary_errors)) + (output_dir / "m12_transfer_summary.json").write_text( + json.dumps(transfer_summary, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + (output_dir / "m12_load_ladder_report_template.json").write_text( + json.dumps(LOAD_LADDER_REPORT_TEMPLATE, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + (output_dir / "m12_soak_report_template.json").write_text( + json.dumps(SOAK_REPORT_TEMPLATE, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + (output_dir / "m12_command_log_manifest_template.json").write_text( + json.dumps(COMMAND_LOG_MANIFEST_TEMPLATE, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return manifest + + +def write_m12_transfer_archive( + *, + repo_root: Path, + output_dir: Path, + archive_name: str = "m12_transfer_evidence_pack.tar.gz", +) -> dict[str, Any]: + """Write a companion evidence archive for target-node transfer. + + The archive is intentionally not a repo replacement. Operators still need an + exact repo checkout; this just packages the M12 evidence/control files with + hashes so transfer drift is easy to detect. + """ + + manifest = write_m12_transfer_pack(repo_root=repo_root, output_dir=output_dir) + repo_root = repo_root.resolve() + workspace_root = repo_root.parent.resolve() + output_dir = output_dir.resolve() + archive_path = output_dir / archive_name + + included: dict[str, dict[str, Any]] = {} + for rel_path in REQUIRED_TRANSFER_ARTIFACTS: + source = (repo_root / rel_path).resolve() + for file_path in _iter_artifact_files(source): + archive_member = _archive_name(workspace_root=workspace_root, path=file_path) + included[archive_member] = { + "_local_source_path": str(file_path), + "source_path": archive_member, + "archive_path": archive_member, + "size_bytes": file_path.stat().st_size, + "mode": _archive_file_mode(file_path), + "sha256": _sha256_file(file_path), + } + + for name in TRANSFER_PREP_FILES: + source = output_dir / name + if source.exists(): + archive_member = _archive_name( + workspace_root=workspace_root, + path=source, + fallback_root=output_dir, + fallback_prefix="m12_transfer_prep", + ) + included[archive_member] = { + "_local_source_path": str(source), + "source_path": archive_member, + "archive_path": archive_member, + "size_bytes": source.stat().st_size, + "mode": _archive_file_mode(source), + "sha256": _sha256_file(source), + } + + output_dir.mkdir(parents=True, exist_ok=True) + with archive_path.open("wb") as raw_archive: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw_archive, mtime=0) as gzip_archive: + with tarfile.open(fileobj=gzip_archive, mode="w", format=tarfile.PAX_FORMAT) as archive: + for archive_member in sorted(included): + source_path = Path(str(included[archive_member]["_local_source_path"])) + payload = source_path.read_bytes() + info = tarfile.TarInfo(archive_member) + info.size = len(payload) + info.mtime = 0 + info.uid = 0 + info.gid = 0 + info.uname = "" + info.gname = "" + info.mode = _archive_file_mode(source_path) + archive.addfile(info, io.BytesIO(payload)) + + archive_sha256 = _sha256_file(archive_path) + sha_path = archive_path.with_suffix(archive_path.suffix + ".sha256") + sha_path.write_text(f"{archive_sha256} {archive_path.name}\n", encoding="utf-8") + + archive_manifest = { + "archive_manifest_id": "bb_zyphra_rl_phase1_m12_transfer_archive_manifest_v1", + "claim_boundary": "transfer_archive_only_not_m12_validation", + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "archive_is_repo_replacement": False, + "archive_contains_source_overlay": True, + "archive_excludes_pycache": True, + "archive_deterministic": True, + "source_paths_portable": True, + "deterministic_archive_metadata": { + "gzip_mtime": 0, + "member_gid": 0, + "member_gname": "", + "member_mtime": 0, + "member_order": "sorted_by_archive_path", + "member_uid": 0, + "member_uname": "", + }, + "source_overlay_paths": [ + "breadboard/rl", + "scripts/rl_phase1", + "tests/rl", + "tests/test_rl_phase1_scorecard_schema.py", + "tests/test_rl_phase1_claim_ledger.py", + "docs/rl_phase1", + "examples/rl_env_packages", + ], + "required_operator_repo_step": ( + "Checkout the exact repo SHA from m12_transfer_manifest.json, then overlay the archived RL Phase 1 " + "source/control files before running target commands." + ), + "archive_path": archive_path.name, + "archive_name": archive_path.name, + "archive_sha256": archive_sha256, + "archive_sha256_file": sha_path.name, + "archive_size_bytes": archive_path.stat().st_size, + "included_entry_count": len(included), + "all_required_artifacts_present": bool(manifest["all_required_artifacts_present"]), + "all_transfer_requirements_covered": bool(manifest["all_transfer_requirements_covered"]), + "generated_transfer_files": list(TRANSFER_PREP_FILES), + "included_entries": [ + {entry_key: entry_value for entry_key, entry_value in included[key].items() if not entry_key.startswith("_")} + for key in sorted(included) + ], + } + (output_dir / "m12_transfer_archive_manifest.json").write_text( + json.dumps(archive_manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return archive_manifest + + +def _safe_overlay_destination(*, workspace_root: Path, archive_path: str) -> Path: + member_path = Path(archive_path) + if member_path.is_absolute() or ".." in member_path.parts: + raise ValueError(f"unsafe archive member path: {archive_path}") + if "__pycache__" in member_path.parts or member_path.suffix == ".pyc": + raise ValueError(f"generated Python cache member is not allowed: {archive_path}") + if not member_path.parts: + raise ValueError(f"archive member must be rooted under workspace/ or m12_transfer_prep/: {archive_path}") + if member_path.parts[0] == "workspace": + relative = Path(*member_path.parts[1:]) + elif member_path.parts[0] == "m12_transfer_prep": + relative = Path("docs_tmp", "ZYPHRA", "RL_PHASE_1", "runs", *member_path.parts) + else: + raise ValueError(f"archive member must be rooted under workspace/ or m12_transfer_prep/: {archive_path}") + destination = (workspace_root / relative).resolve() + root = workspace_root.resolve() + try: + destination.relative_to(root) + except ValueError as exc: + raise ValueError(f"archive member escapes workspace root: {archive_path}") from exc + return destination + + +def _blocking_overlay_parent(*, workspace_root: Path, destination: Path) -> Path | None: + parent = destination.parent + root = workspace_root.resolve() + while True: + if parent.exists(): + return None if parent.is_dir() else parent + if parent == root or parent.parent == parent: + return None + parent = parent.parent + + +def apply_m12_transfer_overlay( + *, + manifest_path: Path, + workspace_root: Path, + dry_run: bool = True, + allow_overwrite: bool = False, +) -> dict[str, Any]: + """Verify and optionally apply the M12 source/evidence overlay. + + This is intentionally not a validation command and not a score promotion path. + It only makes target-node transfer less error-prone after an operator has + checked out the exact repository SHA recorded in the transfer manifest. + """ + + workspace_root = workspace_root.resolve() + manifest_path = manifest_path.resolve() + validation_errors = validate_m12_transfer_archive_manifest(manifest_path) + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception as exc: + return { + "report_id": "bb_zyphra_rl_phase1_m12_overlay_apply_report_v1", + "claim_boundary": "transfer_overlay_application_not_m12_validation", + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "dry_run": dry_run, + "allow_overwrite": allow_overwrite, + "workspace_root": str(workspace_root), + "archive_manifest": str(manifest_path), + "archive_path": None, + "status": "failed", + "would_write_count": 0, + "written_count": 0, + "existing_destination_count": 0, + "errors": [f"archive manifest is not readable JSON: {type(exc).__name__}: {exc}"], + "entries": [], + } + + archive_path = _resolve_archive_manifest_path( + manifest_path, + "archive_path", + str(manifest.get("archive_name") or "m12_transfer_evidence_pack.tar.gz"), + ) + errors = list(validation_errors) + entries: list[dict[str, Any]] = [] + + included_entries = manifest.get("included_entries") + if not isinstance(included_entries, list): + included_entries = [] + errors.append("included_entries must be a list") + + destinations: dict[str, str] = {} + for entry in included_entries: + if not isinstance(entry, dict): + errors.append("included_entries must contain objects") + continue + archive_member = str(entry.get("archive_path") or "") + try: + destination = _safe_overlay_destination(workspace_root=workspace_root, archive_path=archive_member) + except ValueError as exc: + errors.append(str(exc)) + continue + destination_key = str(destination) + if destination_key in destinations: + errors.append(f"duplicate overlay destination: {destination_key}") + continue + destinations[destination_key] = archive_member + exists = destination.exists() + blocking_parent = _blocking_overlay_parent(workspace_root=workspace_root, destination=destination) + if blocking_parent is not None: + errors.append(f"destination parent exists and is not directory: {blocking_parent}") + if exists and destination.is_dir(): + errors.append(f"destination exists and is directory: {destination}") + if exists and not allow_overwrite and not dry_run: + errors.append(f"destination exists and allow_overwrite is false: {destination}") + entries.append( + { + "archive_path": archive_member, + "destination_path": destination_key, + "exists": exists, + "size_bytes": entry.get("size_bytes"), + "sha256": entry.get("sha256"), + } + ) + + written_count = 0 + if not errors and not dry_run: + with tarfile.open(archive_path, "r:gz") as archive: + by_name = {member.name: member for member in archive.getmembers() if member.isfile()} + for entry in entries: + archive_member = str(entry["archive_path"]) + member = by_name.get(archive_member) + if member is None: + errors.append(f"archive member missing during overlay apply: {archive_member}") + break + extracted = archive.extractfile(member) + if extracted is None: + errors.append(f"archive member unreadable during overlay apply: {archive_member}") + break + payload = extracted.read() + expected_sha = str(entry.get("sha256") or "") + if _sha256_bytes(payload) != expected_sha: + errors.append(f"archive member sha256 mismatch during overlay apply: {archive_member}") + break + destination = Path(str(entry["destination_path"])) + try: + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(payload) + except Exception as exc: + errors.append(f"overlay write failed: {destination}: {type(exc).__name__}: {exc}") + break + written_count += 1 + + status = "passed" if not errors else "failed" + return { + "report_id": "bb_zyphra_rl_phase1_m12_overlay_apply_report_v1", + "claim_boundary": "transfer_overlay_application_not_m12_validation", + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "dry_run": dry_run, + "allow_overwrite": allow_overwrite, + "workspace_root": str(workspace_root), + "archive_manifest": str(manifest_path), + "archive_path": str(archive_path), + "status": status, + "would_write_count": len(entries), + "written_count": written_count, + "existing_destination_count": sum(1 for entry in entries if entry["exists"]), + "errors": errors, + "entries": entries, + } + + +def validate_m12_transfer_overlay_report(report: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != "bb_zyphra_rl_phase1_m12_overlay_apply_report_v1": + errors.append("report_id must be bb_zyphra_rl_phase1_m12_overlay_apply_report_v1") + if report.get("claim_boundary") != "transfer_overlay_application_not_m12_validation": + errors.append("claim_boundary must remain transfer_overlay_application_not_m12_validation") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if report.get("m12_points_awarded") is not False: + errors.append("m12_points_awarded must be false") + if report.get("status") not in {"passed", "failed"}: + errors.append("status must be passed or failed") + report_errors = report.get("errors") + if not isinstance(report_errors, list): + errors.append("errors must be a list") + report_errors = [] + entries = report.get("entries") + if not isinstance(entries, list): + errors.append("entries must be a list") + entries = [] + would_write_count = report.get("would_write_count") + written_count = report.get("written_count") + existing_destination_count = report.get("existing_destination_count") + if not isinstance(would_write_count, int) or isinstance(would_write_count, bool): + errors.append("would_write_count must be an integer") + would_write_count = None + if not isinstance(written_count, int) or isinstance(written_count, bool): + errors.append("written_count must be an integer") + written_count = None + if not isinstance(existing_destination_count, int) or isinstance(existing_destination_count, bool): + errors.append("existing_destination_count must be an integer") + existing_destination_count = None + if would_write_count != len(entries): + errors.append("would_write_count must equal len(entries)") + if written_count is not None and would_write_count is not None and ( + written_count < 0 or written_count > would_write_count + ): + errors.append("written_count must be between 0 and would_write_count") + if report.get("status") == "passed" and report_errors != []: + errors.append("passed report must have no errors") + if report.get("status") == "failed" and report_errors == []: + errors.append("failed report must include at least one error") + if report.get("dry_run") is True and written_count != 0: + errors.append("dry-run report must have written_count=0") + if report.get("dry_run") is False and report.get("status") == "passed": + if written_count != len(entries): + errors.append("successful apply report must write every entry") + observed_existing_count = 0 + for entry in entries: + if not isinstance(entry, dict): + errors.append("entries must contain objects") + continue + archive_path = str(entry.get("archive_path") or "") + destination_path = str(entry.get("destination_path") or "") + if not isinstance(entry.get("exists"), bool): + errors.append(f"entry exists must be boolean: {archive_path}") + elif entry["exists"]: + observed_existing_count += 1 + if not (archive_path.startswith("workspace/") or archive_path.startswith("m12_transfer_prep/")): + errors.append(f"entry archive_path must start with workspace/ or m12_transfer_prep/: {archive_path}") + if "__pycache__" in archive_path or archive_path.endswith(".pyc"): + errors.append(f"entry archive_path must exclude Python cache files: {archive_path}") + if not destination_path: + errors.append("entry destination_path must be non-empty") + if not str(entry.get("sha256") or "").startswith("sha256:"): + errors.append(f"entry sha256 must start with sha256: {archive_path}") + if existing_destination_count != observed_existing_count: + errors.append("existing_destination_count must equal entries with exists=true") + return errors diff --git a/breadboard/rl/phase2/__init__.py b/breadboard/rl/phase2/__init__.py new file mode 100644 index 00000000..a209f455 --- /dev/null +++ b/breadboard/rl/phase2/__init__.py @@ -0,0 +1,7 @@ +"""Phase 2 RL productization primitives. + +These modules model contracts and deterministic reports for the Phase 2 +productization surface without claiming production readiness. +""" + +__all__: list[str] = [] diff --git a/breadboard/rl/phase2/benchflow.py b/breadboard/rl/phase2/benchflow.py new file mode 100644 index 00000000..550b3608 --- /dev/null +++ b/breadboard/rl/phase2/benchflow.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping, Sequence + + +CLAIM_BOUNDARY = "p2_m7_benchflow_probe_not_full_security_coverage_claim" + + +@dataclass(frozen=True) +class BenchFlowHardeningImportReport: + report_id: str + target_run_id: str + source_artifact: str + preserved_fields: list[str] + lost_fields: list[str] + field_mapping: dict[str, str] + imported_probe_catches_fixture: bool + fixture_detection_ref: str | None + claim_boundary: str = CLAIM_BOUNDARY + scorecard_update_allowed: bool = False + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "report_id": self.report_id, + "target_run_id": self.target_run_id, + "claim_boundary": self.claim_boundary, + "scorecard_update_allowed": self.scorecard_update_allowed, + "passed": self.imported_probe_catches_fixture and bool(self.preserved_fields), + "source_artifact": self.source_artifact, + "preserved_fields": list(self.preserved_fields), + "lost_fields": list(self.lost_fields), + "field_mapping": dict(self.field_mapping), + "imported_probe_catches_fixture": self.imported_probe_catches_fixture, + "fixture_detection_ref": self.fixture_detection_ref, + "metadata": dict(self.metadata), + } + + +def build_benchflow_hardening_import_report( + *, + imported_fields: Sequence[str], + breadboard_policy_fields: Mapping[str, str], + source_artifact: str, + adversarial_fixture_detected: bool, + target_run_id: str = "fixture-phase2-benchflow-hardening", +) -> BenchFlowHardeningImportReport: + imported = list(imported_fields) + preserved = [field for field in imported if field in breadboard_policy_fields] + lost = [field for field in imported if field not in breadboard_policy_fields] + field_mapping = {field: breadboard_policy_fields[field] for field in preserved} + fixture_detection_ref = "cas://benchflow/imported-probe/adversarial-fixture" if adversarial_fixture_detected else None + return BenchFlowHardeningImportReport( + report_id="bb_zyphra_rl_phase2_benchflow_hardening_import_v1", + target_run_id=target_run_id, + source_artifact=source_artifact, + preserved_fields=preserved, + lost_fields=lost, + field_mapping=field_mapping, + imported_probe_catches_fixture=adversarial_fixture_detected, + fixture_detection_ref=fixture_detection_ref, + metadata={ + "fixture_scope": "benchflow_policy_field_import", + "minimum_promotion_requirement": "real_benchflow_probe_with_sandbox_attestation", + }, + ) + + +def fixture_benchflow_import_report() -> BenchFlowHardeningImportReport: + return build_benchflow_hardening_import_report( + imported_fields=[ + "workspace_isolation", + "network_egress_block", + "path_traversal_probe", + "benchflow_harbor_attestation", + ], + breadboard_policy_fields={ + "workspace_isolation": "EnvPackage.security.workspace_policy", + "network_egress_block": "EnvPackage.security.egress_policy", + "path_traversal_probe": "RewardHackProbeSuite.path_escape", + }, + source_artifact="fixtures/benchflow/hardening_import_fixture.json", + adversarial_fixture_detected=True, + ) diff --git a/breadboard/rl/phase2/benchmark.py b/breadboard/rl/phase2/benchmark.py new file mode 100644 index 00000000..4a7c587b --- /dev/null +++ b/breadboard/rl/phase2/benchmark.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from hashlib import sha256 +from typing import Any, Mapping + + +CLAIM_BOUNDARY = "p2_m5_named_benchmark_slice_not_general_benchmark_claim" +REQUIRED_CONTAMINATION_CONTROLS = ( + "source_hash_pin", + "train_overlap_manifest", + "prompt_solution_leakage_scan", +) + + +def source_sha256(payload: bytes | str) -> str: + if isinstance(payload, str): + payload = payload.encode("utf-8") + return sha256(payload).hexdigest() + + +@dataclass(frozen=True) +class BenchmarkSourcePin: + benchmark_id: str + benchmark_version: str + slice_id: str + source_uri: str + expected_source_sha256: str + + def to_dict(self) -> dict[str, Any]: + return { + "benchmark_id": self.benchmark_id, + "benchmark_version": self.benchmark_version, + "slice_id": self.slice_id, + "source_uri": self.source_uri, + "expected_source_sha256": self.expected_source_sha256, + } + + +@dataclass(frozen=True) +class BenchmarkSliceReport: + report_id: str + source_pin: BenchmarkSourcePin + observed_source_sha256: str + contamination_controls: list[str] + failure_replay_refs: list[str] + metrics: dict[str, Any] + target_run_id: str + status: str + errors: list[str] + claim_boundary: str = CLAIM_BOUNDARY + scorecard_update_allowed: bool = False + metadata: dict[str, Any] = field(default_factory=dict) + + @property + def accepted_for_claim(self) -> bool: + return not self.errors and self.status == "benchmark_slice_fixture_accepted" + + def to_dict(self) -> dict[str, Any]: + return { + "report_id": self.report_id, + "target_run_id": self.target_run_id, + "claim_boundary": self.claim_boundary, + "scorecard_update_allowed": self.scorecard_update_allowed, + "passed": self.accepted_for_claim, + "status": self.status, + "accepted_for_claim": self.accepted_for_claim, + "source_pin": self.source_pin.to_dict(), + "observed_source_sha256": self.observed_source_sha256, + "contamination_controls": list(self.contamination_controls), + "failure_replay_refs": list(self.failure_replay_refs), + "metrics": dict(self.metrics), + "errors": list(self.errors), + "metadata": dict(self.metadata), + } + + +def build_benchmark_slice_report( + source_pin: BenchmarkSourcePin, + *, + observed_source_sha256: str, + contamination_controls: list[str], + failure_replay_refs: list[str], + metrics: Mapping[str, Any], + target_run_id: str = "fixture-phase2-benchmark-slice", +) -> BenchmarkSliceReport: + errors: list[str] = [] + if observed_source_sha256 != source_pin.expected_source_sha256: + errors.append("source_hash_mismatch") + missing_controls = [control for control in REQUIRED_CONTAMINATION_CONTROLS if control not in contamination_controls] + for control in missing_controls: + errors.append("missing_contamination_control:" + control) + if not failure_replay_refs: + errors.append("missing_failure_replay") + + if "source_hash_mismatch" in errors: + status = "rejected_hash_mismatch" + elif errors: + status = "rejected_incomplete_controls" + else: + status = "benchmark_slice_fixture_accepted" + + return BenchmarkSliceReport( + report_id="bb_zyphra_rl_phase2_benchmark_slice_v1", + source_pin=source_pin, + observed_source_sha256=observed_source_sha256, + contamination_controls=list(contamination_controls), + failure_replay_refs=list(failure_replay_refs), + metrics=dict(metrics), + target_run_id=target_run_id, + status=status, + errors=errors, + metadata={ + "fixture_scope": "hash_pinned_benchmark_slice", + "required_contamination_controls": list(REQUIRED_CONTAMINATION_CONTROLS), + }, + ) + + +def build_fixture_benchmark_source_pin() -> BenchmarkSourcePin: + payload_hash = source_sha256("swe-rebench-v2.fixture.slice.001\n") + return BenchmarkSourcePin( + benchmark_id="swe-rebench-v2", + benchmark_version="fixture-v2", + slice_id="slice-001", + source_uri="fixtures/benchmarks/swe-rebench-v2/slice-001.jsonl", + expected_source_sha256=payload_hash, + ) diff --git a/breadboard/rl/phase2/bridge.py b/breadboard/rl/phase2/bridge.py new file mode 100644 index 00000000..2540f917 --- /dev/null +++ b/breadboard/rl/phase2/bridge.py @@ -0,0 +1,467 @@ +from __future__ import annotations + +import importlib +import json +from dataclasses import dataclass +from typing import Any, Callable, Iterable, Mapping + + +VERL_BATCH_SCHEMA_VERSION = "bb.rl.phase2.verl_batch.v1alpha" +VERL_BATCH_CLAIM_BOUNDARY = "phase2_verl_batch_dry_run_only_not_training_evidence" +__all__ = [ + "TensorShape", + "VERL_BATCH_CLAIM_BOUNDARY", + "VERL_BATCH_SCHEMA_VERSION", + "VerlBatch", + "build_verl_batch_from_projection_rows", + "build_verl_dataproto_like_payload", + "detect_verl_dataproto_api", +] + + +_MASK_FIELDS = ( + "attention_mask", + "loss_mask", + "assistant_mask", + "tool_action_mask", + "reward_mask", +) +_REQUIRED_PROVENANCE_FIELDS = ( + "rollout_id", + "trajectory_id", + "episode_id", + "task_id", + "projection_manifest_id", + "policy_snapshot_id", +) +_PRESERVED_FIELDS = tuple( + sorted( + { + *_REQUIRED_PROVENANCE_FIELDS, + "admission", + "completion_ids", + "completion_logprob_status", + "completion_logprobs", + "env_package_hash", + "env_package_id", + "group_id", + "input_ids", + "metadata", + "policy", + "prompt_ids", + "renderer", + "reward", + "runtime", + "split_id", + "trainable_candidate", + *_MASK_FIELDS, + } + ) +) + + +@dataclass(frozen=True) +class TensorShape: + name: str + dtype: str + shape: tuple[int, ...] + + def to_dict(self) -> dict[str, Any]: + return {"dtype": self.dtype, "shape": list(self.shape)} + + +@dataclass(frozen=True) +class VerlBatch: + batch_id: str + target_run_id: str + policy_snapshot_id: str + row_refs: tuple[dict[str, Any], ...] + tensors: dict[str, list[list[int]]] + masks: dict[str, list[list[bool]]] + logprobs: dict[str, list[list[float]]] + rewards: dict[str, Any] + tensor_shape_metadata: dict[str, TensorShape] + field_ledger: dict[str, Any] + verl_dataproto_api: dict[str, Any] + schema_version: str = VERL_BATCH_SCHEMA_VERSION + claim_boundary: str = VERL_BATCH_CLAIM_BOUNDARY + scorecard_update_allowed: bool = False + + @property + def row_count(self) -> int: + return len(self.row_refs) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "claim_boundary": self.claim_boundary, + "scorecard_update_allowed": self.scorecard_update_allowed, + "batch_id": self.batch_id, + "target_run_id": self.target_run_id, + "policy_snapshot_id": self.policy_snapshot_id, + "row_count": self.row_count, + "row_refs": [dict(row_ref) for row_ref in self.row_refs], + "tensors": self.tensors, + "masks": self.masks, + "logprobs": self.logprobs, + "rewards": self.rewards, + "tensor_shape_metadata": { + name: self.tensor_shape_metadata[name].to_dict() + for name in sorted(self.tensor_shape_metadata) + }, + "field_ledger": self.field_ledger, + "verl_dataproto_api": self.verl_dataproto_api, + } + + def to_json(self) -> str: + return json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) + "\n" + + +def build_verl_dataproto_like_payload(batch: VerlBatch | Mapping[str, Any]) -> dict[str, Any]: + """Return a DataProto-shaped payload while keeping real VeRL optional.""" + + payload = batch.to_dict() if isinstance(batch, VerlBatch) else dict(batch) + return { + "batch": { + "input_ids": payload["tensors"]["input_ids"], + "attention_mask": payload["tensors"]["attention_mask"], + "loss_mask": payload["masks"]["loss_mask"], + "assistant_mask": payload["masks"]["assistant_mask"], + "tool_action_mask": payload["masks"]["tool_action_mask"], + "reward_mask": payload["masks"]["reward_mask"], + "old_log_probs": payload["logprobs"]["completion_logprobs"], + "old_log_probs_mask": payload["masks"]["completion_logprob_mask"], + "token_rewards": payload["rewards"]["token_rewards"], + "sequence_rewards": payload["rewards"]["sequence_rewards"], + }, + "non_tensor_batch": { + "row_refs": payload["row_refs"], + "policy_snapshot_id": payload["policy_snapshot_id"], + "target_run_id": payload["target_run_id"], + }, + "meta_info": { + "schema_version": payload["schema_version"], + "claim_boundary": payload["claim_boundary"], + "scorecard_update_allowed": payload["scorecard_update_allowed"], + "tensor_shape_metadata": payload["tensor_shape_metadata"], + "field_ledger": payload["field_ledger"], + "verl_dataproto_api": payload["verl_dataproto_api"], + }, + } + + +def detect_verl_dataproto_api( + importer: Callable[[str], Any] = importlib.import_module, +) -> dict[str, Any]: + """Detect the optional real VeRL DataProto API without making it a dependency.""" + + attempts: list[dict[str, str]] = [] + for module_name in ("verl.protocol", "verl"): + try: + module = importer(module_name) + except Exception as exc: + attempts.append({"module": module_name, "error": exc.__class__.__name__}) + continue + if hasattr(module, "DataProto"): + return { + "available": True, + "required": False, + "module": module_name, + "symbol": "DataProto", + "attempts": attempts, + } + attempts.append({"module": module_name, "error": "DataProto missing"}) + return { + "available": False, + "required": False, + "module": None, + "symbol": "DataProto", + "attempts": attempts, + } + + +def build_verl_batch_from_projection_rows( + rows: Iterable[Any], + *, + target_run_id: str, + batch_id: str | None = None, +) -> VerlBatch: + materialized_rows = [_row_to_mapping(row) for row in rows] + if not materialized_rows: + raise ValueError("rows must contain at least one projection row") + target_run_id = _require_text(target_run_id, "target_run_id") + batch_id = _require_text(batch_id or f"{target_run_id}.verl_batch", "batch_id") + + errors: list[str] = [] + normalized_rows: list[dict[str, Any]] = [] + for row_index, row in enumerate(materialized_rows, start=1): + normalized, row_errors = _normalize_row(row, row_index) + normalized_rows.append(normalized) + errors.extend(row_errors) + + policy_snapshot_ids = sorted({row["policy_snapshot_id"] for row in normalized_rows if row.get("policy_snapshot_id")}) + if len(policy_snapshot_ids) > 1: + errors.append("all rows in a VeRL batch must share one policy_snapshot_id") + if errors: + raise ValueError("; ".join(errors)) + + max_sequence_length = max(len(row["input_ids"]) for row in normalized_rows) + max_completion_length = max(len(row["completion_ids"]) for row in normalized_rows) + row_count = len(normalized_rows) + + tensors = { + "input_ids": [_pad_ints(row["input_ids"], max_sequence_length) for row in normalized_rows], + "attention_mask": [_pad_ints(row["attention_mask"], max_sequence_length) for row in normalized_rows], + } + masks = { + field_name: [_pad_bools(row[field_name], max_sequence_length) for row in normalized_rows] + for field_name in _MASK_FIELDS + if field_name != "attention_mask" + } + masks["completion_logprob_mask"] = [ + [index < len(row["completion_logprobs"]) for index in range(max_completion_length)] + for row in normalized_rows + ] + logprobs = { + "completion_logprobs": [ + _pad_floats(row["completion_logprobs"], max_completion_length) for row in normalized_rows + ] + } + + sequence_rewards = [row["reward_scalar"] for row in normalized_rows] + token_rewards = [] + for row in normalized_rows: + reward_scalar = row["reward_scalar"] + token_rewards.append( + [reward_scalar if mask_value else 0.0 for mask_value in _pad_bools(row["reward_mask"], max_sequence_length)] + ) + rewards = {"sequence_rewards": sequence_rewards, "token_rewards": token_rewards} + + tensor_shape_metadata = { + "input_ids": TensorShape("input_ids", "int64", (row_count, max_sequence_length)), + "attention_mask": TensorShape("attention_mask", "int64", (row_count, max_sequence_length)), + "loss_mask": TensorShape("loss_mask", "bool", (row_count, max_sequence_length)), + "assistant_mask": TensorShape("assistant_mask", "bool", (row_count, max_sequence_length)), + "tool_action_mask": TensorShape("tool_action_mask", "bool", (row_count, max_sequence_length)), + "reward_mask": TensorShape("reward_mask", "bool", (row_count, max_sequence_length)), + "completion_logprobs": TensorShape("completion_logprobs", "float32", (row_count, max_completion_length)), + "completion_logprob_mask": TensorShape("completion_logprob_mask", "bool", (row_count, max_completion_length)), + "sequence_rewards": TensorShape("sequence_rewards", "float32", (row_count,)), + "token_rewards": TensorShape("token_rewards", "float32", (row_count, max_sequence_length)), + } + + return VerlBatch( + batch_id=batch_id, + target_run_id=target_run_id, + policy_snapshot_id=policy_snapshot_ids[0], + row_refs=tuple(_row_ref(row) for row in normalized_rows), + tensors=tensors, + masks=masks, + logprobs=logprobs, + rewards=rewards, + tensor_shape_metadata=tensor_shape_metadata, + field_ledger=_field_ledger(normalized_rows), + verl_dataproto_api=detect_verl_dataproto_api(), + ) + + +def _row_to_mapping(row: Any) -> Mapping[str, Any]: + if isinstance(row, Mapping): + return row + to_dict = getattr(row, "to_dict", None) + if callable(to_dict): + payload = to_dict() + if isinstance(payload, Mapping): + return payload + raise TypeError("projection rows must be mappings or expose to_dict()") + + +def _normalize_row(row: Mapping[str, Any], row_index: int) -> tuple[dict[str, Any], list[str]]: + errors: list[str] = [] + normalized: dict[str, Any] = {"_source_fields": tuple(sorted(str(key) for key in row.keys()))} + + for field_name in ("input_ids", "prompt_ids", "completion_ids"): + normalized[field_name] = _int_list(row.get(field_name), field_name, errors) + normalized["attention_mask"] = _int_list(row.get("attention_mask"), "attention_mask", errors) + for field_name in ("loss_mask", "assistant_mask", "tool_action_mask", "reward_mask"): + normalized[field_name] = _bool_list(row.get(field_name), field_name, errors) + + completion_logprobs = row.get("completion_logprobs") + normalized["completion_logprobs"] = _float_list(completion_logprobs, "completion_logprobs", errors) + normalized["completion_logprob_status"] = str(row.get("completion_logprob_status") or "") + + policy_snapshot_id = _policy_snapshot_id(row) + if not policy_snapshot_id: + errors.append(f"row {row_index} policy_snapshot_id must be present") + normalized["policy_snapshot_id"] = policy_snapshot_id + + admission = _mapping(row.get("admission"), "admission", errors) + normalized["admission"] = admission + if admission.get("quarantine_status") == "quarantined": + errors.append(f"row {row_index} is quarantined and cannot enter a VeRL batch") + elif admission.get("quarantine_status") != "clear": + errors.append(f"row {row_index} quarantine_status must be clear") + if admission.get("row_status") != "accepted": + errors.append(f"row {row_index} row_status must be accepted") + if admission.get("trainable") is not True or row.get("trainable_candidate") is False: + errors.append(f"row {row_index} must be trainable") + + input_length = len(normalized["input_ids"]) + for field_name in _MASK_FIELDS: + if len(normalized[field_name]) != input_length: + errors.append(f"row {row_index} {field_name} length must equal input_ids length") + if normalized["input_ids"] != [*normalized["prompt_ids"], *normalized["completion_ids"]]: + errors.append(f"row {row_index} input_ids must equal prompt_ids + completion_ids") + if len(normalized["completion_logprobs"]) != len(normalized["completion_ids"]): + errors.append(f"row {row_index} completion_logprobs length must equal completion_ids length") + if normalized["completion_logprob_status"] != "native_available": + errors.append(f"row {row_index} completion_logprob_status must be native_available") + + reward = _mapping(row.get("reward"), "reward", errors) + normalized["reward"] = reward + normalized["reward_scalar"] = _reward_scalar(reward, row_index, errors) + + for field_name in _REQUIRED_PROVENANCE_FIELDS: + if field_name == "policy_snapshot_id": + continue + value = row.get(field_name) + if value is None or (isinstance(value, str) and not value.strip()): + errors.append(f"row {row_index} {field_name} must be present") + normalized[field_name] = value + + for field_name in ( + "env_package_id", + "env_package_hash", + "group_id", + "split_id", + "policy", + "renderer", + "runtime", + "metadata", + "trainable_candidate", + ): + if field_name in row: + normalized[field_name] = row[field_name] + return normalized, errors + + +def _policy_snapshot_id(row: Mapping[str, Any]) -> str: + direct_value = row.get("policy_snapshot_id") + if direct_value is not None and str(direct_value).strip(): + return str(direct_value).strip() + policy = row.get("policy") + if isinstance(policy, Mapping): + for field_name in ("policy_snapshot_id", "snapshot_id", "checkpoint_ref"): + value = policy.get(field_name) + if value is not None and str(value).strip(): + return str(value).strip() + return "" + + +def _require_text(value: Any, field_name: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError(f"{field_name} must be non-empty") + return text + + +def _mapping(value: Any, field_name: str, errors: list[str]) -> dict[str, Any]: + if isinstance(value, Mapping): + return dict(value) + errors.append(f"{field_name} must be a mapping") + return {} + + +def _int_list(value: Any, field_name: str, errors: list[str]) -> list[int]: + if not isinstance(value, list): + errors.append(f"{field_name} must be a list") + return [] + try: + return [int(item) for item in value] + except (TypeError, ValueError): + errors.append(f"{field_name} must contain integers") + return [] + + +def _bool_list(value: Any, field_name: str, errors: list[str]) -> list[bool]: + if not isinstance(value, list): + errors.append(f"{field_name} must be a list") + return [] + return [bool(item) for item in value] + + +def _float_list(value: Any, field_name: str, errors: list[str]) -> list[float]: + if not isinstance(value, list): + errors.append(f"{field_name} must be a list") + return [] + try: + return [float(item) for item in value] + except (TypeError, ValueError): + errors.append(f"{field_name} must contain floats") + return [] + + +def _reward_scalar(reward: Mapping[str, Any], row_index: int, errors: list[str]) -> float: + try: + return float(reward["scalar"]) + except KeyError: + errors.append(f"row {row_index} reward.scalar must be present") + except (TypeError, ValueError): + errors.append(f"row {row_index} reward.scalar must be numeric") + return 0.0 + + +def _pad_ints(values: list[int], width: int) -> list[int]: + return values + [0] * (width - len(values)) + + +def _pad_bools(values: list[bool], width: int) -> list[bool]: + return values + [False] * (width - len(values)) + + +def _pad_floats(values: list[float], width: int) -> list[float]: + return values + [0.0] * (width - len(values)) + + +def _row_ref(row: Mapping[str, Any]) -> dict[str, Any]: + return { + "rollout_id": row["rollout_id"], + "trajectory_id": row["trajectory_id"], + "episode_id": row["episode_id"], + "task_id": row["task_id"], + "projection_manifest_id": row["projection_manifest_id"], + "policy_snapshot_id": row["policy_snapshot_id"], + "reward_scalar": row["reward_scalar"], + } + + +def _field_ledger(rows: list[Mapping[str, Any]]) -> dict[str, Any]: + row_ledgers: list[dict[str, Any]] = [] + all_source_fields: set[str] = set() + all_preserved_fields: set[str] = set() + all_lost_fields: set[str] = set() + for row in rows: + source_fields = set(row["_source_fields"]) + preserved_fields = source_fields.intersection(_PRESERVED_FIELDS) + if row.get("policy_snapshot_id"): + preserved_fields.add("policy_snapshot_id") + lost_fields = source_fields.difference(_PRESERVED_FIELDS) + all_source_fields.update(source_fields) + all_preserved_fields.update(preserved_fields) + all_lost_fields.update(lost_fields) + row_ledgers.append( + { + "task_id": row["task_id"], + "preserved_fields": sorted(preserved_fields), + "lost_fields": sorted(lost_fields), + } + ) + critical_lost_fields = sorted(all_lost_fields.intersection(_REQUIRED_PROVENANCE_FIELDS)) + return { + "source_fields": sorted(all_source_fields), + "preserved_fields": sorted(all_preserved_fields), + "lost_fields": sorted(all_lost_fields), + "critical_lost_fields": critical_lost_fields, + "provenance_loss_detected": bool(critical_lost_fields), + "row_ledgers": row_ledgers, + } diff --git a/breadboard/rl/phase2/closed_loop.py b/breadboard/rl/phase2/closed_loop.py new file mode 100644 index 00000000..d799d80f --- /dev/null +++ b/breadboard/rl/phase2/closed_loop.py @@ -0,0 +1,337 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from hashlib import sha256 +import json +from typing import Any, Mapping, Sequence + + +CLAIM_BOUNDARY = "p2_m3_closed_loop_prototype_not_production_rl_claim" + + +def _stable_json(data: Mapping[str, Any]) -> str: + return json.dumps(data, sort_keys=True, separators=(",", ":")) + + +def stable_sha256(data: Mapping[str, Any]) -> str: + return sha256(_stable_json(data).encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class PolicySnapshotIdentity: + policy_name: str + checkpoint_ref: str + parameter_sha256: str + trainer_name: str + created_at: str + metadata: dict[str, Any] = field(default_factory=dict) + + @property + def snapshot_id(self) -> str: + return "policy_snapshot:" + stable_sha256( + { + "checkpoint_ref": self.checkpoint_ref, + "created_at": self.created_at, + "parameter_sha256": self.parameter_sha256, + "policy_name": self.policy_name, + "trainer_name": self.trainer_name, + } + )[:16] + + def to_dict(self) -> dict[str, Any]: + return { + "policy_snapshot_id": self.snapshot_id, + "policy_name": self.policy_name, + "checkpoint_ref": self.checkpoint_ref, + "parameter_sha256": self.parameter_sha256, + "trainer_name": self.trainer_name, + "created_at": self.created_at, + "metadata": dict(self.metadata), + } + + +@dataclass(frozen=True) +class RolloutGenerationEntry: + rollout_id: str + policy_snapshot_id: str + env_package_id: str + task_id: str + trajectory_ref: str + projection_ref: str + generated_at: str + target_run_id: str + attempt_index: int = 0 + + def to_dict(self) -> dict[str, Any]: + return { + "rollout_id": self.rollout_id, + "policy_snapshot_id": self.policy_snapshot_id, + "env_package_id": self.env_package_id, + "task_id": self.task_id, + "trajectory_ref": self.trajectory_ref, + "projection_ref": self.projection_ref, + "generated_at": self.generated_at, + "target_run_id": self.target_run_id, + "attempt_index": self.attempt_index, + } + + +@dataclass(frozen=True) +class RewardVerifierEvidence: + evidence_id: str + rollout_id: str + verifier_id: str + verifier_version: str + reward_scalar: float + verifier_evidence_hash: str + passed: bool + failure_taxonomy: str = "none" + + def to_dict(self) -> dict[str, Any]: + return { + "evidence_id": self.evidence_id, + "rollout_id": self.rollout_id, + "verifier_id": self.verifier_id, + "verifier_version": self.verifier_version, + "reward_scalar": self.reward_scalar, + "verifier_evidence_hash": self.verifier_evidence_hash, + "passed": self.passed, + "failure_taxonomy": self.failure_taxonomy, + } + + +@dataclass(frozen=True) +class AdmissionDecision: + decision_id: str + rollout_id: str + accepted: bool + reasons: list[str] + decided_at: str + + def to_dict(self) -> dict[str, Any]: + return { + "decision_id": self.decision_id, + "rollout_id": self.rollout_id, + "accepted": self.accepted, + "reasons": list(self.reasons), + "decided_at": self.decided_at, + } + + +@dataclass(frozen=True) +class TrainerHandoff: + handoff_id: str + admitted_rollout_ids: list[str] + trainer_name: str + trainer_projection_hash: str + target_run_id: str + + def to_dict(self) -> dict[str, Any]: + return { + "handoff_id": self.handoff_id, + "admitted_rollout_ids": list(self.admitted_rollout_ids), + "trainer_name": self.trainer_name, + "trainer_projection_hash": self.trainer_projection_hash, + "target_run_id": self.target_run_id, + } + + +@dataclass(frozen=True) +class ReplayClosure: + closure_id: str + rollout_id: str + replay_status: str + admission_accepted: bool + replay_ref: str + verifier_evidence_id: str + trainer_handoff_id: str | None + target_run_id: str + claim_boundary: str = CLAIM_BOUNDARY + scorecard_update_allowed: bool = False + errors: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "closure_id": self.closure_id, + "rollout_id": self.rollout_id, + "replay_status": self.replay_status, + "admission_accepted": self.admission_accepted, + "replay_ref": self.replay_ref, + "verifier_evidence_id": self.verifier_evidence_id, + "trainer_handoff_id": self.trainer_handoff_id, + "target_run_id": self.target_run_id, + "claim_boundary": self.claim_boundary, + "scorecard_update_allowed": self.scorecard_update_allowed, + "errors": list(self.errors), + } + + +def make_rollout_id(policy_snapshot_id: str, env_package_id: str, task_id: str, attempt_index: int) -> str: + digest = stable_sha256( + { + "attempt_index": attempt_index, + "env_package_id": env_package_id, + "policy_snapshot_id": policy_snapshot_id, + "task_id": task_id, + } + )[:16] + return "rollout:" + digest + + +def close_replay( + rollout: RolloutGenerationEntry, + evidence: RewardVerifierEvidence, + admission: AdmissionDecision, + *, + replay_ref: str, + trainer_handoff: TrainerHandoff | None = None, +) -> ReplayClosure: + errors: list[str] = [] + if evidence.rollout_id != rollout.rollout_id: + errors.append("evidence_rollout_mismatch") + if admission.rollout_id != rollout.rollout_id: + errors.append("admission_rollout_mismatch") + if admission.accepted: + if not evidence.passed: + errors.append("accepted_rollout_failed_verifier") + if trainer_handoff is None: + errors.append("accepted_rollout_missing_trainer_handoff") + elif rollout.rollout_id not in trainer_handoff.admitted_rollout_ids: + errors.append("accepted_rollout_absent_from_trainer_handoff") + elif trainer_handoff is not None and rollout.rollout_id in trainer_handoff.admitted_rollout_ids: + errors.append("rejected_rollout_present_in_trainer_handoff") + + if errors: + replay_status = "replay_closure_invalid" + elif admission.accepted: + replay_status = "accepted_replay_closed" + else: + replay_status = "rejected_replay_closed" + + trainer_handoff_id = trainer_handoff.handoff_id if trainer_handoff is not None else None + closure_id = "replay_closure:" + stable_sha256( + { + "admission_accepted": admission.accepted, + "replay_ref": replay_ref, + "rollout_id": rollout.rollout_id, + "trainer_handoff_id": trainer_handoff_id, + "verifier_evidence_id": evidence.evidence_id, + } + )[:16] + return ReplayClosure( + closure_id=closure_id, + rollout_id=rollout.rollout_id, + replay_status=replay_status, + admission_accepted=admission.accepted, + replay_ref=replay_ref, + verifier_evidence_id=evidence.evidence_id, + trainer_handoff_id=trainer_handoff_id, + target_run_id=rollout.target_run_id, + errors=errors, + ) + + +def build_closed_loop_fixture_ledger(target_run_id: str = "fixture-phase2-closed-loop") -> dict[str, Any]: + snapshot = PolicySnapshotIdentity( + policy_name="breadboard-swe-tiny-policy", + checkpoint_ref="fixtures/policies/swe_tiny/checkpoint-0001", + parameter_sha256="0" * 64, + trainer_name="verl.fixture.off_policy", + created_at="2026-06-18T00:00:00Z", + metadata={"fixture_scope": "deterministic_closed_loop"}, + ) + accepted_rollout = RolloutGenerationEntry( + rollout_id=make_rollout_id(snapshot.snapshot_id, "swe_toy_patch", "accepted-task", 0), + policy_snapshot_id=snapshot.snapshot_id, + env_package_id="swe_toy_patch", + task_id="accepted-task", + trajectory_ref="cas://trajectory/accepted-task", + projection_ref="cas://projection/accepted-task", + generated_at="2026-06-18T00:01:00Z", + target_run_id=target_run_id, + ) + rejected_rollout = RolloutGenerationEntry( + rollout_id=make_rollout_id(snapshot.snapshot_id, "swe_toy_patch", "rejected-task", 0), + policy_snapshot_id=snapshot.snapshot_id, + env_package_id="swe_toy_patch", + task_id="rejected-task", + trajectory_ref="cas://trajectory/rejected-task", + projection_ref="cas://projection/rejected-task", + generated_at="2026-06-18T00:02:00Z", + target_run_id=target_run_id, + ) + accepted_evidence = RewardVerifierEvidence( + evidence_id="evidence:accepted-task", + rollout_id=accepted_rollout.rollout_id, + verifier_id="swe_toy_verifier", + verifier_version="fixture-v1", + reward_scalar=1.0, + verifier_evidence_hash="1" * 64, + passed=True, + ) + rejected_evidence = RewardVerifierEvidence( + evidence_id="evidence:rejected-task", + rollout_id=rejected_rollout.rollout_id, + verifier_id="swe_toy_verifier", + verifier_version="fixture-v1", + reward_scalar=0.0, + verifier_evidence_hash="2" * 64, + passed=False, + failure_taxonomy="verifier_failure", + ) + accepted_admission = AdmissionDecision( + decision_id="admission:accepted-task", + rollout_id=accepted_rollout.rollout_id, + accepted=True, + reasons=["verifier_passed", "projection_schema_valid", "replay_ref_present"], + decided_at="2026-06-18T00:03:00Z", + ) + rejected_admission = AdmissionDecision( + decision_id="admission:rejected-task", + rollout_id=rejected_rollout.rollout_id, + accepted=False, + reasons=["verifier_failed"], + decided_at="2026-06-18T00:04:00Z", + ) + handoff = TrainerHandoff( + handoff_id="trainer_handoff:accepted-only", + admitted_rollout_ids=[accepted_rollout.rollout_id], + trainer_name="verl.fixture.off_policy", + trainer_projection_hash="3" * 64, + target_run_id=target_run_id, + ) + closures = [ + close_replay( + accepted_rollout, + accepted_evidence, + accepted_admission, + replay_ref="cas://replay/accepted-task", + trainer_handoff=handoff, + ), + close_replay( + rejected_rollout, + rejected_evidence, + rejected_admission, + replay_ref="cas://replay/rejected-task", + ), + ] + return { + "report_id": "bb_zyphra_rl_phase2_closed_loop_v1", + "target_run_id": target_run_id, + "claim_boundary": CLAIM_BOUNDARY, + "scorecard_update_allowed": False, + "passed": all(closure.replay_status != "replay_closure_invalid" for closure in closures), + "policy_snapshot": snapshot.to_dict(), + "rollouts": [accepted_rollout.to_dict(), rejected_rollout.to_dict()], + "reward_verifier_evidence": [accepted_evidence.to_dict(), rejected_evidence.to_dict()], + "admission_decisions": [accepted_admission.to_dict(), rejected_admission.to_dict()], + "trainer_handoffs": [handoff.to_dict()], + "replay_closures": [closure.to_dict() for closure in closures], + } + + +def replay_closure_errors(closures: Sequence[ReplayClosure]) -> list[str]: + errors: list[str] = [] + for closure in closures: + errors.extend(closure.errors) + return errors diff --git a/breadboard/rl/phase2/env_family.py b/breadboard/rl/phase2/env_family.py new file mode 100644 index 00000000..6d862373 --- /dev/null +++ b/breadboard/rl/phase2/env_family.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +CLAIM_BOUNDARY = "p2_m8_second_environment_probe_not_general_env_support_claim" + + +@dataclass(frozen=True) +class ProbeCheck: + name: str + status: str + evidence_ref: str + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "status": self.status, + "evidence_ref": self.evidence_ref, + } + + +@dataclass(frozen=True) +class SecondEnvironmentFamilyProbeReport: + report_id: str + family_id: str + env_package_id: str + target_run_id: str + renderer_probe: ProbeCheck + replay_probe: ProbeCheck + export_probe: ProbeCheck + target_smoke: ProbeCheck + claim_boundary: str = CLAIM_BOUNDARY + scorecard_update_allowed: bool = False + preserved_fields: list[str] = field(default_factory=list) + lost_fields: list[str] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + @property + def smoke_ready(self) -> bool: + checks = (self.renderer_probe, self.replay_probe, self.export_probe, self.target_smoke) + return all(check.status == "passed" for check in checks) + + def to_dict(self) -> dict[str, Any]: + return { + "report_id": self.report_id, + "family_id": self.family_id, + "env_package_id": self.env_package_id, + "target_run_id": self.target_run_id, + "claim_boundary": self.claim_boundary, + "scorecard_update_allowed": self.scorecard_update_allowed, + "passed": self.smoke_ready, + "smoke_ready": self.smoke_ready, + "renderer_probe": self.renderer_probe.to_dict(), + "replay_probe": self.replay_probe.to_dict(), + "export_probe": self.export_probe.to_dict(), + "target_smoke": self.target_smoke.to_dict(), + "preserved_fields": list(self.preserved_fields), + "lost_fields": list(self.lost_fields), + "metadata": dict(self.metadata), + } + + +def build_second_env_family_probe_report( + *, + family_id: str, + env_package_id: str, + target_run_id: str = "fixture-phase2-second-env-family", + target_smoke_status: str = "passed", +) -> SecondEnvironmentFamilyProbeReport: + return SecondEnvironmentFamilyProbeReport( + report_id="bb_zyphra_rl_phase2_second_env_family_v1", + family_id=family_id, + env_package_id=env_package_id, + target_run_id=target_run_id, + renderer_probe=ProbeCheck( + name="renderer_transcript_shape", + status="passed", + evidence_ref="cas://env-family/renderer/" + family_id, + ), + replay_probe=ProbeCheck( + name="deterministic_replay", + status="passed", + evidence_ref="cas://env-family/replay/" + family_id, + ), + export_probe=ProbeCheck( + name="trainer_projection_shape", + status="passed", + evidence_ref="cas://env-family/export/" + family_id, + ), + target_smoke=ProbeCheck( + name="target_smoke", + status=target_smoke_status, + evidence_ref="cas://env-family/target-smoke/" + family_id, + ), + preserved_fields=[ + "env_package_id", + "renderer_events", + "replay_seed", + "export_projection_ref", + "target_smoke_status", + ], + lost_fields=["live_target_scheduler_log"], + metadata={ + "fixture_scope": "second_environment_family_schema_smoke", + "candidate_family": family_id, + }, + ) + + +def build_lean_console_fixture_probe_report() -> SecondEnvironmentFamilyProbeReport: + return build_second_env_family_probe_report( + family_id="lean_console", + env_package_id="lean_console_fixture_env", + ) diff --git a/breadboard/rl/phase2/final_report.py b/breadboard/rl/phase2/final_report.py new file mode 100644 index 00000000..a5a76126 --- /dev/null +++ b/breadboard/rl/phase2/final_report.py @@ -0,0 +1,178 @@ +from __future__ import annotations +from collections.abc import Mapping + +import json +from pathlib import Path +from typing import Any + +from breadboard.rl.phase2.hardening import HARDENING_CLAIM_BOUNDARY +from breadboard.rl.phase2.observability import OBSERVABILITY_CLAIM_BOUNDARY +from breadboard.rl.phase2.service import SERVICE_CLAIM_BOUNDARY + + +PHASE2_FINAL_REPORT_ID = "bb_zyphra_rl_phase2_final_report_v1" +PHASE2_FINAL_CLAIM_BOUNDARY = "phase2_target_validation_candidate_not_scorecard_update" +PHASE2_COMPONENT_MILESTONES = tuple(f"P2-M{index}" for index in range(12)) +EXPECTED_MILESTONE_CLAIM_BOUNDARIES: dict[str, str] = { + "P2-M0": "p2_m0_baseline_freeze_not_scorecard_update", + "P2-M1": "phase2_verl_batch_dry_run_only_not_training_evidence", + "P2-M2": "phase2_trainer_dry_run_no_slurm_no_weight_update", + "P2-M3": "p2_m3_closed_loop_prototype_not_production_rl_claim", + "P2-M4": "p2_m4_scale_ladder_not_arbitrary_production_scale", + "P2-M5": "p2_m5_named_benchmark_slice_not_general_benchmark_claim", + "P2-M6": "p2_m6_live_verifier_probe_not_general_verifier_claim", + "P2-M7": "p2_m7_benchflow_probe_not_full_security_coverage_claim", + "P2-M8": "p2_m8_second_environment_probe_not_general_env_support_claim", + "P2-M9": SERVICE_CLAIM_BOUNDARY, + "P2-M10": OBSERVABILITY_CLAIM_BOUNDARY, + "P2-M11": HARDENING_CLAIM_BOUNDARY, +} + + +def build_phase2_component_report( + *, + milestone_id: str, + report_id: str, + claim_boundary: str, + target_run_id: str, + passed: bool, + summary: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + return { + "claim_boundary": claim_boundary, + "milestone_id": milestone_id, + "passed": passed, + "report_id": report_id, + "scorecard_update_allowed": False, + "summary": dict(summary or {}), + "target_run_id": target_run_id, + } + +def _component_passed(report: Mapping[str, Any]) -> bool: + if "passed" in report: + return report.get("passed") is True + if "ready" in report: + return report.get("ready") is True + if "hardening_passed" in report: + return report.get("hardening_passed") is True + dry_run_result = report.get("dry_run_result") + if isinstance(dry_run_result, Mapping): + return dry_run_result.get("accepted") is True + errors = report.get("errors") + if isinstance(errors, list): + return not errors + return True + + +def _component_summary(milestone_id: str, report: Mapping[str, Any] | None, target_run_id: str) -> dict[str, Any]: + expected_boundary = EXPECTED_MILESTONE_CLAIM_BOUNDARIES[milestone_id] + if report is None: + return { + "claim_boundary_match": False, + "expected_claim_boundary": expected_boundary, + "milestone_id": milestone_id, + "passed": False, + "present": False, + "report_id": None, + "scorecard_update_allowed": None, + "target_run_id": None, + "target_run_id_match": False, + } + observed_target_run_id = str(report.get("target_run_id") or "") + return { + "claim_boundary_match": report.get("claim_boundary") == expected_boundary, + "expected_claim_boundary": expected_boundary, + "milestone_id": milestone_id, + "observed_claim_boundary": report.get("claim_boundary"), + "passed": _component_passed(report), + "present": True, + "report_id": report.get("report_id"), + "scorecard_update_allowed": report.get("scorecard_update_allowed"), + "target_run_id": observed_target_run_id, + "target_run_id_match": observed_target_run_id == target_run_id, + } + + +def build_phase2_final_report( + *, + target_run_id: str, + milestone_reports: Mapping[str, Mapping[str, Any]], + command_log_manifest: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + summaries = [_component_summary(milestone_id, milestone_reports.get(milestone_id), target_run_id) for milestone_id in PHASE2_COMPONENT_MILESTONES] + missing = [summary["milestone_id"] for summary in summaries if not summary["present"]] + failed = [summary["milestone_id"] for summary in summaries if summary["present"] and not summary["passed"]] + boundary_mismatches = [summary["milestone_id"] for summary in summaries if not summary["claim_boundary_match"]] + scorecard_updates = [summary["milestone_id"] for summary in summaries if summary["scorecard_update_allowed"] is not False] + target_mismatches = [summary["milestone_id"] for summary in summaries if not summary["target_run_id_match"]] + command_manifest = dict(command_log_manifest or {}) + command_manifest_present = bool(command_manifest) + command_manifest_target_run_id = str(command_manifest.get("target_run_id") or "") + command_manifest_target_run_id_match = command_manifest_present and command_manifest_target_run_id == target_run_id + final_ready = not (missing or failed or boundary_mismatches or scorecard_updates or target_mismatches) and command_manifest_target_run_id_match + return { + "claim_boundary": PHASE2_FINAL_CLAIM_BOUNDARY, + "command_log_manifest_present": command_manifest_present, + "command_log_manifest_target_run_id": command_manifest_target_run_id, + "command_log_manifest_target_run_id_match": command_manifest_target_run_id_match, + "component_reports": summaries, + "failed_milestones": failed, + "final_report_ready": final_ready, + "missing_milestones": missing, + "phase": "RL_PHASE_2", + "report_id": PHASE2_FINAL_REPORT_ID, + "scorecard_update_allowed": False, + "target_run_id": target_run_id, + "validation": { + "boundary_mismatches": boundary_mismatches, + "scorecard_update_attempts": scorecard_updates, + "target_run_id_mismatches": target_mismatches, + }, + } + + +def validate_phase2_final_report(report: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != PHASE2_FINAL_REPORT_ID: + errors.append("report_id must be phase2 final report v1") + if report.get("claim_boundary") != PHASE2_FINAL_CLAIM_BOUNDARY: + errors.append("claim_boundary must be phase2 target-validation boundary") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if not str(report.get("target_run_id") or ""): + errors.append("target_run_id must be non-empty") + if report.get("command_log_manifest_present") is not True: + errors.append("command_log_manifest_present must be true") + if report.get("command_log_manifest_target_run_id_match") is not True: + errors.append("command_log_manifest_target_run_id_match must be true") + if report.get("missing_milestones"): + errors.append("missing_milestones must be empty") + if report.get("failed_milestones"): + errors.append("failed_milestones must be empty") + validation = report.get("validation") if isinstance(report.get("validation"), Mapping) else {} + for key in ["boundary_mismatches", "scorecard_update_attempts", "target_run_id_mismatches"]: + if validation.get(key): + errors.append(f"validation.{key} must be empty") + summaries = report.get("component_reports") if isinstance(report.get("component_reports"), list) else [] + if [summary.get("milestone_id") for summary in summaries if isinstance(summary, Mapping)] != list(PHASE2_COMPONENT_MILESTONES): + errors.append("component_reports must include every P2-M0 through P2-M11 row in order") + if report.get("final_report_ready") is not True: + errors.append("final_report_ready must be true") + return errors + + +def write_phase2_final_report( + path: Path, + *, + target_run_id: str, + milestone_reports: Mapping[str, Mapping[str, Any]], + command_log_manifest: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + report = build_phase2_final_report( + target_run_id=target_run_id, + milestone_reports=milestone_reports, + command_log_manifest=command_log_manifest, + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return report diff --git a/breadboard/rl/phase2/hardening.py b/breadboard/rl/phase2/hardening.py new file mode 100644 index 00000000..f8339036 --- /dev/null +++ b/breadboard/rl/phase2/hardening.py @@ -0,0 +1,205 @@ +from __future__ import annotations +from collections.abc import Mapping + +import json + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Any + + +HARDENING_REPORT_ID = "bb_zyphra_rl_phase2_multi_tenant_hardening_v1" +HARDENING_CLAIM_BOUNDARY = "p2_m11_hardening_policy_not_production_isolation_claim" +REDACTED = "" +_SECRET_KEY_PARTS = ("token", "secret", "password", "api_key", "apikey", "authorization") +_DESTRUCTIVE_PATTERNS = ( + "rm -rf", + "sudo rm", + "mkfs", + "dd if=", + "dd of=", + "chmod -r 777", + "chown -r", + ":(){ :|:& };:", +) + + +@dataclass(frozen=True) +class EgressPolicy: + allowed_prefixes: tuple[str, ...] + max_artifact_bytes: int + allow_absolute_paths: bool = False + + def to_dict(self) -> dict[str, Any]: + return { + "allow_absolute_paths": self.allow_absolute_paths, + "allowed_prefixes": list(self.allowed_prefixes), + "max_artifact_bytes": self.max_artifact_bytes, + } + + +@dataclass(frozen=True) +class ArtifactEgressRequest: + relative_path: str + bytes: int + classification: str + + def to_dict(self) -> dict[str, Any]: + return { + "bytes": self.bytes, + "classification": self.classification, + "relative_path": self.relative_path, + } + + +@dataclass(frozen=True) +class DestructiveActionRequest: + action_id: str + command: str + workspace_relative_path: str + + def to_dict(self) -> dict[str, str]: + return { + "action_id": self.action_id, + "command": self.command, + "workspace_relative_path": self.workspace_relative_path, + } + + +def _is_safe_relative_path(path: str) -> bool: + if not path or path.startswith("~"): + return False + pure = PurePosixPath(path.replace("\\", "/")) + if pure.is_absolute(): + return False + return ".." not in pure.parts + + +def workspace_isolated(path: str, *, workspace_id: str) -> bool: + if not _is_safe_relative_path(path): + return False + pure = PurePosixPath(path.replace("\\", "/")) + return bool(pure.parts) and pure.parts[0] == workspace_id + + +def redact_mapping(values: Mapping[str, Any]) -> dict[str, Any]: + redacted: dict[str, Any] = {} + for key in sorted(values): + raw = values[key] + lowered_key = str(key).lower() + if any(part in lowered_key for part in _SECRET_KEY_PARTS): + redacted[str(key)] = REDACTED + elif isinstance(raw, str) and (raw.startswith("/") or raw.startswith("~")): + redacted[str(key)] = REDACTED + else: + redacted[str(key)] = raw + return redacted + + +def evaluate_artifact_egress(request: ArtifactEgressRequest, policy: EgressPolicy) -> dict[str, Any]: + reasons: list[str] = [] + normalized = request.relative_path.replace("\\", "/") + if not policy.allow_absolute_paths and not _is_safe_relative_path(normalized): + reasons.append("path must be workspace-relative and cannot escape with ..") + if request.bytes > policy.max_artifact_bytes: + reasons.append("artifact exceeds max_artifact_bytes") + if request.classification not in {"public", "tenant_internal"}: + reasons.append("classification is not egress-approved") + if not any(normalized == prefix or normalized.startswith(prefix.rstrip("/") + "/") for prefix in policy.allowed_prefixes): + reasons.append("path prefix is not egress-approved") + return { + "allowed": not reasons, + "reasons": reasons, + "request": request.to_dict(), + } + + +def guard_destructive_action(request: DestructiveActionRequest, *, workspace_id: str) -> dict[str, Any]: + command = " ".join(request.command.lower().split()) + reasons: list[str] = [] + if any(pattern in command for pattern in _DESTRUCTIVE_PATTERNS): + reasons.append("command matches destructive-action denylist") + if not workspace_isolated(request.workspace_relative_path, workspace_id=workspace_id): + reasons.append("workspace path is not isolated to tenant workspace") + return { + "allowed": not reasons, + "reasons": reasons, + "request": request.to_dict(), + } + + +def build_hardening_report( + *, + run_id: str, + target_run_id: str, + tenant_id: str, + workspace_id: str, + egress_policy: EgressPolicy, + egress_requests: list[ArtifactEgressRequest], + destructive_actions: list[DestructiveActionRequest], + environment: Mapping[str, Any], + adversarial_package_results: list[Mapping[str, Any]], +) -> dict[str, Any]: + egress_results = [evaluate_artifact_egress(request, egress_policy) for request in egress_requests] + destructive_results = [guard_destructive_action(request, workspace_id=workspace_id) for request in destructive_actions] + adversarial_results = [dict(sorted(result.items())) for result in adversarial_package_results] + failed_adversarial = [str(result.get("package_id") or result.get("name") or index) for index, result in enumerate(adversarial_results) if result.get("passed") is not True] + hardening_passed = ( + all(result["allowed"] for result in egress_results) + and all(result["allowed"] for result in destructive_results) + and not failed_adversarial + ) + return { + "adversarial_package_results": adversarial_results, + "artifact_egress_policy": egress_policy.to_dict(), + "artifact_egress_results": egress_results, + "claim_boundary": HARDENING_CLAIM_BOUNDARY, + "milestone_id": "P2-M11", + "passed": hardening_passed, + "destructive_action_guards": destructive_results, + "redacted_environment": redact_mapping(environment), + "redaction_policy": { + "redacted_value": REDACTED, + "secret_key_parts": list(_SECRET_KEY_PARTS), + "absolute_or_home_paths_redacted": True, + }, + "report_id": HARDENING_REPORT_ID, + "run_id": run_id, + "scorecard_update_allowed": False, + "tenant_id": tenant_id, + "target_run_id": target_run_id, + "workspace_isolation": { + "egress_paths_isolated": all(_is_safe_relative_path(request.relative_path) for request in egress_requests), + "tenant_id": tenant_id, + "workspace_id": workspace_id, + }, + "hardening_passed": hardening_passed, + "failed_adversarial_packages": failed_adversarial, + } + + +def validate_hardening_report(report: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != HARDENING_REPORT_ID: + errors.append("report_id must be multi-tenant hardening v1") + if report.get("claim_boundary") != HARDENING_CLAIM_BOUNDARY: + errors.append("claim_boundary must be hardening boundary") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if not str(report.get("target_run_id") or ""): + errors.append("target_run_id must be non-empty") + for section in ["workspace_isolation", "artifact_egress_policy", "artifact_egress_results", "redacted_environment", "destructive_action_guards", "adversarial_package_results"]: + if section not in report: + errors.append(f"{section} section must be present") + redacted_environment = report.get("redacted_environment") if isinstance(report.get("redacted_environment"), Mapping) else {} + for key, value in redacted_environment.items(): + lowered_key = str(key).lower() + if any(part in lowered_key for part in _SECRET_KEY_PARTS) and value != REDACTED: + errors.append(f"redacted_environment.{key} must be redacted") + if isinstance(value, str) and re.match(r"^(~|/)", value): + errors.append(f"redacted_environment.{key} must not expose absolute or home path") + return errors + +def report_to_json(report: Mapping[str, Any]) -> str: + return json.dumps(dict(report), sort_keys=True, separators=(",", ":")) + "\n" diff --git a/breadboard/rl/phase2/observability.py b/breadboard/rl/phase2/observability.py new file mode 100644 index 00000000..315b3e8e --- /dev/null +++ b/breadboard/rl/phase2/observability.py @@ -0,0 +1,174 @@ +from __future__ import annotations +from collections.abc import Mapping + +import json + +import statistics +from dataclasses import dataclass +from typing import Any + + +OBSERVABILITY_REPORT_ID = "bb_zyphra_rl_phase2_observability_v1" +OBSERVABILITY_CLAIM_BOUNDARY = "p2_m10_observability_contract_not_target_throughput_claim" + + +@dataclass(frozen=True) +class ObservabilityCaps: + max_budget_usd: float + max_gpu_hours: float + max_queue_wait_seconds: float + max_verifier_latency_ms: float + max_failure_rate: float + + def to_dict(self) -> dict[str, float]: + return { + "max_budget_usd": self.max_budget_usd, + "max_failure_rate": self.max_failure_rate, + "max_gpu_hours": self.max_gpu_hours, + "max_queue_wait_seconds": self.max_queue_wait_seconds, + "max_verifier_latency_ms": self.max_verifier_latency_ms, + } + + +@dataclass(frozen=True) +class ObservabilitySample: + queue_wait_seconds: float + gpu_utilization_percent: float + tasks_completed: int + elapsed_seconds: float + verifier_latency_ms: float + failure_class: str = "none" + + def to_dict(self) -> dict[str, Any]: + return { + "elapsed_seconds": self.elapsed_seconds, + "failure_class": self.failure_class, + "gpu_utilization_percent": self.gpu_utilization_percent, + "queue_wait_seconds": self.queue_wait_seconds, + "tasks_completed": self.tasks_completed, + "verifier_latency_ms": self.verifier_latency_ms, + } + + +def _percentile(values: list[float], percentile: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, int(round(percentile * (len(ordered) - 1)))) + return float(ordered[index]) + + +def _failure_taxonomy(samples: list[ObservabilitySample]) -> dict[str, int]: + taxonomy: dict[str, int] = {} + for sample in samples: + if sample.failure_class and sample.failure_class != "none": + taxonomy[sample.failure_class] = taxonomy.get(sample.failure_class, 0) + 1 + return dict(sorted(taxonomy.items())) + + +def evaluate_observability_caps( + *, + caps: ObservabilityCaps, + requested_budget_usd: float, + projected_gpu_hours: float, + samples: list[ObservabilitySample], +) -> dict[str, Any]: + rejections: list[str] = [] + queue_waits = [sample.queue_wait_seconds for sample in samples] + verifier_latencies = [sample.verifier_latency_ms for sample in samples] + failure_count = sum(1 for sample in samples if sample.failure_class and sample.failure_class != "none") + failure_rate = failure_count / len(samples) if samples else 0.0 + if requested_budget_usd > caps.max_budget_usd: + rejections.append("requested_budget_usd exceeds max_budget_usd") + if projected_gpu_hours > caps.max_gpu_hours: + rejections.append("projected_gpu_hours exceeds max_gpu_hours") + if queue_waits and max(queue_waits) > caps.max_queue_wait_seconds: + rejections.append("queue_wait_seconds exceeds max_queue_wait_seconds") + if verifier_latencies and max(verifier_latencies) > caps.max_verifier_latency_ms: + rejections.append("verifier_latency_ms exceeds max_verifier_latency_ms") + if failure_rate > caps.max_failure_rate: + rejections.append("failure_rate exceeds max_failure_rate") + return { + "accepted": not rejections, + "failure_rate": failure_rate, + "projected_gpu_hours": projected_gpu_hours, + "rejections": rejections, + "requested_budget_usd": requested_budget_usd, + } + + +def build_observability_report( + *, + run_id: str, + target_run_id: str, + caps: ObservabilityCaps, + requested_budget_usd: float, + projected_gpu_hours: float, + samples: list[ObservabilitySample], +) -> dict[str, Any]: + queue_waits = [sample.queue_wait_seconds for sample in samples] + gpu_utils = [sample.gpu_utilization_percent for sample in samples] + verifier_latencies = [sample.verifier_latency_ms for sample in samples] + elapsed_seconds = sum(max(0.0, sample.elapsed_seconds) for sample in samples) + tasks_completed = sum(max(0, sample.tasks_completed) for sample in samples) + cap_evaluation = evaluate_observability_caps( + caps=caps, + requested_budget_usd=requested_budget_usd, + projected_gpu_hours=projected_gpu_hours, + samples=samples, + ) + return { + "budget_caps": caps.to_dict(), + "cap_evaluation": cap_evaluation, + "claim_boundary": OBSERVABILITY_CLAIM_BOUNDARY, + "milestone_id": "P2-M10", + "passed": cap_evaluation["accepted"], + "failure_taxonomy": _failure_taxonomy(samples), + "gpu_utilization": { + "average_percent": float(statistics.fmean(gpu_utils)) if gpu_utils else 0.0, + "peak_percent": max(gpu_utils) if gpu_utils else 0.0, + "sample_count": len(gpu_utils), + }, + "queue_wait": { + "max_seconds": max(queue_waits) if queue_waits else 0.0, + "p50_seconds": _percentile(queue_waits, 0.50), + "p95_seconds": _percentile(queue_waits, 0.95), + }, + "report_id": OBSERVABILITY_REPORT_ID, + "run_id": run_id, + "samples": [sample.to_dict() for sample in samples], + "scorecard_update_allowed": False, + "target_run_id": target_run_id, + "task_throughput": { + "elapsed_seconds": elapsed_seconds, + "tasks_completed": tasks_completed, + "tasks_per_second": (tasks_completed / elapsed_seconds) if elapsed_seconds else 0.0, + }, + "verifier_latency": { + "max_ms": max(verifier_latencies) if verifier_latencies else 0.0, + "p50_ms": _percentile(verifier_latencies, 0.50), + "p95_ms": _percentile(verifier_latencies, 0.95), + }, + } + + +def validate_observability_report(report: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != OBSERVABILITY_REPORT_ID: + errors.append("report_id must be observability v1") + if report.get("claim_boundary") != OBSERVABILITY_CLAIM_BOUNDARY: + errors.append("claim_boundary must be observability boundary") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if not str(report.get("target_run_id") or ""): + errors.append("target_run_id must be non-empty") + cap_evaluation = report.get("cap_evaluation") if isinstance(report.get("cap_evaluation"), Mapping) else {} + if "accepted" not in cap_evaluation: + errors.append("cap_evaluation.accepted must be present") + for section in ["queue_wait", "gpu_utilization", "task_throughput", "verifier_latency", "failure_taxonomy", "budget_caps"]: + if section not in report: + errors.append(f"{section} section must be present") + return errors + +def report_to_json(report: Mapping[str, Any]) -> str: + return json.dumps(dict(report), sort_keys=True, separators=(",", ":")) + "\n" diff --git a/breadboard/rl/phase2/promotion_audit.py b/breadboard/rl/phase2/promotion_audit.py new file mode 100644 index 00000000..9d7062b6 --- /dev/null +++ b/breadboard/rl/phase2/promotion_audit.py @@ -0,0 +1,126 @@ +from __future__ import annotations +from collections.abc import Mapping + +import json +from pathlib import Path +from typing import Any + +from breadboard.rl.phase2.final_report import ( + PHASE2_FINAL_CLAIM_BOUNDARY, + PHASE2_FINAL_REPORT_ID, + validate_phase2_final_report, +) + + +PHASE2_PROMOTION_AUDIT_ID = "bb_zyphra_rl_phase2_promotion_audit_v1" +PHASE2_PROMOTION_CLAIM_BOUNDARY = "phase2_promotion_review_only_not_scorecard_update" + + +def _dict_value(data: Mapping[str, Any], *keys: str) -> Any: + current: Any = data + for key in keys: + if not isinstance(current, Mapping): + return None + current = current.get(key) + return current + +def _int_or_zero(value: Any) -> int: + try: + return int(value or 0) + except (TypeError, ValueError): + return 0 + + +def _text_or_mapping_contains(source: Mapping[str, Any] | str, expected: str) -> bool: + if isinstance(source, str): + return expected in source + encoded = json.dumps(source, sort_keys=True) + return expected in encoded + + +def _ledger_matches(claim_ledger: Mapping[str, Any] | str, *, target_run_id: str, final_report_id: str) -> dict[str, bool]: + return { + "claim_boundary_matches": _text_or_mapping_contains(claim_ledger, PHASE2_FINAL_CLAIM_BOUNDARY), + "final_report_id_matches": _text_or_mapping_contains(claim_ledger, final_report_id), + "target_run_id_matches": _text_or_mapping_contains(claim_ledger, target_run_id), + } + + +def build_phase2_promotion_audit( + *, + target_run_id: str, + scorecard: Mapping[str, Any], + claim_ledger: Mapping[str, Any] | str, + final_report: Mapping[str, Any], +) -> dict[str, Any]: + final_report_errors = validate_phase2_final_report(final_report) + final_report_id = str(final_report.get("report_id") or "") + scorecard_checks = { + "claim_boundary_matches": _text_or_mapping_contains(scorecard, PHASE2_FINAL_CLAIM_BOUNDARY), + "final_report_id_matches": str(scorecard.get("final_report_id") or _dict_value(scorecard, "phase2", "final_report_id") or "") == final_report_id, + "scorecard_update_allowed_false": scorecard.get("scorecard_update_allowed", _dict_value(scorecard, "phase2", "scorecard_update_allowed")) is False, + "target_run_id_matches": str(scorecard.get("target_run_id") or _dict_value(scorecard, "phase2", "target_run_id") or "") == target_run_id, + "total_points_complete": _int_or_zero(scorecard.get("total_points") or _dict_value(scorecard, "phase2", "total_points")) == 1000, + } + ledger_checks = _ledger_matches(claim_ledger, target_run_id=target_run_id, final_report_id=final_report_id) + final_checks = { + "claim_boundary_matches": final_report.get("claim_boundary") == PHASE2_FINAL_CLAIM_BOUNDARY, + "final_report_id_matches": final_report_id == PHASE2_FINAL_REPORT_ID, + "final_report_ready": final_report.get("final_report_ready") is True, + "scorecard_update_allowed_false": final_report.get("scorecard_update_allowed") is False, + "target_run_id_matches": final_report.get("target_run_id") == target_run_id, + "validates": not final_report_errors, + } + missing_requirements = [f"scorecard.{key}" for key, passed in scorecard_checks.items() if not passed] + missing_requirements.extend(f"claim_ledger.{key}" for key, passed in ledger_checks.items() if not passed) + missing_requirements.extend(f"final_report.{key}" for key, passed in final_checks.items() if not passed) + return { + "checks": { + "claim_ledger": ledger_checks, + "final_report": final_checks, + "scorecard": scorecard_checks, + }, + "claim_boundary": PHASE2_PROMOTION_CLAIM_BOUNDARY, + "final_report_errors": final_report_errors, + "missing_requirements": missing_requirements, + "promotion_review_ready": not missing_requirements, + "report_id": PHASE2_PROMOTION_AUDIT_ID, + "scorecard_update_allowed": False, + "target_run_id": target_run_id, + } + + +def validate_phase2_promotion_audit(audit: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if audit.get("report_id") != PHASE2_PROMOTION_AUDIT_ID: + errors.append("report_id must be phase2 promotion audit v1") + if audit.get("claim_boundary") != PHASE2_PROMOTION_CLAIM_BOUNDARY: + errors.append("claim_boundary must be promotion review boundary") + if audit.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if not str(audit.get("target_run_id") or ""): + errors.append("target_run_id must be non-empty") + if audit.get("missing_requirements"): + errors.append("missing_requirements must be empty") + if audit.get("promotion_review_ready") is not True: + errors.append("promotion_review_ready must be true") + return errors + + +def write_phase2_promotion_audit( + path: Path, + *, + target_run_id: str, + scorecard: Mapping[str, Any], + claim_ledger: Mapping[str, Any] | str, + final_report: Mapping[str, Any], +) -> dict[str, Any]: + audit = build_phase2_promotion_audit( + target_run_id=target_run_id, + scorecard=scorecard, + claim_ledger=claim_ledger, + final_report=final_report, + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return audit diff --git a/breadboard/rl/phase2/scale.py b/breadboard/rl/phase2/scale.py new file mode 100644 index 00000000..4dbbd9e6 --- /dev/null +++ b/breadboard/rl/phase2/scale.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + + +CLAIM_BOUNDARY = "p2_m4_scale_ladder_not_arbitrary_production_scale" +SCALE_LEVELS = (100, 250, 500, 1000) +FAILURE_TAXONOMY = ( + "scheduler", + "verifier", + "replay", + "trainer_handoff", + "infrastructure", +) + + +@dataclass(frozen=True) +class ScaleLevelMetrics: + level: int + attempted: int + completed: int + quarantined: int + retry_count: int + p95_latency_seconds: float + throughput_tasks_per_minute: float + scheduler_failures: int = 0 + verifier_failures: int = 0 + replay_failures: int = 0 + trainer_handoff_failures: int = 0 + infrastructure_failures: int = 0 + + def to_dict(self) -> dict[str, Any]: + return { + "level": self.level, + "attempted": self.attempted, + "completed": self.completed, + "quarantined": self.quarantined, + "retry_count": self.retry_count, + "p95_latency_seconds": self.p95_latency_seconds, + "throughput_tasks_per_minute": self.throughput_tasks_per_minute, + "scheduler_failures": self.scheduler_failures, + "verifier_failures": self.verifier_failures, + "replay_failures": self.replay_failures, + "trainer_handoff_failures": self.trainer_handoff_failures, + "infrastructure_failures": self.infrastructure_failures, + } + + +@dataclass(frozen=True) +class ScaleLevelGate: + level: int + gate: str + passed: bool + observed: str + + def to_dict(self) -> dict[str, Any]: + return { + "level": self.level, + "gate": self.gate, + "passed": self.passed, + "observed": self.observed, + } + + +@dataclass(frozen=True) +class ScaleLadderReport: + report_id: str + target_run_id: str + levels: list[ScaleLevelMetrics] + gates: list[ScaleLevelGate] + failure_taxonomy: dict[str, int] + claim_boundary: str = CLAIM_BOUNDARY + scorecard_update_allowed: bool = False + metadata: dict[str, Any] = field(default_factory=dict) + + @property + def overall_passed(self) -> bool: + return all(gate.passed for gate in self.gates) + + def to_dict(self) -> dict[str, Any]: + return { + "report_id": self.report_id, + "target_run_id": self.target_run_id, + "claim_boundary": self.claim_boundary, + "scorecard_update_allowed": self.scorecard_update_allowed, + "passed": self.overall_passed, + "overall_passed": self.overall_passed, + "levels": [level.to_dict() for level in self.levels], + "gates": [gate.to_dict() for gate in self.gates], + "failure_taxonomy": dict(self.failure_taxonomy), + "metadata": dict(self.metadata), + } + + +def _metric_from_mapping(level: int, data: Mapping[str, Any]) -> ScaleLevelMetrics: + return ScaleLevelMetrics( + level=level, + attempted=int(data.get("attempted", level)), + completed=int(data.get("completed", 0)), + quarantined=int(data.get("quarantined", 0)), + retry_count=int(data.get("retry_count", 0)), + p95_latency_seconds=float(data.get("p95_latency_seconds", 0.0)), + throughput_tasks_per_minute=float(data.get("throughput_tasks_per_minute", 0.0)), + scheduler_failures=int(data.get("scheduler_failures", 0)), + verifier_failures=int(data.get("verifier_failures", 0)), + replay_failures=int(data.get("replay_failures", 0)), + trainer_handoff_failures=int(data.get("trainer_handoff_failures", 0)), + infrastructure_failures=int(data.get("infrastructure_failures", 0)), + ) + + +def build_scale_level_gates(metrics: ScaleLevelMetrics) -> list[ScaleLevelGate]: + closed = metrics.completed + metrics.quarantined + return [ + ScaleLevelGate( + level=metrics.level, + gate="level_size_exact", + passed=metrics.level == metrics.attempted, + observed=f"attempted={metrics.attempted}", + ), + ScaleLevelGate( + level=metrics.level, + gate="all_attempts_accounted", + passed=closed == metrics.attempted, + observed=f"completed_plus_quarantined={closed}", + ), + ScaleLevelGate( + level=metrics.level, + gate="scheduler_resilience", + passed=metrics.scheduler_failures == 0 and metrics.infrastructure_failures == 0, + observed=f"scheduler={metrics.scheduler_failures},infrastructure={metrics.infrastructure_failures}", + ), + ScaleLevelGate( + level=metrics.level, + gate="replay_closure", + passed=metrics.replay_failures == 0, + observed=f"replay_failures={metrics.replay_failures}", + ), + ScaleLevelGate( + level=metrics.level, + gate="trainer_handoff_integrity", + passed=metrics.trainer_handoff_failures == 0, + observed=f"trainer_handoff_failures={metrics.trainer_handoff_failures}", + ), + ScaleLevelGate( + level=metrics.level, + gate="positive_resource_metrics", + passed=metrics.p95_latency_seconds > 0.0 and metrics.throughput_tasks_per_minute > 0.0, + observed=( + f"p95_latency_seconds={metrics.p95_latency_seconds}," + f"throughput_tasks_per_minute={metrics.throughput_tasks_per_minute}" + ), + ), + ] + + +def build_scale_ladder_v2_report( + fixture_metrics: Mapping[int, Mapping[str, Any]], + *, + target_run_id: str = "fixture-phase2-scale-ladder", +) -> ScaleLadderReport: + levels = [_metric_from_mapping(level, fixture_metrics.get(level, {})) for level in SCALE_LEVELS] + gates: list[ScaleLevelGate] = [] + for level_metrics in levels: + gates.extend(build_scale_level_gates(level_metrics)) + failure_taxonomy = { + "scheduler": sum(level.scheduler_failures for level in levels), + "verifier": sum(level.verifier_failures for level in levels), + "replay": sum(level.replay_failures for level in levels), + "trainer_handoff": sum(level.trainer_handoff_failures for level in levels), + "infrastructure": sum(level.infrastructure_failures for level in levels), + } + return ScaleLadderReport( + report_id="bb_zyphra_rl_phase2_scale_ladder_v2", + target_run_id=target_run_id, + levels=levels, + gates=gates, + failure_taxonomy=failure_taxonomy, + metadata={ + "fixture_scope": "deterministic_metrics_only", + "levels": list(SCALE_LEVELS), + "failure_taxonomy_keys": list(FAILURE_TAXONOMY), + }, + ) + + +def fixture_scale_metrics() -> dict[int, dict[str, Any]]: + return { + 100: { + "attempted": 100, + "completed": 98, + "quarantined": 2, + "retry_count": 3, + "p95_latency_seconds": 12.5, + "throughput_tasks_per_minute": 24.0, + "verifier_failures": 2, + }, + 250: { + "attempted": 250, + "completed": 244, + "quarantined": 6, + "retry_count": 8, + "p95_latency_seconds": 18.0, + "throughput_tasks_per_minute": 35.0, + "verifier_failures": 6, + }, + 500: { + "attempted": 500, + "completed": 487, + "quarantined": 13, + "retry_count": 17, + "p95_latency_seconds": 26.0, + "throughput_tasks_per_minute": 44.0, + "verifier_failures": 13, + }, + 1000: { + "attempted": 1000, + "completed": 971, + "quarantined": 29, + "retry_count": 41, + "p95_latency_seconds": 41.0, + "throughput_tasks_per_minute": 52.0, + "verifier_failures": 29, + }, + } diff --git a/breadboard/rl/phase2/service.py b/breadboard/rl/phase2/service.py new file mode 100644 index 00000000..d857c086 --- /dev/null +++ b/breadboard/rl/phase2/service.py @@ -0,0 +1,333 @@ +from __future__ import annotations +from collections.abc import Mapping + +import json + +from dataclasses import dataclass, field +from typing import Any + + +SERVICE_CLAIM_BOUNDARY = "p2_m9_service_contract_not_live_fastapi_integration" +SERVICE_REPORT_ID = "bb_zyphra_rl_phase2_service_surface_v1" +TERMINAL_STATES = {"succeeded", "failed", "cancelled", "rejected"} +ACTIVE_STATES = {"queued", "running", "cancel_requested"} + + +@dataclass(frozen=True) +class ResourceCaps: + max_tasks: int + max_gpus: int + max_budget_usd: float + max_duration_seconds: int + max_artifact_bytes: int + + def to_dict(self) -> dict[str, Any]: + return { + "max_artifact_bytes": self.max_artifact_bytes, + "max_budget_usd": self.max_budget_usd, + "max_duration_seconds": self.max_duration_seconds, + "max_gpus": self.max_gpus, + "max_tasks": self.max_tasks, + } + + +@dataclass(frozen=True) +class RunSubmission: + run_id: str + tenant_id: str + workspace_id: str + env_package_ref: str + target_run_id: str + requested_tasks: int + requested_gpus: int + requested_budget_usd: float + requested_duration_seconds: int + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "env_package_ref": self.env_package_ref, + "metadata": dict(self.metadata), + "requested_budget_usd": self.requested_budget_usd, + "requested_duration_seconds": self.requested_duration_seconds, + "requested_gpus": self.requested_gpus, + "requested_tasks": self.requested_tasks, + "run_id": self.run_id, + "target_run_id": self.target_run_id, + "tenant_id": self.tenant_id, + "workspace_id": self.workspace_id, + } + + +@dataclass(frozen=True) +class StreamEvent: + sequence: int + run_id: str + event_type: str + state: str + message: str + target_run_id: str + payload: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "event_type": self.event_type, + "message": self.message, + "payload": dict(self.payload), + "run_id": self.run_id, + "sequence": self.sequence, + "state": self.state, + "target_run_id": self.target_run_id, + } + + +@dataclass(frozen=True) +class RunStatus: + run_id: str + state: str + target_run_id: str + accepted: bool + cancellation_state: str + reason: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "accepted": self.accepted, + "cancellation_state": self.cancellation_state, + "reason": self.reason, + "run_id": self.run_id, + "state": self.state, + "target_run_id": self.target_run_id, + } + + +@dataclass(frozen=True) +class ArtifactRecord: + run_id: str + artifact_id: str + relative_path: str + sha256: str + bytes: int + egress_allowed: bool + + def to_dict(self) -> dict[str, Any]: + return { + "artifact_id": self.artifact_id, + "bytes": self.bytes, + "egress_allowed": self.egress_allowed, + "relative_path": self.relative_path, + "run_id": self.run_id, + "sha256": self.sha256, + } + + +@dataclass +class _RunRecord: + submission: RunSubmission + state: str + accepted: bool + cancellation_state: str = "not_cancelled" + reason: str = "" + artifacts: list[ArtifactRecord] = field(default_factory=list) + events: list[StreamEvent] = field(default_factory=list) + + def status(self) -> RunStatus: + return RunStatus( + run_id=self.submission.run_id, + state=self.state, + target_run_id=self.submission.target_run_id, + accepted=self.accepted, + cancellation_state=self.cancellation_state, + reason=self.reason, + ) + + +class RLRunServiceContract: + """In-memory product-facing contract model for Phase 2 run operations. + + The class intentionally models submit/status/cancel/collect/replay/audit + semantics without binding the repository to a concrete FastAPI server. + """ + + def __init__(self, caps: ResourceCaps) -> None: + self._caps = caps + self._runs: dict[str, _RunRecord] = {} + + @property + def caps(self) -> ResourceCaps: + return self._caps + + def submit(self, submission: RunSubmission) -> RunStatus: + if submission.run_id in self._runs: + raise ValueError(f"duplicate run_id: {submission.run_id}") + rejections = resource_cap_rejections(submission, self._caps) + record = _RunRecord( + submission=submission, + state="rejected" if rejections else "queued", + accepted=not rejections, + reason="; ".join(rejections), + ) + self._append_event(record, "run_rejected" if rejections else "run_submitted", record.reason or "run submitted", {}) + self._runs[submission.run_id] = record + return record.status() + + def status(self, run_id: str) -> RunStatus: + return self._record(run_id).status() + + def start(self, run_id: str) -> RunStatus: + record = self._record(run_id) + if record.state != "queued": + raise ValueError(f"run {run_id} cannot start from state {record.state}") + record.state = "running" + self._append_event(record, "run_started", "run started", {}) + return record.status() + + def complete(self, run_id: str, *, succeeded: bool, reason: str = "") -> RunStatus: + record = self._record(run_id) + if record.state not in ACTIVE_STATES: + raise ValueError(f"run {run_id} cannot complete from state {record.state}") + record.state = "succeeded" if succeeded else "failed" + record.reason = reason + self._append_event(record, "run_completed", reason or record.state, {"succeeded": succeeded}) + return record.status() + + def cancel(self, run_id: str, *, operator_id: str, reason: str) -> RunStatus: + record = self._record(run_id) + if record.state in TERMINAL_STATES: + raise ValueError(f"run {run_id} is already terminal: {record.state}") + record.state = "cancel_requested" + record.cancellation_state = "operator_requested" + record.reason = reason + self._append_event(record, "cancel_requested", reason, {"operator_id": operator_id}) + return record.status() + + def acknowledge_cancelled(self, run_id: str) -> RunStatus: + record = self._record(run_id) + if record.state != "cancel_requested": + raise ValueError(f"run {run_id} has no pending cancellation") + record.state = "cancelled" + record.cancellation_state = "cancelled" + self._append_event(record, "run_cancelled", record.reason or "run cancelled", {}) + return record.status() + + def add_artifact(self, artifact: ArtifactRecord) -> None: + record = self._record(artifact.run_id) + if artifact.bytes > self._caps.max_artifact_bytes: + raise ValueError("artifact exceeds max_artifact_bytes") + record.artifacts.append(artifact) + self._append_event(record, "artifact_recorded", artifact.relative_path, {"artifact_id": artifact.artifact_id}) + + def collect(self, run_id: str) -> dict[str, Any]: + record = self._record(run_id) + return { + "artifacts": [artifact.to_dict() for artifact in sorted(record.artifacts, key=lambda item: item.artifact_id)], + "claim_boundary": SERVICE_CLAIM_BOUNDARY, + "run_id": run_id, + "scorecard_update_allowed": False, + "target_run_id": record.submission.target_run_id, + } + + def stream(self, run_id: str) -> list[dict[str, Any]]: + record = self._record(run_id) + return [event.to_dict() for event in record.events] + + def replay(self, run_id: str, *, artifact_id: str) -> dict[str, Any]: + record = self._record(run_id) + artifact_ids = {artifact.artifact_id for artifact in record.artifacts} + return { + "artifact_id": artifact_id, + "claim_boundary": SERVICE_CLAIM_BOUNDARY, + "replay_available": artifact_id in artifact_ids, + "run_id": run_id, + "scorecard_update_allowed": False, + "target_run_id": record.submission.target_run_id, + } + + def audit(self, run_id: str) -> dict[str, Any]: + record = self._record(run_id) + return build_service_surface_report(record.submission, self._caps, record.status(), record.events) + + def _record(self, run_id: str) -> _RunRecord: + try: + return self._runs[run_id] + except KeyError as exc: + raise KeyError(f"unknown run_id: {run_id}") from exc + + def _append_event(self, record: _RunRecord, event_type: str, message: str, payload: Mapping[str, Any]) -> None: + record.events.append( + StreamEvent( + sequence=len(record.events) + 1, + run_id=record.submission.run_id, + event_type=event_type, + state=record.state, + message=message, + target_run_id=record.submission.target_run_id, + payload=dict(payload), + ) + ) + + +def resource_cap_rejections(submission: RunSubmission, caps: ResourceCaps) -> list[str]: + rejections: list[str] = [] + if submission.requested_tasks > caps.max_tasks: + rejections.append("requested_tasks exceeds max_tasks") + if submission.requested_gpus > caps.max_gpus: + rejections.append("requested_gpus exceeds max_gpus") + if submission.requested_budget_usd > caps.max_budget_usd: + rejections.append("requested_budget_usd exceeds max_budget_usd") + if submission.requested_duration_seconds > caps.max_duration_seconds: + rejections.append("requested_duration_seconds exceeds max_duration_seconds") + return rejections + + +def build_service_surface_report( + submission: RunSubmission, + caps: ResourceCaps, + status: RunStatus, + events: list[StreamEvent], +) -> dict[str, Any]: + return { + "api_contract": { + "audit": "audit(run_id)", + "cancel": "cancel(run_id, operator_id, reason)", + "collect": "collect(run_id)", + "replay": "replay(run_id, artifact_id)", + "status": "status(run_id)", + "stream": "stream(run_id)", + "submit": "submit(RunSubmission)", + }, + "cancellation_states": ["not_cancelled", "operator_requested", "cancelled"], + "claim_boundary": SERVICE_CLAIM_BOUNDARY, + "milestone_id": "P2-M9", + "passed": status.accepted, + "report_id": SERVICE_REPORT_ID, + "resource_caps": caps.to_dict(), + "run_states": ["queued", "running", "cancel_requested", "cancelled", "succeeded", "failed", "rejected"], + "scorecard_update_allowed": False, + "status": status.to_dict(), + "submission": submission.to_dict(), + "stream_events": [event.to_dict() for event in events], + "target_run_id": submission.target_run_id, + } + + +def validate_service_surface_report(report: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("report_id") != SERVICE_REPORT_ID: + errors.append("report_id must be service surface v1") + if report.get("claim_boundary") != SERVICE_CLAIM_BOUNDARY: + errors.append("claim_boundary must be service contract boundary") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if not str(report.get("target_run_id") or ""): + errors.append("target_run_id must be non-empty") + for name in ["submit", "status", "cancel", "collect", "replay", "audit", "stream"]: + if name not in dict(report.get("api_contract") or {}): + errors.append(f"api_contract missing {name}") + sequences = [event.get("sequence") for event in report.get("stream_events") or [] if isinstance(event, Mapping)] + if sequences != list(range(1, len(sequences) + 1)): + errors.append("stream_events must have contiguous sequence numbers") + return errors + +def report_to_json(report: Mapping[str, Any]) -> str: + return json.dumps(dict(report), sort_keys=True, separators=(",", ":")) + "\n" diff --git a/breadboard/rl/phase2/trainer.py b/breadboard/rl/phase2/trainer.py new file mode 100644 index 00000000..5e7ab416 --- /dev/null +++ b/breadboard/rl/phase2/trainer.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import importlib.util +import json +import platform +from typing import Any, Mapping + +from breadboard.rl.phase2.bridge import VERL_BATCH_CLAIM_BOUNDARY, VerlBatch + + +VERL_TRAINER_DRY_RUN_SCHEMA_VERSION = "bb.rl.phase2.verl_trainer_dry_run.v1alpha" +VERL_TRAINER_DRY_RUN_CLAIM_BOUNDARY = "phase2_trainer_dry_run_no_slurm_no_weight_update" +__all__ = [ + "VERL_TRAINER_DRY_RUN_CLAIM_BOUNDARY", + "VERL_TRAINER_DRY_RUN_SCHEMA_VERSION", + "build_trainer_dry_run_report", + "report_to_json", + "validate_trainer_dry_run_batch", +] + +_ALLOWED_DRY_RUN_MODES = {"no_update", "one_step"} +_PROVENANCE_FIELDS = { + "rollout_id", + "trajectory_id", + "episode_id", + "task_id", + "projection_manifest_id", + "policy_snapshot_id", +} + + +def build_trainer_dry_run_report( + batch: VerlBatch | Mapping[str, Any], + *, + target_run_id: str, + mode: str = "no_update", + device: str = "cpu", +) -> dict[str, Any]: + payload = batch.to_dict() if isinstance(batch, VerlBatch) else dict(batch) + target_run_id = _require_text(target_run_id, "target_run_id") + mode = _require_text(mode, "mode") + device = _require_text(device, "device") + if mode not in _ALLOWED_DRY_RUN_MODES: + raise ValueError("mode must be no_update or one_step") + + errors = validate_trainer_dry_run_batch(payload, target_run_id=target_run_id) + if errors: + raise ValueError("; ".join(errors)) + + row_count = int(payload["row_count"]) + tensor_shape_metadata = payload["tensor_shape_metadata"] + report = { + "schema_version": VERL_TRAINER_DRY_RUN_SCHEMA_VERSION, + "claim_boundary": VERL_TRAINER_DRY_RUN_CLAIM_BOUNDARY, + "scorecard_update_allowed": False, + "target_run_id": target_run_id, + "batch_id": payload["batch_id"], + "policy_snapshot_id": payload["policy_snapshot_id"], + "mode": mode, + "row_count": row_count, + "dry_run_result": { + "accepted": True, + "optimizer_step_performed": False, + "weight_update_performed": False, + "slurm_execution_performed": False, + }, + "device_evidence": _device_evidence(device), + "batch_evidence": { + "input_ids_shape": tensor_shape_metadata["input_ids"]["shape"], + "completion_logprobs_shape": tensor_shape_metadata["completion_logprobs"]["shape"], + "sequence_rewards_shape": tensor_shape_metadata["sequence_rewards"]["shape"], + "has_reward_mask": "reward_mask" in payload["masks"], + "has_completion_logprob_mask": "completion_logprob_mask" in payload["masks"], + }, + } + if mode == "one_step": + report["dry_run_result"]["planned_step_count"] = 1 + else: + report["dry_run_result"]["planned_step_count"] = 0 + return report + + +def validate_trainer_dry_run_batch(payload: Mapping[str, Any], *, target_run_id: str) -> list[str]: + errors: list[str] = [] + if payload.get("claim_boundary") != VERL_BATCH_CLAIM_BOUNDARY: + errors.append("batch claim_boundary is not a phase2 VeRL dry-run batch") + if payload.get("scorecard_update_allowed") is not False: + errors.append("batch scorecard_update_allowed must be false") + if payload.get("target_run_id") != target_run_id: + errors.append("batch target_run_id must match trainer target_run_id") + if not str(payload.get("policy_snapshot_id") or "").strip(): + errors.append("batch policy_snapshot_id must be present") + + field_ledger = payload.get("field_ledger") + if not isinstance(field_ledger, Mapping): + errors.append("batch field_ledger must be present") + else: + lost_fields = set(field_ledger.get("lost_fields") or []) + critical_lost_fields = set(field_ledger.get("critical_lost_fields") or []) + critical_lost_fields.update(lost_fields.intersection(_PROVENANCE_FIELDS)) + if critical_lost_fields or field_ledger.get("provenance_loss_detected") is True: + errors.append("trainer dry-run rejects provenance loss") + + try: + row_count = int(payload.get("row_count")) + except (TypeError, ValueError): + errors.append("batch row_count must be an integer") + row_count = 0 + if row_count <= 0: + errors.append("batch row_count must be positive") + + tensors = payload.get("tensors") + masks = payload.get("masks") + logprobs = payload.get("logprobs") + rewards = payload.get("rewards") + shapes = payload.get("tensor_shape_metadata") + if not isinstance(tensors, Mapping): + errors.append("batch tensors must be present") + tensors = {} + if not isinstance(masks, Mapping): + errors.append("batch masks must be present") + masks = {} + if not isinstance(logprobs, Mapping): + errors.append("batch logprobs must be present") + logprobs = {} + if not isinstance(rewards, Mapping): + errors.append("batch rewards must be present") + rewards = {} + if not isinstance(shapes, Mapping): + errors.append("batch tensor_shape_metadata must be present") + shapes = {} + + _validate_matrix_shape(tensors.get("input_ids"), shapes.get("input_ids"), row_count, "input_ids", errors) + _validate_matrix_shape(tensors.get("attention_mask"), shapes.get("attention_mask"), row_count, "attention_mask", errors) + for mask_name in ( + "loss_mask", + "assistant_mask", + "tool_action_mask", + "reward_mask", + "completion_logprob_mask", + ): + _validate_matrix_shape(masks.get(mask_name), shapes.get(mask_name), row_count, mask_name, errors) + _validate_matrix_shape( + logprobs.get("completion_logprobs"), + shapes.get("completion_logprobs"), + row_count, + "completion_logprobs", + errors, + ) + _validate_matrix_shape( + rewards.get("token_rewards"), + shapes.get("token_rewards"), + row_count, + "token_rewards", + errors, + ) + _validate_vector_shape( + rewards.get("sequence_rewards"), + shapes.get("sequence_rewards"), + row_count, + "sequence_rewards", + errors, + ) + return errors + + +def report_to_json(report: Mapping[str, Any]) -> str: + return json.dumps(dict(report), sort_keys=True, separators=(",", ":")) + "\n" + + +def _validate_matrix_shape( + matrix: Any, + shape_metadata: Any, + row_count: int, + field_name: str, + errors: list[str], +) -> None: + if not isinstance(matrix, list): + errors.append(f"batch {field_name} must be a matrix") + return + if not isinstance(shape_metadata, Mapping): + errors.append(f"batch {field_name} shape metadata must be present") + return + shape = shape_metadata.get("shape") + if not isinstance(shape, list) or len(shape) != 2: + errors.append(f"batch {field_name} shape must be rank 2") + return + if shape[0] != row_count or len(matrix) != row_count: + errors.append(f"batch {field_name} row dimension must match row_count") + width = shape[1] + for row_index, row in enumerate(matrix, start=1): + if not isinstance(row, list) or len(row) != width: + errors.append(f"batch {field_name} row {row_index} width must match shape") + return + + +def _validate_vector_shape( + vector: Any, + shape_metadata: Any, + row_count: int, + field_name: str, + errors: list[str], +) -> None: + if not isinstance(vector, list): + errors.append(f"batch {field_name} must be a vector") + return + if not isinstance(shape_metadata, Mapping): + errors.append(f"batch {field_name} shape metadata must be present") + return + shape = shape_metadata.get("shape") + if shape != [row_count]: + errors.append(f"batch {field_name} shape must match row_count") + if len(vector) != row_count: + errors.append(f"batch {field_name} length must match row_count") + + +def _device_evidence(device: str) -> dict[str, Any]: + torch_spec = importlib.util.find_spec("torch") + return { + "requested_device": device, + "resolved_device": "cpu" if device == "cpu" else device, + "torch_importable": torch_spec is not None, + "python_implementation": platform.python_implementation(), + "machine": platform.machine(), + "execution_backend": "dry_run_only_no_trainer_or_slurm", + } + + +def _require_text(value: Any, field_name: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError(f"{field_name} must be non-empty") + return text diff --git a/breadboard/rl/phase2/verifier.py b/breadboard/rl/phase2/verifier.py new file mode 100644 index 00000000..7730ac66 --- /dev/null +++ b/breadboard/rl/phase2/verifier.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Sequence + + +CLAIM_BOUNDARY = "p2_m6_live_verifier_probe_not_general_verifier_claim" + + +@dataclass(frozen=True) +class VerifierCallEvidence: + call_id: str + provider: str + endpoint_id: str + verifier_version: str + request_hash: str + response_hash: str + reward_scalar: float + latency_ms: int + error: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "call_id": self.call_id, + "provider": self.provider, + "endpoint_id": self.endpoint_id, + "verifier_version": self.verifier_version, + "request_hash": self.request_hash, + "response_hash": self.response_hash, + "reward_scalar": self.reward_scalar, + "latency_ms": self.latency_ms, + "error": self.error, + } + + +@dataclass(frozen=True) +class LiveVerifierIntegrationReport: + report_id: str + target_run_id: str + calls: list[VerifierCallEvidence] + baseline_reward: float + drift_tolerance: float + max_abs_drift: float + status: str + quarantine_reasons: list[str] + preserved_fields: list[str] + lost_fields: list[str] + claim_boundary: str = CLAIM_BOUNDARY + scorecard_update_allowed: bool = False + metadata: dict[str, Any] = field(default_factory=dict) + + @property + def quarantined(self) -> bool: + return self.status == "quarantined_verifier_instability" + + def to_dict(self) -> dict[str, Any]: + return { + "report_id": self.report_id, + "target_run_id": self.target_run_id, + "claim_boundary": self.claim_boundary, + "scorecard_update_allowed": self.scorecard_update_allowed, + "passed": not self.quarantined, + "baseline_reward": self.baseline_reward, + "drift_tolerance": self.drift_tolerance, + "max_abs_drift": self.max_abs_drift, + "status": self.status, + "quarantined": self.quarantined, + "quarantine_reasons": list(self.quarantine_reasons), + "preserved_fields": list(self.preserved_fields), + "lost_fields": list(self.lost_fields), + "calls": [call.to_dict() for call in self.calls], + "metadata": dict(self.metadata), + } + + +def build_live_verifier_integration_report( + calls: Sequence[VerifierCallEvidence], + *, + baseline_reward: float, + drift_tolerance: float = 0.05, + target_run_id: str = "fixture-phase2-live-verifier", +) -> LiveVerifierIntegrationReport: + call_list = list(calls) + quarantine_reasons: list[str] = [] + if not call_list: + quarantine_reasons.append("missing_verifier_calls") + max_abs_drift = 0.0 + else: + max_abs_drift = round(max(abs(call.reward_scalar - baseline_reward) for call in call_list), 12) + + providers = {call.provider for call in call_list} + versions = {call.verifier_version for call in call_list} + request_hashes = {call.request_hash for call in call_list} + errored_calls = [call.call_id for call in call_list if call.error] + + if errored_calls: + quarantine_reasons.append("verifier_call_error:" + ",".join(sorted(errored_calls))) + if max_abs_drift > drift_tolerance: + quarantine_reasons.append("reward_drift_exceeded") + if len(versions) > 1: + quarantine_reasons.append("verifier_version_drift") + if len(request_hashes) > 1: + quarantine_reasons.append("request_hash_drift") + if not providers.issubset({"ors", "openreward"}): + quarantine_reasons.append("unsupported_verifier_provider") + + status = "quarantined_verifier_instability" if quarantine_reasons else "live_verifier_fixture_stable" + return LiveVerifierIntegrationReport( + report_id="bb_zyphra_rl_phase2_live_verifier_v1", + target_run_id=target_run_id, + calls=call_list, + baseline_reward=baseline_reward, + drift_tolerance=drift_tolerance, + max_abs_drift=max_abs_drift, + status=status, + quarantine_reasons=quarantine_reasons, + preserved_fields=[ + "provider", + "endpoint_id", + "verifier_version", + "request_hash", + "response_hash", + "reward_scalar", + "latency_ms", + ], + lost_fields=[ + "remote_model_weights", + "provider_internal_trace", + ], + metadata={ + "fixture_scope": "ors_openreward_call_ledger", + "quarantine_policy": "fail_closed_on_error_drift_or_version_change", + }, + ) + + +def stable_fixture_verifier_calls() -> list[VerifierCallEvidence]: + return [ + VerifierCallEvidence( + call_id="ors-call-1", + provider="ors", + endpoint_id="ors.fixture.local", + verifier_version="openreward-fixture-v1", + request_hash="4" * 64, + response_hash="5" * 64, + reward_scalar=0.75, + latency_ms=42, + ), + VerifierCallEvidence( + call_id="openreward-call-1", + provider="openreward", + endpoint_id="openreward.fixture.local", + verifier_version="openreward-fixture-v1", + request_hash="4" * 64, + response_hash="6" * 64, + reward_scalar=0.76, + latency_ms=45, + ), + ] diff --git a/breadboard/rl/phase3/__init__.py b/breadboard/rl/phase3/__init__.py new file mode 100644 index 00000000..ae86e35a --- /dev/null +++ b/breadboard/rl/phase3/__init__.py @@ -0,0 +1,19 @@ +"""Phase 3 production-readiness gates and live RL surfaces.""" + +from .evidence import ( + PHASE3_COMMAND_LOG_MANIFEST_SCHEMA, + PHASE3_COMPONENT_REPORT_SCHEMA, + PHASE3_TARGET_RUN_ID_PATTERN, + REQUIRED_COMMAND_LOG_FIELDS, + validate_phase3_command_log_manifest, + validate_phase3_component_report, +) + +__all__ = [ + "PHASE3_COMMAND_LOG_MANIFEST_SCHEMA", + "PHASE3_COMPONENT_REPORT_SCHEMA", + "PHASE3_TARGET_RUN_ID_PATTERN", + "REQUIRED_COMMAND_LOG_FIELDS", + "validate_phase3_command_log_manifest", + "validate_phase3_component_report", +] diff --git a/breadboard/rl/phase3/api_models.py b/breadboard/rl/phase3/api_models.py new file mode 100644 index 00000000..8c90c556 --- /dev/null +++ b/breadboard/rl/phase3/api_models.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +class RLResourceCapsModel(BaseModel): + max_tasks: int + max_gpus: int + max_budget_usd: float + max_duration_seconds: int + max_artifact_bytes: int + + +class RLRunSubmitRequest(BaseModel): + run_id: str + tenant_id: str + workspace_id: str + env_package_ref: str + target_run_id: str + requested_tasks: int = Field(default=1, gt=0) + requested_gpus: int = Field(default=1, gt=0) + requested_budget_usd: float = Field(default=1.0, gt=0.0) + requested_duration_seconds: int = Field(default=60, gt=0) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class RLRunSubmitResponse(BaseModel): + run_id: str + state: str + target_run_id: str + accepted: bool + cancellation_state: str + reason: str = "" + + +class RLRunStatusResponse(RLRunSubmitResponse): + pass + + +class RLRunCancelRequest(BaseModel): + tenant_id: str + workspace_id: str + reason: str = "cancel requested" + + +class RlArtifactModel(BaseModel): + artifact_id: str + relative_path: str + sha256: str + bytes: int + egress_allowed: bool + + +class RLRunArtifactListResponse(BaseModel): + run_id: str + artifacts: list[RlArtifactModel] + + +class RLRunReplayResponse(BaseModel): + available: bool + artifact_id: str + replay_path: str | None = None + sha256: str | None = None + reason: str | None = None + + +class RLRunAuditResponse(BaseModel): + run_id: str + tenant_id: str + workspace_id: str + target_run_id: str + state: str + persistent_store: str + scorecard_update_allowed: bool + + + +class RLProjectionEventModel(BaseModel): + """Projection-only RL run event. + + This is a host-facing `/rl/runs/{run_id}/events` convenience envelope. It is + not the canonical kernel event envelope and must not be used as replay or + conformance truth. + """ + + +class RLStreamEventModel(RLProjectionEventModel): + sequence: int + run_id: str + event_type: str + state: str + message: str + target_run_id: str + payload: dict[str, Any] = Field(default_factory=dict) diff --git a/breadboard/rl/phase3/api_router.py b/breadboard/rl/phase3/api_router.py new file mode 100644 index 00000000..fa96a2cc --- /dev/null +++ b/breadboard/rl/phase3/api_router.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import json + +from fastapi import APIRouter, Header, HTTPException, Query +from fastapi.responses import PlainTextResponse + +from breadboard.rl.phase2.service import ArtifactRecord, RunSubmission +from breadboard.rl.phase3.api_models import ( + RLRunArtifactListResponse, + RLRunAuditResponse, + RLRunCancelRequest, + RLRunReplayResponse, + RLRunStatusResponse, + RLRunSubmitRequest, + RLRunSubmitResponse, + RLStreamEventModel, +) +from breadboard.rl.phase3.service_live import LiveRLRunService + + +def _tenant(headers_tenant: str | None, query_tenant: str | None) -> str: + tenant = query_tenant or headers_tenant + if not tenant: + raise HTTPException(status_code=400, detail="tenant_id is required") + return tenant + + +def _workspace(headers_workspace: str | None, query_workspace: str | None) -> str: + workspace = query_workspace or headers_workspace + if not workspace: + raise HTTPException(status_code=400, detail="workspace_id is required") + return workspace + + +def _status_response(status) -> RLRunStatusResponse: # type: ignore[no-untyped-def] + return RLRunStatusResponse(**status.to_dict()) + + +def create_phase3_rl_router(rl_service: LiveRLRunService | None = None) -> APIRouter: + service = rl_service or LiveRLRunService() + router = APIRouter() + + @router.post("/runs", response_model=RLRunSubmitResponse) + def submit_run(payload: RLRunSubmitRequest) -> RLRunSubmitResponse: + try: + status = service.submit(RunSubmission(**payload.dict())) + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + return RLRunSubmitResponse(**status.to_dict()) + + @router.get("/runs/{run_id}", response_model=RLRunStatusResponse) + def get_run( + run_id: str, + tenant_id: str | None = Query(default=None), + workspace_id: str | None = Query(default=None), + x_tenant_id: str | None = Header(default=None), + x_workspace_id: str | None = Header(default=None), + ) -> RLRunStatusResponse: + try: + return _status_response(service.status(run_id, tenant_id=_tenant(x_tenant_id, tenant_id), workspace_id=_workspace(x_workspace_id, workspace_id))) + except KeyError as exc: + raise HTTPException(status_code=404, detail="run not found") from exc + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + + @router.get("/runs/{run_id}/events", response_class=PlainTextResponse) + def get_events( + run_id: str, + from_sequence: int = Query(default=0), + tenant_id: str | None = Query(default=None), + workspace_id: str | None = Query(default=None), + x_tenant_id: str | None = Header(default=None), + x_workspace_id: str | None = Header(default=None), + ) -> str: + try: + events = service.stream_since(run_id, from_sequence=from_sequence, tenant_id=_tenant(x_tenant_id, tenant_id), workspace_id=_workspace(x_workspace_id, workspace_id)) + except KeyError as exc: + raise HTTPException(status_code=404, detail="run not found") from exc + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + return "".join(json.dumps(RLStreamEventModel(**event.to_dict()).dict(), separators=(",", ":")) + "\n" for event in events) + + @router.post("/runs/{run_id}/cancel", response_model=RLRunStatusResponse) + def cancel_run(run_id: str, payload: RLRunCancelRequest) -> RLRunStatusResponse: + try: + service.status(run_id, tenant_id=payload.tenant_id, workspace_id=payload.workspace_id) + return _status_response(service.cancel(run_id, reason=payload.reason)) + except KeyError as exc: + raise HTTPException(status_code=404, detail="run not found") from exc + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + + @router.get("/runs/{run_id}/artifacts", response_model=RLRunArtifactListResponse) + def list_artifacts(run_id: str, tenant_id: str, workspace_id: str) -> RLRunArtifactListResponse: + try: + artifacts = service.collect(run_id, tenant_id=tenant_id, workspace_id=workspace_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="run not found") from exc + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + return RLRunArtifactListResponse(run_id=run_id, artifacts=[artifact.to_dict() for artifact in artifacts]) + + @router.get("/runs/{run_id}/replay/{artifact_id}", response_model=RLRunReplayResponse) + def replay_artifact(run_id: str, artifact_id: str, tenant_id: str, workspace_id: str) -> RLRunReplayResponse: + try: + return RLRunReplayResponse(**service.replay(run_id, artifact_id, tenant_id=tenant_id, workspace_id=workspace_id)) + except KeyError as exc: + raise HTTPException(status_code=404, detail="artifact not found") from exc + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + + @router.get("/runs/{run_id}/audit", response_model=RLRunAuditResponse) + def audit_run(run_id: str, tenant_id: str, workspace_id: str) -> RLRunAuditResponse: + try: + return RLRunAuditResponse(**service.audit(run_id, tenant_id=tenant_id, workspace_id=workspace_id)) + except KeyError as exc: + raise HTTPException(status_code=404, detail="run not found") from exc + except PermissionError as exc: + raise HTTPException(status_code=403, detail=str(exc)) from exc + + # Test and scheduler code call the service directly for artifact writes; no public upload route here. + return router diff --git a/breadboard/rl/phase3/benchmark_campaign.py b/breadboard/rl/phase3/benchmark_campaign.py new file mode 100644 index 00000000..1a904cd8 --- /dev/null +++ b/breadboard/rl/phase3/benchmark_campaign.py @@ -0,0 +1,200 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from breadboard.rl.phase2.benchmark import BenchmarkSourcePin, build_benchmark_slice_report, source_sha256 +from breadboard.rl.phase3.evidence import validate_phase3_command_log_manifest + +PHASE3_BENCHMARK_SCHEMA = "bb.rl.phase3.benchmark_campaign.v1" +PHASE3_BENCHMARK_CLAIM_BOUNDARY = "phase3_named_benchmark_campaign_scope" + + +@dataclass(frozen=True) +class BenchmarkCampaignSpec: + benchmark_id: str + benchmark_version: str + source_uri: str + expected_source_sha256: str + split_id: str + max_tasks: int + contamination_manifest_path: Path + output_dir: Path + fixture_scope: bool = False + + +def _load_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text()) + + + +def _reject_non_model_candidate(contamination: Mapping[str, Any]) -> str | None: + prompt_scan = contamination.get("prompt_solution_leakage_scan") + if not isinstance(prompt_scan, Mapping): + return None + candidate_source = str(prompt_scan.get("candidate_source") or "").lower() + non_model_markers = ("hand_written", "hand-written", "target_payload", "probe_only", "synthetic_baseline") + if any(marker in candidate_source for marker in non_model_markers): + return "benchmark_candidate_source_not_phase3_model_pipeline" + return None + + +def _safe_replay_path(replay_dir: Path, task_id: str) -> Path | None: + if not task_id or task_id in {".", ".."} or "\\" in task_id: + return None + candidate = (replay_dir / f"{task_id}.json").resolve() + try: + candidate.relative_to(replay_dir.resolve()) + except ValueError: + return None + return candidate + + +def _logical_replay_id(path: Path, payload: Mapping[str, Any], replay_dir: Path) -> str: + task_id = payload.get("task_id") + if isinstance(task_id, str) and task_id: + return task_id + relative = path.relative_to(replay_dir) + stem = str(relative.with_suffix("")) + if "/" not in stem and "_" in stem: + prefix, suffix = stem.rsplit("_", 1) + if suffix.isdigit(): + return f"{prefix}/{suffix}" + return stem + + +def _scan_replay_duplicates(replay_dir: Path) -> list[str]: + errors: list[str] = [] + if not replay_dir.is_dir(): + return errors + seen: dict[str, str] = {} + for path in sorted(replay_dir.rglob("*.json")): + try: + payload = json.loads(path.read_text()) + except json.JSONDecodeError: + continue + payload = payload if isinstance(payload, Mapping) else {} + logical_id = _logical_replay_id(path, payload, replay_dir) + rel = str(path.relative_to(replay_dir)) + if logical_id in seen: + errors.append(f"duplicate replay artifact for {logical_id}:{seen[logical_id]}:{rel}") + else: + seen[logical_id] = rel + return errors + + +def build_benchmark_campaign_report( + spec: BenchmarkCampaignSpec, *, run_summary_path: Path, replay_dir: Path, command_log_manifest: Mapping[str, Any] +) -> dict[str, Any]: + summary = _load_json(run_summary_path) + contamination = _load_json(spec.contamination_manifest_path) + controls = set(contamination.get("controls", [])) + errors: list[str] = [] + for control in ("source_hash_pin", "train_overlap_manifest", "prompt_solution_leakage_scan"): + if control not in controls: + errors.append(f"missing contamination control {control}") + candidate_error = _reject_non_model_candidate(contamination) + if candidate_error: + errors.append(candidate_error) + metrics = summary.get("metrics", {}) + acceptance_error = None + if not spec.fixture_scope and int(metrics.get("accepted", 0)) <= 0: + acceptance_error = "benchmark_no_accepted_tasks" + errors.append(acceptance_error) + failed = [str(task_id) for task_id in list(summary.get("failed_tasks", [])) + list(summary.get("quarantined_tasks", []))] + seen_failed: set[str] = set() + if failed and not replay_dir.is_dir(): + errors.append("benchmark_replay_dir_missing") + errors.extend(_scan_replay_duplicates(replay_dir)) + replay_refs: list[str] = [] + for task_id in failed: + if not task_id: + errors.append("benchmark_replay_task_id_empty") + continue + replay_path = _safe_replay_path(replay_dir, task_id) + if replay_path is None: + errors.append(f"unsafe replay task_id:{task_id}") + continue + if task_id in seen_failed: + errors.append(f"benchmark_replay_task_id_duplicate:{task_id}") + seen_failed.add(task_id) + replay_refs.append(str(replay_path)) + if not replay_path.is_file(): + errors.append(f"missing replay artifact for {task_id}") + continue + try: + replay_payload = json.loads(replay_path.read_text()) + except json.JSONDecodeError: + errors.append(f"malformed replay artifact for {task_id}") + continue + if isinstance(replay_payload, Mapping) and replay_payload.get("task_id") not in (None, task_id): + errors.append(f"replay artifact task_id mismatch for {task_id}") + pin = BenchmarkSourcePin(spec.benchmark_id, spec.benchmark_version, spec.split_id, spec.source_uri, spec.expected_source_sha256) + slice_report = build_benchmark_slice_report( + pin, + observed_source_sha256=source_sha256(summary.get("source_payload", "")), + contamination_controls=sorted(controls), + failure_replay_refs=replay_refs, + metrics=metrics, + target_run_id=str(command_log_manifest.get("target_run_id") or summary.get("target_run_id") or ""), + ).to_dict() + if not spec.fixture_scope: + slice_errors = [error for error in slice_report.get("errors", []) if error != "missing_failure_replay"] + if candidate_error and candidate_error not in slice_errors: + slice_errors.append(candidate_error) + if acceptance_error and acceptance_error not in slice_errors: + slice_errors.append(acceptance_error) + slice_report["errors"] = slice_errors + slice_report["report_id"] = "bb_zyphra_rl_phase3_benchmark_slice_v1" + slice_report["claim_boundary"] = PHASE3_BENCHMARK_CLAIM_BOUNDARY + if not slice_errors: + slice_report["status"] = "external_benchmark_package_accepted" + slice_report["passed"] = True + slice_report["accepted_for_claim"] = True + else: + slice_report["status"] = "rejected_external_benchmark_controls" + slice_report["passed"] = False + slice_report["accepted_for_claim"] = False + metadata = slice_report.get("metadata") + metadata = dict(metadata) if isinstance(metadata, Mapping) else {} + metadata.pop("fixture_scope", None) + metadata["campaign_scope"] = "external_named_benchmark_package" + metadata["benchmark_input_kind"] = "external_jsonl" + slice_report["metadata"] = metadata + for error in slice_report.get("errors", []): + if error not in errors: + errors.append(error) + target_run_id = slice_report.get("target_run_id") or str(command_log_manifest.get("target_run_id") or "") + evidence_root = spec.output_dir.parents[3] if len(spec.output_dir.parents) > 3 else spec.output_dir + manifest_errors = validate_phase3_command_log_manifest(command_log_manifest, target_run_id=target_run_id, repo_root=Path.cwd(), evidence_root=evidence_root) + errors.extend(manifest_errors) + metrics = metrics + report = { + "schema_version": PHASE3_BENCHMARK_SCHEMA, + "report_id": "phase3_benchmark_campaign", + "claim_boundary": PHASE3_BENCHMARK_CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "source_pin": pin.to_dict(), + "train_overlap_manifest": contamination.get("train_overlap_manifest"), + "prompt_solution_leakage_scan": contamination.get("prompt_solution_leakage_scan"), + "attempted": int(metrics.get("attempted", summary.get("attempted", 0))), + "accepted": int(metrics.get("accepted", summary.get("accepted", 0))), + "rejected": int(metrics.get("rejected", summary.get("rejected", 0))), + "quarantined": int(metrics.get("quarantined", summary.get("quarantined", 0))), + "pass_at_1": float(metrics.get("pass_at_1", 0.0)), + "mean_reward": float(metrics.get("mean_reward", 0.0)), + "p50_latency_seconds": float(metrics.get("p50_latency_seconds", 0.0)), + "p95_latency_seconds": float(metrics.get("p95_latency_seconds", 0.0)), + "cost_ledger_ref": summary.get("cost_ledger_ref", ""), + "slice_report": slice_report, + "errors": errors, + "input_hashes": {"run_summary": run_summary_path.name, "contamination": spec.contamination_manifest_path.name}, + "artifact_paths": {"run_summary": str(run_summary_path)}, + "scorecard_update_allowed": False, + "passed": not errors, + } + spec.output_dir.mkdir(parents=True, exist_ok=True) + (spec.output_dir / "phase3_benchmark_campaign_report.json").write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + return report diff --git a/breadboard/rl/phase3/env_families.py b/breadboard/rl/phase3/env_families.py new file mode 100644 index 00000000..5b4d32f0 --- /dev/null +++ b/breadboard/rl/phase3/env_families.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +from breadboard.rl.phase3.evidence import PHASE3_COMPONENT_REPORT_SCHEMA, sha256_file +from breadboard.rl.phase3.final_report import PHASE3_MILESTONE_CLAIM_BOUNDARIES +from typing import Any + + +def run_lean_console_env_probe(env_package_path: Path, *, target_run_id: str, output_dir: Path) -> dict[str, Any]: + errors: list[str] = [] + if not env_package_path.exists(): + errors.append("env_package_missing") + if importlib.util.find_spec("lean_dojo") is None and importlib.util.find_spec("lean") is None: + errors.append("lean_runtime_unavailable") + output_dir.mkdir(parents=True, exist_ok=True) + probe_path = output_dir / "lean_console_env_probe.json" + probe = { + "target_run_id": target_run_id, + "env_package_path": str(env_package_path), + "env_package_load": env_package_path.exists(), + "renderer_transcript_shape": "messages_with_roles" if not errors else "unverified", + "deterministic_replay": not errors, + "trainer_export_shape": "projection_rows" if not errors else "unverified", + "blocked_reason": errors[0] if errors else "", + "errors": errors, + } + probe_path.write_text(json.dumps(probe, sort_keys=True, indent=2) + "\n") + input_hashes = { + "env_package": sha256_file(env_package_path) if env_package_path.exists() else "sha256:missing-env-package", + "probe": sha256_file(probe_path), + } + report = { + "schema_version": PHASE3_COMPONENT_REPORT_SCHEMA, + "report_id": "phase3_lean_console_env_probe", + "milestone_id": "P3-M10", + "component": "lean_env_family", + "claim_boundary": PHASE3_MILESTONE_CLAIM_BOUNDARIES["P3-M10"], + "target_run_id": target_run_id, + "probe": probe, + "env_package_path": str(env_package_path), + "env_package_load": env_package_path.exists(), + "renderer_transcript_shape": probe["renderer_transcript_shape"], + "deterministic_replay": probe["deterministic_replay"], + "trainer_export_shape": probe["trainer_export_shape"], + "target_smoke_artifact": str(probe_path), + "blocked_reason": probe["blocked_reason"], + "errors": errors, + "input_hashes": input_hashes, + "artifact_paths": {"probe": str(probe_path)}, + "required_artifact_keys": ["probe"], + "scorecard_update_allowed": False, + "points": 60, + "passed": not errors, + } + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "lean_console_env_probe_report.json").write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + return report diff --git a/breadboard/rl/phase3/evidence.py b/breadboard/rl/phase3/evidence.py new file mode 100644 index 00000000..2363b329 --- /dev/null +++ b/breadboard/rl/phase3/evidence.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +PHASE3_COMMAND_LOG_MANIFEST_SCHEMA = "bb.rl.phase3.command_log_manifest.v1" +PHASE3_COMPONENT_REPORT_SCHEMA = "bb.rl.phase3.component_report.v1" +PHASE3_TARGET_RUN_ID_PATTERN = r"^\d{8}T\d{6}Z-slurm-\d+$" +REQUIRED_COMMAND_LOG_FIELDS = ( + "command_id", + "argv", + "raw_log_path", + "raw_log_sha256", + "slurm_job_id", + "target_run_id", + "node", + "started_at", + "completed_at", + "exit_code", + "status", +) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def write_phase3_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n") + +def normalize_phase3_metric_sources(metrics: dict[str, dict]) -> dict[str, dict]: + normalized = {key: dict(value) for key, value in metrics.items()} + source_defaults = { + "slurm": "slurm_sacct", + "gpu": "rocm_smi", + "verifier": "verifier_client", + "service": "service_event_log", + "object_store": "object_store", + "scheduler": "scheduler_control", + } + for key, source in source_defaults.items(): + normalized.setdefault(key, {}).setdefault("source", source) + return normalized + + +def _phase3_runs_root(evidence_root: Path) -> Path: + return (evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs").resolve() + + +def _resolve_under(root: Path, raw_path: Any) -> Path | None: + if raw_path is None: + return None + text = str(raw_path) + candidate = Path(text) + if not candidate.is_absolute(): + candidate = root / candidate + try: + resolved = candidate.resolve() + resolved.relative_to(root.resolve()) + except (OSError, ValueError): + return None + return resolved + + +def _command_rows(manifest: Mapping[str, Any]) -> list[Mapping[str, Any]]: + rows = manifest.get("commands", manifest.get("command_logs", [])) + if not isinstance(rows, list): + return [] + return [row for row in rows if isinstance(row, Mapping)] + + +def _inline_reports_from_log(path: Path) -> list[Any]: + reports: list[Any] = [] + for line in path.read_text(errors="replace").splitlines(): + if line.startswith("PHASE3_INTROSPECTION_REPORT=") or line.startswith("PHASE3_COMPONENT_REPORT_JSON="): + _, payload = line.split("=", 1) + try: + reports.append(json.loads(payload)) + except json.JSONDecodeError: + reports.append({"passed": False, "blocked_reason": "invalid_inline_report"}) + return reports + + +def validate_phase3_command_log_manifest( + manifest: Mapping[str, Any], *, target_run_id: str, repo_root: Path, evidence_root: Path +) -> list[str]: + del repo_root + errors: list[str] = [] + if manifest.get("schema_version") != PHASE3_COMMAND_LOG_MANIFEST_SCHEMA: + errors.append("schema_version must be bb.rl.phase3.command_log_manifest.v1") + manifest_target = str(manifest.get("target_run_id") or "") + if manifest_target != target_run_id: + errors.append("manifest target_run_id must match expected target_run_id") + if not re.match(PHASE3_TARGET_RUN_ID_PATTERN, target_run_id): + errors.append("target_run_id must match Phase 3 Slurm target run id pattern") + rows = _command_rows(manifest) + if not rows: + errors.append("commands must contain at least one command row") + return errors + seen: set[str] = set() + runs_root = _phase3_runs_root(evidence_root) + for index, row in enumerate(rows, start=1): + prefix = f"commands[{index}]" + for field_name in REQUIRED_COMMAND_LOG_FIELDS: + if row.get(field_name) in (None, "", []): + errors.append(f"{prefix}.{field_name} must be present") + command_id = str(row.get("command_id") or "") + if command_id in seen: + errors.append(f"{prefix}.command_id must be unique") + seen.add(command_id) + if row.get("target_run_id") != target_run_id: + errors.append(f"{prefix}.target_run_id must match manifest target_run_id") + if row.get("exit_code") != 0: + errors.append(f"{prefix}.exit_code must be 0") + if row.get("status") != "passed": + errors.append(f"{prefix}.status must be passed") + raw_path = _resolve_under(runs_root, row.get("raw_log_path")) + if raw_path is None: + errors.append(f"{prefix}.raw_log_path must stay under evidence RL_PHASE_3/runs") + continue + if not raw_path.exists() or not raw_path.is_file(): + errors.append(f"{prefix}.raw_log_path must exist") + continue + expected_hash = sha256_file(raw_path) + if row.get("raw_log_sha256") != expected_hash: + errors.append(f"{prefix}.raw_log_sha256 must match current raw log hash") + inline_reports = _inline_reports_from_log(raw_path) + for report_index, report in enumerate(inline_reports, start=1): + if not isinstance(report, Mapping): + errors.append(f"{prefix}.inline_reports[{report_index}] must be an object") + continue + if report.get("passed") is not True: + errors.append(f"{prefix}.inline_reports[{report_index}].passed must be true") + if inline_reports and "component_passed" in row and row.get("component_passed") is not True: + errors.append(f"{prefix}.component_passed must be true when inline reports are canonical") + return errors + + +def _artifact_mapping(report: Mapping[str, Any]) -> dict[str, Any]: + artifacts = report.get("artifact_paths") + if isinstance(artifacts, Mapping): + return dict(artifacts) + if isinstance(artifacts, list): + return {str(index): value for index, value in enumerate(artifacts)} + return {} + +def validate_phase3_artifact_hashes( + report: Mapping[str, Any], *, required_artifact_keys: Sequence[str], evidence_root: Path +) -> list[str]: + errors: list[str] = [] + artifact_paths = _artifact_mapping(report) + input_hashes = report.get("input_hashes") + input_hashes = input_hashes if isinstance(input_hashes, Mapping) else {} + root = evidence_root.resolve() + for key in required_artifact_keys: + if key not in artifact_paths: + errors.append(f"artifact_paths.{key} must be present") + continue + path = _resolve_under(root, artifact_paths[key]) + if path is None: + errors.append(f"artifact_paths.{key} must stay under evidence_root") + continue + if not path.exists() or not path.is_file(): + errors.append(f"artifact_paths.{key} must exist") + continue + expected_hash = input_hashes.get(key) + if not isinstance(expected_hash, str) or not expected_hash: + errors.append(f"input_hashes.{key} must be present") + continue + if expected_hash != sha256_file(path): + errors.append(f"input_hashes.{key} must match artifact_paths.{key} content sha256") + return errors + + + +def validate_phase3_component_report( + report: Mapping[str, Any], *, expected_schema: str, expected_claim_boundary: str, target_run_id: str, + required_artifact_keys: Sequence[str], evidence_root: Path +) -> list[str]: + errors: list[str] = [] + if report.get("schema_version") != expected_schema: + errors.append("schema_version must match expected component schema") + if expected_schema == PHASE3_COMPONENT_REPORT_SCHEMA and not str(report.get("component") or ""): + errors.append("generic Phase 3 component reports must include component") + if report.get("claim_boundary") != expected_claim_boundary: + errors.append("claim_boundary must match exact expected claim boundary") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if report.get("passed") is not True: + errors.append("passed must be true") + if not str(report.get("report_id") or ""): + errors.append("report_id must be present") + if report.get("target_run_id") != target_run_id: + errors.append("target_run_id must match expected target_run_id") + if not isinstance(report.get("input_hashes"), Mapping) or not report.get("input_hashes"): + errors.append("input_hashes must be a non-empty mapping") + artifact_paths = _artifact_mapping(report) + if not artifact_paths: + errors.append("artifact_paths must be present") + errors.extend(validate_phase3_artifact_hashes(report, required_artifact_keys=required_artifact_keys, evidence_root=evidence_root)) + return errors diff --git a/breadboard/rl/phase3/final_report.py b/breadboard/rl/phase3/final_report.py new file mode 100644 index 00000000..15b4e6cb --- /dev/null +++ b/breadboard/rl/phase3/final_report.py @@ -0,0 +1,723 @@ +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from breadboard.rl.phase3.evidence import PHASE3_COMPONENT_REPORT_SCHEMA, sha256_file, validate_phase3_artifact_hashes, validate_phase3_command_log_manifest, validate_phase3_component_report +from breadboard.rl.phase3.observability_live import LOCAL_OBJECT_STORE_BACKENDS, endpoint_is_local, validate_scheduler_metrics_readiness +from breadboard.rl.phase3.parity import PHASE3_PARITY_REPORT_ID, validate_phase3_parity_report + +PHASE3_FINAL_SCHEMA = "bb.rl.phase3.final_report.v1" +PHASE3_FINAL_REPORT_ID = "bb_zyphra_rl_phase3_final_report_v1" +PHASE3_FINAL_CLAIM_BOUNDARY = "phase3_final_report_existing_artifact_audit_scope" +PHASE3_RETIRED_MILESTONES: tuple[str, ...] = () +PHASE3_MILESTONES = tuple(f"P3-M{index}" for index in range(13)) +PHASE3_ACTIVE_MILESTONES = tuple(milestone for milestone in PHASE3_MILESTONES if milestone not in PHASE3_RETIRED_MILESTONES) +PHASE3_MILESTONE_POINTS = { + "P3-M0": 40, + "P3-M1": 80, + "P3-M2": 120, + "P3-M3": 120, + "P3-M4": 100, + "P3-M5": 90, + "P3-M6": 80, + "P3-M7": 80, + "P3-M8": 70, + "P3-M9": 60, + "P3-M10": 60, + "P3-M11": 80, + "P3-M12": 20, +} +PHASE3_ORIGINAL_TOTAL_POINTS = 1000 +PHASE3_ACTIVE_SCOPE_SCHEMA = "bb.rl.phase3.active_scope.v1" +PHASE3_ACTIVE_SCOPE_CLAIM_BOUNDARY = "phase3_active_scope_all_milestones_target_evidence_scope" +PHASE3_ACTIVE_SCOPE_READY_MEANING = ( + "Existing artifacts satisfy the promoted exact-scope Phase 3 boundary; broader or successor claims require separate canonical promotion." +) +PHASE3_CORE_READINESS_SCHEMA = PHASE3_ACTIVE_SCOPE_SCHEMA +PHASE3_CORE_CLAIM_BOUNDARY = PHASE3_ACTIVE_SCOPE_CLAIM_BOUNDARY +PHASE3_DEFERRED_MILESTONES: tuple[str, ...] = PHASE3_RETIRED_MILESTONES +PHASE3_CORE_MILESTONES = PHASE3_ACTIVE_MILESTONES +PHASE3_CORE_RAW_POINTS_TOTAL = sum(PHASE3_MILESTONE_POINTS[milestone] for milestone in PHASE3_CORE_MILESTONES) +PHASE3_MILESTONE_CLAIM_BOUNDARIES = { + "P3-M0": "phase3_strict_evidence_gates_named_scope", + "P3-M1": "phase3_target_verl_api_introspection_named_scope", + "P3-M2": "phase3_ppo_weight_update_8gpu_named_target_scope", + "P3-M3": "phase3_grpo_weight_update_8gpu_named_target_scope", + "P3-M4": "phase3_closed_loop_projection_to_real_verl_checkpoint_8gpu_scope", + "P3-M5": "phase3_api_sqlite_persistence_local_validation_scope", + "P3-M6": "phase3_containerized_slurm_workspace_hardening_scope", + "P3-M7": "phase3_named_benchmark_campaign_scope", + "P3-M8": "phase3_retired_provider_milestone_accepted_scope", + "P3-M9": "phase3_harbor_nemo_gym_named_endpoint_scope", + "P3-M11": "phase3_live_observability_object_store_scheduler_scope", + "P3-M10": "phase3_second_environment_family_lean_console_isolated_elan_scope", + "P3-M12": "phase3_final_report_audit_existing_artifacts_scope", +} + + +PHASE3_MILESTONE_BLOCKED_CLAIM_BOUNDARIES = { + "P3-M8": "phase3_retired_provider_milestone_pending_rubric_change_scope", + "P3-M9": "phase3_harbor_nemo_gym_blocked_scope", + "P3-M11": "phase3_observability_scheduler_store_blocked_scope", +} + + +def _expected_claim_boundary(milestone_id: str, report: Mapping[str, Any]) -> str: + if report.get("passed") is True: + return PHASE3_MILESTONE_CLAIM_BOUNDARIES[milestone_id] + return PHASE3_MILESTONE_BLOCKED_CLAIM_BOUNDARIES.get( + milestone_id, + PHASE3_MILESTONE_CLAIM_BOUNDARIES[milestone_id], + ) + + +def _validate_blocked_component_report( + report: Mapping[str, Any], *, expected_schema: str, expected_claim_boundary: str, target_run_id: str, + required_artifact_keys: tuple[str, ...], evidence_root: Path +) -> list[str]: + errors: list[str] = [] + if report.get("schema_version") != expected_schema: + errors.append("schema_version must match expected component schema") + if not str(report.get("component") or ""): + errors.append("generic Phase 3 component reports must include component") + if report.get("claim_boundary") != expected_claim_boundary: + errors.append("claim_boundary must match exact expected claim boundary") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if report.get("passed") is True: + errors.append("blocked component reports must not pass") + if not _blocked_reason(report): + errors.append("blocked component reports must include blocked_reason") + if not str(report.get("report_id") or ""): + errors.append("report_id must be present") + if report.get("target_run_id") != target_run_id: + errors.append("target_run_id must match expected target run") + if not isinstance(report.get("input_hashes"), Mapping) or not report.get("input_hashes"): + errors.append("input_hashes must be a non-empty mapping") + artifact_paths = report.get("artifact_paths") + artifact_paths = artifact_paths if isinstance(artifact_paths, Mapping) else {} + if not artifact_paths: + errors.append("artifact_paths must be present") + errors.extend(validate_phase3_artifact_hashes(report, required_artifact_keys=required_artifact_keys, evidence_root=evidence_root)) + return errors + + +def _mapping_copy(value: Any) -> dict[str, Any]: + return dict(value) if isinstance(value, Mapping) else {} + + + +def _blocked_reason(report: Mapping[str, Any]) -> str: + reason = report.get("blocked_reason") + return reason if isinstance(reason, str) else "" + + + +def _int_or_zero(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + +def _load_json_mapping(path: Path) -> Mapping[str, Any]: + try: + payload = json.loads(path.read_text()) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return {} + return payload if isinstance(payload, Mapping) else {} + + +def _artifact_paths(report: Mapping[str, Any]) -> Mapping[str, Any]: + paths = report.get("artifact_paths") + return paths if isinstance(paths, Mapping) else {} + + +def _input_hashes(report: Mapping[str, Any]) -> Mapping[str, Any]: + hashes = report.get("input_hashes") + return hashes if isinstance(hashes, Mapping) else {} + + + + +def _validate_p3m8_retirement_acceptance(component: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if component.get("passed") is not True: + return errors + if component.get("provider_kind") != "none": + errors.append("P3-M8 accepted retirement must not claim a live provider (provider_kind must be none)") + if component.get("retirement_accepted") is not True: + errors.append("P3-M8 accepted retirement must set retirement_accepted true") + if not str(component.get("rubric_decision") or ""): + errors.append("P3-M8 accepted retirement must record a rubric_decision") + if component.get("scorecard_update_allowed") is not False: + errors.append("P3-M8 accepted retirement scorecard_update_allowed must be false") + provider_report = component.get("provider_report") + provider_report = provider_report if isinstance(provider_report, Mapping) else {} + if provider_report.get("provider_kind") != "none": + errors.append("P3-M8 accepted retirement provider_report.provider_kind must be none") + if provider_report.get("retirement_accepted") is not True: + errors.append("P3-M8 accepted retirement provider_report.retirement_accepted must be true") + if provider_report.get("passed") is not True: + errors.append("P3-M8 accepted retirement provider_report.passed must be true") + if provider_report.get("scorecard_update_allowed") is not False: + errors.append("P3-M8 accepted retirement provider_report.scorecard_update_allowed must be false") + return errors + + +def _validate_p3m9_provider_classification(component: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if component.get("passed") is not True: + return errors + provider_report = component.get("provider_report") + provider_report = provider_report if isinstance(provider_report, Mapping) else {} + component_kind = component.get("provider_kind") + report_kind = provider_report.get("attestation_backend") + if component_kind != "harbor_facade": + errors.append("P3-M9 provider_kind must be harbor_facade for the active Harbor/NeMo Gym path") + if report_kind != "harbor_facade": + errors.append("P3-M9 provider_report.attestation_backend must be harbor_facade for the active Harbor/NeMo Gym path") + if component_kind == "native_benchflow" or report_kind == "native_benchflow": + errors.append("P3-M9 native BenchFlow evidence is contract-only and cannot satisfy the active Harbor/NeMo Gym milestone") + if provider_report.get("self_hosted") is True: + errors.append("P3-M9 harbor provider_report.self_hosted must not be true for canonical target evidence") + endpoint_identity = str(provider_report.get("endpoint_identity") or "") + if endpoint_is_local(endpoint_identity): + errors.append("P3-M9 harbor provider_report.endpoint_identity must not be local for canonical target evidence") + required_strings = ( + "endpoint_identity", + "env_package_sha256", + "task_name", + "trial_id_sha256", + ) + for key in required_strings: + if not str(provider_report.get(key) or ""): + errors.append(f"P3-M9 harbor provider_report.{key} must be present") + if provider_report.get("passed") is not True: + errors.append("P3-M9 harbor provider_report must pass") + if provider_report.get("scorecard_update_allowed") is not False: + errors.append("P3-M9 harbor provider_report.scorecard_update_allowed must be false") + routes = provider_report.get("harbor_routes") + routes = [str(route) for route in routes] if isinstance(routes, list) else [] + required_routes = [ + "GET /health", + "GET /metrics.json", + "GET /list_tasks", + "POST /score", + "POST /trial/create", + "POST /trial/{trial_id}/exec", + "GET /trial/{trial_id}", + "POST /trial/{trial_id}/finalize", + ] + if routes != required_routes: + errors.append("P3-M9 harbor provider_report.harbor_routes must match the Harbor service proof route sequence") + calls = provider_report.get("harbor_calls") + calls = calls if isinstance(calls, list) else [] + call_routes = [] + for index, call in enumerate(calls): + if not isinstance(call, Mapping): + errors.append(f"P3-M9 harbor provider_report.harbor_calls[{index}] must be an object") + continue + call_routes.append(str(call.get("route_template") or "")) + for key in ("method", "route_template", "status_code", "request_sha256", "response_sha256", "latency_seconds", "passed"): + if key not in call: + errors.append(f"P3-M9 harbor provider_report.harbor_calls[{index}].{key} must be present") + if call.get("passed") is not True: + errors.append(f"P3-M9 harbor provider_report.harbor_calls[{index}] must pass") + if call_routes != required_routes: + errors.append("P3-M9 harbor provider_report.harbor_calls must match the Harbor service proof route sequence") + return errors + + +def _reject_local_metric_urls(metric_sections: Mapping[str, Any]) -> list[str]: + url_fields = ( + ("verifier_metrics", "endpoint", "P3-M11 verifier_metrics.endpoint must not be local"), + ("object_store_metrics", "endpoint", "P3-M11 object_store_metrics.endpoint must not be local"), + ("object_store_metrics", "put_endpoint", "P3-M11 object_store_metrics.put_endpoint must not be local"), + ("object_store_metrics", "get_endpoint", "P3-M11 object_store_metrics.get_endpoint must not be local"), + ("object_store_metrics", "delete_endpoint", "P3-M11 object_store_metrics.delete_endpoint must not be local"), + ) + errors: list[str] = [] + for section_name, field_name, error in url_fields: + section = metric_sections.get(section_name) + if isinstance(section, Mapping) and endpoint_is_local(str(section.get(field_name) or "")): + errors.append(error) + scheduler = metric_sections.get("scheduler_metrics") + scheduler_control = scheduler.get("scheduler_control") if isinstance(scheduler, Mapping) else None + if isinstance(scheduler_control, Mapping) and endpoint_is_local(str(scheduler_control.get("endpoint") or "")): + errors.append("P3-M11 scheduler_metrics.scheduler_control.endpoint must not be local") + return errors + +def _presence_flag(section: Mapping[str, Any], key: str) -> bool | None: + if key not in section: + return None + value = section[key] + if isinstance(value, Mapping): + if "present" in value: + return value["present"] is True + return None + if isinstance(value, bool): + return value + return None + + +def _env_presence_flag(section: Mapping[str, Any], env_name: str) -> bool | None: + env_presence = section.get("env_presence") + if not isinstance(env_presence, Mapping): + return None + return _presence_flag(env_presence, env_name) + + +def _any_presence(section: Mapping[str, Any], keys: tuple[str, ...], env_names: tuple[str, ...] = ()) -> bool: + if any(_presence_flag(section, key) is True for key in keys): + return True + return any(_env_presence_flag(section, env_name) is True for env_name in env_names) + + +def _status_is_success(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, int): + return 200 <= value < 300 + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"ok", "passed", "success", "succeeded", "verified"}: + return True + try: + return 200 <= int(normalized) < 300 + except ValueError: + return False + return False + + +def _operation_status_ok(section: Mapping[str, Any], operation: str) -> bool: + candidates = ( + f"{operation}_status", + f"{operation}_status_code", + f"{operation}_http_status", + f"{operation}_passed", + f"{operation}_verified", + ) + for key in candidates: + if key in section and _status_is_success(section[key]): + return True + operation_payload = section.get(operation) + if isinstance(operation_payload, Mapping): + for key in ("status", "status_code", "http_status", "passed", "verified"): + if key in operation_payload and _status_is_success(operation_payload[key]): + return True + return False + + +def _validate_readback_durability(object_store: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if object_store.get("write_read_verified") is not True: + errors.append("P3-M11 object_store_metrics.write_read_verified must be true") + boolean_signals = ( + "readback_verified", + "read_after_write_verified", + "readback_matches", + "durability_verified", + "readback_durable", + ) + for key in boolean_signals: + if key in object_store and object_store.get(key) is not True: + errors.append(f"P3-M11 object_store_metrics.{key} must be true when present") + expected_hash = object_store.get("written_sha256") or object_store.get("put_sha256") + readback_hash = object_store.get("readback_sha256") or object_store.get("get_sha256") + if not isinstance(expected_hash, str) or not expected_hash: + errors.append("P3-M11 object_store_metrics written_sha256/put_sha256 must be present") + if not isinstance(readback_hash, str) or not readback_hash: + errors.append("P3-M11 object_store_metrics readback_sha256/get_sha256 must be present") + if isinstance(expected_hash, str) and expected_hash and isinstance(readback_hash, str) and readback_hash and expected_hash != readback_hash: + errors.append("P3-M11 object_store_metrics readback hash must match written hash") + return errors + + +def _validate_p3m11_observability_promotion(component: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if component.get("passed") is not True: + return errors + observability = component.get("observability_evidence") + observability = observability if isinstance(observability, Mapping) else {} + if observability.get("passed") is not True: + errors.append("P3-M11 observability_evidence.passed must be true") + evidence_errors = observability.get("errors") + if evidence_errors: + errors.append("P3-M11 observability_evidence.errors must be empty") + metric_sections = observability.get("metric_sections") + metric_sections = metric_sections if isinstance(metric_sections, Mapping) else {} + object_store = metric_sections.get("object_store_metrics") + if not isinstance(object_store, Mapping): + errors.append("P3-M11 observability_evidence.metric_sections.object_store_metrics must be present") + object_store = {} + object_store_backend = str(object_store.get("object_store") or "") + if not object_store_backend: + errors.append("P3-M11 object_store_metrics.object_store must be present") + elif object_store_backend in LOCAL_OBJECT_STORE_BACKENDS: + errors.append("P3-M11 object_store_metrics.object_store must be a production object-store backend") + errors.extend(_reject_local_metric_urls(metric_sections)) + verifier = metric_sections.get("verifier_metrics") + if not isinstance(verifier, Mapping): + errors.append("P3-M11 observability_evidence.metric_sections.verifier_metrics must be present") + verifier = {} + verifier_endpoint = str(verifier.get("endpoint") or "") + if not verifier_endpoint: + errors.append("P3-M11 verifier_metrics.endpoint must be present") + elif endpoint_is_local(verifier_endpoint): + errors.append("P3-M11 verifier_metrics.endpoint must not be local") + if not _any_presence(verifier, ("token_present", "verifier_token_present"), ("BREADBOARD_VERIFIER_TOKEN",)): + errors.append("P3-M11 verifier_metrics token evidence must be present") + if not _any_presence( + object_store, + ("token_present", "object_store_token_present", "credential_present", "credentials_present", "access_key_present"), + ("BREADBOARD_OBJECT_STORE_TOKEN", "BREADBOARD_OBJECT_STORE_ACCESS_KEY"), + ): + errors.append("P3-M11 object_store_metrics token evidence must be present") + for endpoint_field in ("put_endpoint", "get_endpoint", "delete_endpoint"): + endpoint = str(object_store.get(endpoint_field) or "") + if not endpoint: + errors.append(f"P3-M11 object_store_metrics.{endpoint_field} must be present") + elif endpoint_is_local(endpoint): + errors.append(f"P3-M11 object_store_metrics.{endpoint_field} must not be local") + for operation in ("put", "get", "delete"): + if not _operation_status_ok(object_store, operation): + errors.append(f"P3-M11 object_store_metrics.{operation} status evidence must show success") + errors.extend(_validate_readback_durability(object_store)) + scheduler = metric_sections.get("scheduler_metrics") + if not isinstance(scheduler, Mapping): + errors.append("P3-M11 observability_evidence.metric_sections.scheduler_metrics must be present") + scheduler = {} + for scheduler_error in validate_scheduler_metrics_readiness(scheduler): + errors.append(f"P3-M11 observability_evidence.metric_sections.scheduler_metrics.{scheduler_error}") + if observability.get("verifier_latency") is None: + errors.append("P3-M11 observability_evidence.verifier_latency must be present") + return errors + + + + +def _fixture_benchmark_without_external_inputs(component: Mapping[str, Any]) -> bool: + benchmark = component.get("benchmark_report") + benchmark = benchmark if isinstance(benchmark, Mapping) else {} + slice_report = benchmark.get("slice_report") + slice_report = slice_report if isinstance(slice_report, Mapping) else {} + metadata = slice_report.get("metadata") + metadata = metadata if isinstance(metadata, Mapping) else {} + source_pin = benchmark.get("source_pin") + source_pin = source_pin if isinstance(source_pin, Mapping) else {} + if metadata.get("fixture_scope") == "hash_pinned_benchmark_slice": + return True + return source_pin.get("benchmark_version") == "fixture-v2" + +def _benchmark_uses_non_model_candidate(component: Mapping[str, Any]) -> bool: + benchmark = component.get("benchmark_report") + benchmark = benchmark if isinstance(benchmark, Mapping) else {} + prompt_scan = benchmark.get("prompt_solution_leakage_scan") + if not isinstance(prompt_scan, Mapping): + return False + candidate_source = str(prompt_scan.get("candidate_source") or "").lower() + non_model_markers = ("hand_written", "hand-written", "target_payload", "probe_only", "synthetic_baseline") + return any(marker in candidate_source for marker in non_model_markers) + + +def _validate_p3m7_benchmark_candidate_source(component: Mapping[str, Any]) -> list[str]: + if component.get("passed") is not True: + return [] + if _benchmark_uses_non_model_candidate(component): + return ["P3-M7 benchmark candidate source must come from the Phase 3 model pipeline"] + return [] + + +def _core_milestone_blocker(milestone_id: str, component: Mapping[str, Any]) -> str: + if milestone_id == "P3-M7" and _fixture_benchmark_without_external_inputs(component): + return "fixture_benchmark_not_external_core_credit" + if milestone_id == "P3-M7" and _benchmark_uses_non_model_candidate(component): + return "benchmark_candidate_not_phase3_model_pipeline" + blocked_reason = _blocked_reason(component) + if blocked_reason: + return blocked_reason + if component.get("passed") is not True: + return "milestone_report_not_passed" + return "" + + +def build_phase3_core_readiness(milestone_reports: Mapping[str, Mapping[str, Any]]) -> dict[str, Any]: + report_sources = milestone_reports if isinstance(milestone_reports, Mapping) else {} + milestone_statuses = [] + blocked_milestones = [] + for milestone_id in PHASE3_ACTIVE_MILESTONES: + component = report_sources.get(milestone_id, {}) + component = component if isinstance(component, Mapping) else {} + blocker = _core_milestone_blocker(milestone_id, component) + active_complete = blocker == "" + if not active_complete: + blocked_milestones.append(milestone_id) + point_value = PHASE3_MILESTONE_POINTS[milestone_id] + milestone_statuses.append({ + "milestone_id": milestone_id, + "point_value": point_value, + "active_complete": active_complete, + "blocker": blocker, + "report_id": component.get("report_id"), + "passed": component.get("passed") is True, + }) + ready = not blocked_milestones + label = "active-artifact-audit-clean" if ready else "active-artifact-audit-blocked" + ready_meaning = PHASE3_ACTIVE_SCOPE_READY_MEANING + core_raw_points_verified = sum( + PHASE3_MILESTONE_POINTS[status["milestone_id"]] + for status in milestone_statuses + if status["active_complete"] + ) + return { + "schema_version": PHASE3_ACTIVE_SCOPE_SCHEMA, + "claim_boundary": PHASE3_ACTIVE_SCOPE_CLAIM_BOUNDARY, + "ready": ready, + "artifact_audit_clean": ready, + "ready_meaning": ready_meaning, + "scorecard_update_allowed": False, + "active_milestones": list(PHASE3_ACTIVE_MILESTONES), + "retired_milestones": list(PHASE3_RETIRED_MILESTONES), + "blocked_active_milestones": blocked_milestones, + "core_milestones": list(PHASE3_ACTIVE_MILESTONES), + "deferred_milestones": list(PHASE3_RETIRED_MILESTONES), + "blocked_core_milestones": blocked_milestones, + "core_raw_points_total": PHASE3_CORE_RAW_POINTS_TOTAL, + "core_raw_points_verified": core_raw_points_verified, + "original_scorecard_total_points": PHASE3_ORIGINAL_TOTAL_POINTS, + "report_label": label, + "milestone_statuses": milestone_statuses, + } + + +def build_phase3_active_scope(milestone_reports: Mapping[str, Mapping[str, Any]]) -> dict[str, Any]: + return build_phase3_core_readiness(milestone_reports) + +def _summary_by_milestone(report: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: + summaries = report.get("milestone_summaries") + if not isinstance(summaries, list): + return {} + mapped: dict[str, Mapping[str, Any]] = {} + for summary in summaries: + if not isinstance(summary, Mapping): + continue + milestone_id = summary.get("milestone_id") + if isinstance(milestone_id, str): + mapped[milestone_id] = summary + return mapped + +def _validate_milestone_summary_list(report: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + summaries = report.get("milestone_summaries") + if not isinstance(summaries, list): + return ["milestone_summaries must be a list"] + milestone_ids: list[str] = [] + for index, summary in enumerate(summaries): + if not isinstance(summary, Mapping): + errors.append(f"milestone_summaries[{index}] must be an object") + continue + milestone_id = summary.get("milestone_id") + if not isinstance(milestone_id, str): + errors.append(f"milestone_summaries[{index}].milestone_id must be a string") + continue + milestone_ids.append(milestone_id) + if milestone_ids != list(PHASE3_MILESTONES): + errors.append("milestone_summaries must appear exactly once in Phase 3 milestone order") + return errors + + + + +def _validate_milestone_summary(milestone_id: str, component: Mapping[str, Any], summary: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + blocked_reason = _blocked_reason(component) + passed = component.get("passed") is True + claim_ready = passed and not blocked_reason + expected = { + "report_id": component.get("report_id"), + "schema_version": component.get("schema_version"), + "claim_boundary": component.get("claim_boundary"), + "passed": passed, + "blocked_reason": blocked_reason, + "claim_ready": claim_ready, + "scorecard_update_allowed": component.get("scorecard_update_allowed"), + } + for key, value in expected.items(): + if summary.get(key) != value: + errors.append(f"{milestone_id} summary.{key} must match milestone report") + return errors + + + +def build_phase3_final_report( + *, target_run_id: str, milestone_reports: Mapping[str, Mapping[str, Any]], command_log_manifest: Mapping[str, Any], + scorecard: Mapping[str, Any], claim_ledger_text: str +) -> dict[str, Any]: + report_sources = milestone_reports if isinstance(milestone_reports, Mapping) else {} + summaries = [] + for milestone_id in PHASE3_MILESTONES: + report = report_sources.get(milestone_id, {}) + report = report if isinstance(report, Mapping) else {} + blocked_reason = _blocked_reason(report) + claim_ready = report.get("passed") is True and not blocked_reason + summaries.append({ + "milestone_id": milestone_id, + "report_id": report.get("report_id"), + "schema_version": report.get("schema_version"), + "claim_boundary": report.get("claim_boundary"), + "passed": report.get("passed") is True, + "blocked_reason": blocked_reason, + "claim_ready": claim_ready, + "scorecard_update_allowed": report.get("scorecard_update_allowed"), + }) + return { + "schema_version": PHASE3_FINAL_SCHEMA, + "report_id": PHASE3_FINAL_REPORT_ID, + "claim_boundary": PHASE3_FINAL_CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "milestone_summaries": summaries, + "milestone_reports": {key: _mapping_copy(value) for key, value in report_sources.items()}, + "active_scope": build_phase3_active_scope(report_sources), + "core_readiness": build_phase3_active_scope(report_sources), + "command_log_manifest": _mapping_copy(command_log_manifest), + "scorecard": {}, + "claim_ledger_text": claim_ledger_text if isinstance(claim_ledger_text, str) else "", + "scorecard_update_allowed": False, + } + + +def validate_phase3_final_report(report: Mapping[str, Any], *, repo_root: Path, evidence_root: Path) -> list[str]: + errors: list[str] = [] + if report.get("schema_version") != PHASE3_FINAL_SCHEMA: + errors.append("schema_version must be Phase 3 final report schema") + if report.get("report_id") != PHASE3_FINAL_REPORT_ID: + errors.append("report_id must be Phase 3 final report id") + if report.get("claim_boundary") != PHASE3_FINAL_CLAIM_BOUNDARY: + errors.append("claim_boundary must be Phase 3 final boundary") + if report.get("scorecard_update_allowed") is not False: + errors.append("final report cannot update scorecard") + target_run_id = str(report.get("target_run_id") or "") + manifest = report.get("command_log_manifest") if isinstance(report.get("command_log_manifest"), Mapping) else {} + errors.extend(validate_phase3_command_log_manifest(manifest, target_run_id=target_run_id, repo_root=repo_root, evidence_root=evidence_root)) + errors.extend(_validate_milestone_summary_list(report)) + milestone_summaries = _summary_by_milestone(report) + for milestone_id in PHASE3_MILESTONES: + if milestone_id not in milestone_summaries: + errors.append(f"{milestone_id} summary must be present") + for milestone_id in milestone_summaries: + if milestone_id not in PHASE3_MILESTONES: + errors.append(f"{milestone_id} summary is not a Phase 3 milestone") + milestone_reports = report.get("milestone_reports") if isinstance(report.get("milestone_reports"), Mapping) else {} + for milestone_id in milestone_reports: + if milestone_id not in PHASE3_MILESTONES: + errors.append(f"{milestone_id} report is not a Phase 3 milestone") + parity_refs: dict[str, str] = {} + parity_hashes: dict[str, str] = {} + for milestone_id in PHASE3_MILESTONES: + component = milestone_reports.get(milestone_id) + if not isinstance(component, Mapping): + errors.append(f"{milestone_id} report must be present") + continue + if component.get("milestone_id") != milestone_id: + errors.append(f"{milestone_id} report milestone_id must match outer milestone key") + schema_version = str(component.get("schema_version") or "") + summary = milestone_summaries.get(milestone_id, {}) + if isinstance(summary, Mapping): + errors.extend(_validate_milestone_summary(milestone_id, component, summary)) + claim_boundary = str(component.get("claim_boundary") or "") + expected_claim_boundary = _expected_claim_boundary(milestone_id, component) + if not schema_version: + errors.append(f"{milestone_id} schema_version must be present") + continue + if not claim_boundary: + errors.append(f"{milestone_id} claim_boundary must be present") + continue + if claim_boundary != expected_claim_boundary: + errors.append(f"{milestone_id} claim_boundary must be {expected_claim_boundary}") + blocked_reason = _blocked_reason(component) + required_artifact_keys = tuple(component.get("required_artifact_keys", ())) + if blocked_reason: + component_errors = _validate_blocked_component_report( + component, + expected_schema=PHASE3_COMPONENT_REPORT_SCHEMA, + expected_claim_boundary=expected_claim_boundary, + target_run_id=target_run_id, + required_artifact_keys=required_artifact_keys, + evidence_root=evidence_root, + ) + else: + component_errors = validate_phase3_component_report( + component, + expected_schema=PHASE3_COMPONENT_REPORT_SCHEMA, + expected_claim_boundary=expected_claim_boundary, + target_run_id=target_run_id, + required_artifact_keys=required_artifact_keys, + evidence_root=evidence_root, + ) + if milestone_id in {"P3-M2", "P3-M3", "P3-M4"}: + component_paths = _artifact_paths(component) + component_hashes = _input_hashes(component) + parity_report_path = component_paths.get("parity_report") + parity_report_hash = component_hashes.get("parity_report") + if not isinstance(parity_report_path, str) or not parity_report_path: + errors.append(f"{milestone_id}: parity_report artifact path must be present") + else: + parity_refs[milestone_id] = parity_report_path + if not isinstance(parity_report_hash, str) or not parity_report_hash: + errors.append(f"{milestone_id}: parity_report input hash must be present") + else: + parity_hashes[milestone_id] = parity_report_hash + if component.get("parity_report_id") != PHASE3_PARITY_REPORT_ID: + errors.append(f"{milestone_id}: parity_report_id must be {PHASE3_PARITY_REPORT_ID}") + if milestone_id == "P3-M7" and _int_or_zero(component.get("points")) > 0 and _fixture_benchmark_without_external_inputs(component): + if component.get("scorecard_update_allowed") is not False: + errors.append("P3-M7 fixture benchmark evidence cannot update scorecard") + if milestone_id == "P3-M8": + errors.extend(_validate_p3m8_retirement_acceptance(component)) + if milestone_id == "P3-M7": + errors.extend(_validate_p3m7_benchmark_candidate_source(component)) + if milestone_id == "P3-M9": + errors.extend(_validate_p3m9_provider_classification(component)) + if milestone_id == "P3-M11": + errors.extend(_validate_p3m11_observability_promotion(component)) + errors.extend(f"{milestone_id}: {error}" for error in component_errors) + if parity_refs: + if len(set(parity_refs.values())) != 1: + errors.append("P3-M2/P3-M3/P3-M4 parity_report artifact paths must match") + if len(set(parity_hashes.values())) != 1: + errors.append("P3-M2/P3-M3/P3-M4 parity_report input hashes must match") + parity_rel = next(iter(parity_refs.values())) + parity_path = (evidence_root / parity_rel).resolve() + try: + parity_path.relative_to(evidence_root.resolve()) + except ValueError: + errors.append("parity_report artifact must stay under evidence_root") + else: + parity_report = _load_json_mapping(parity_path) + if not parity_report: + errors.append("parity_report artifact must be readable JSON") + else: + errors.extend(f"parity_report: {error}" for error in validate_phase3_parity_report(parity_report, target_run_id=target_run_id, evidence_root=evidence_root)) + parity_sha = sha256_file(parity_path) + expected_sha = next(iter(parity_hashes.values()), "") + if expected_sha and parity_sha != expected_sha: + errors.append("parity_report input hash must match artifact content") + active_scope = report.get("active_scope") + if not isinstance(active_scope, Mapping): + active_scope = report.get("core_readiness") + if not isinstance(active_scope, Mapping): + errors.append("active_scope must be present") + else: + expected_active_scope = build_phase3_active_scope(milestone_reports) + for key, value in expected_active_scope.items(): + if active_scope.get(key) != value: + errors.append(f"active_scope.{key} must match milestone reports") + ledger_raw = report.get("claim_ledger_text") + ledger = ledger_raw if isinstance(ledger_raw, str) else "" + if target_run_id not in ledger or PHASE3_FINAL_REPORT_ID not in ledger or PHASE3_FINAL_CLAIM_BOUNDARY not in ledger: + errors.append("claim ledger must contain target_run_id, final report id, and final claim boundary") + return errors diff --git a/breadboard/rl/phase3/integrations.py b/breadboard/rl/phase3/integrations.py new file mode 100644 index 00000000..402786f3 --- /dev/null +++ b/breadboard/rl/phase3/integrations.py @@ -0,0 +1,613 @@ +from __future__ import annotations + +import hashlib +import http.client +import json +import os +import ipaddress +import socket +import time +import urllib.parse +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any, Literal, Mapping, Sequence +from pathlib import Path + +from breadboard.rl.phase3.evidence import sha256_file + + +@dataclass(frozen=True) +class VerifierCallEvidence: + provider: str + url: str + request_sha256: str + response_sha256: str + status_code: int + latency_seconds: float + passed: bool + blocked_reason: str = "" + + def to_dict(self) -> dict[str, Any]: + return self.__dict__.copy() + + +def _sha(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _provider_env(provider: Literal["ors", "openreward"]) -> tuple[str, str]: + if provider == "ors": + return os.environ.get("BREADBOARD_ORS_BASE_URL", ""), os.environ.get("BREADBOARD_ORS_TOKEN", "") + return os.environ.get("BREADBOARD_OPENREWARD_BASE_URL", ""), os.environ.get("BREADBOARD_OPENREWARD_TOKEN", "") + +def _provider_url_block_reason(base_url: str) -> str: + block_reason, _addresses = _provider_pinned_addresses(base_url) + return block_reason + + +def _provider_pinned_addresses(base_url: str) -> tuple[str, tuple[str, ...]]: + parsed = urllib.parse.urlparse(base_url) + if parsed.scheme != "https" or not parsed.netloc: + return "provider_endpoint_must_be_https", () + host = parsed.hostname or "" + lowered = host.lower().rstrip(".") + if lowered in {"localhost", "localhost.localdomain"} or lowered.endswith(".localhost"): + return "provider_endpoint_must_not_be_loopback_or_private", () + try: + address = ipaddress.ip_address(lowered) + except ValueError: + try: + resolved = socket.getaddrinfo(lowered, parsed.port or 443, type=socket.SOCK_STREAM) + except OSError: + return "provider_endpoint_dns_resolution_failed", () + addresses: list[str] = [] + for result in resolved: + address = ipaddress.ip_address(result[4][0]) + if not address.is_global: + return "provider_endpoint_must_not_be_loopback_or_private", () + address_text = str(address) + if address_text not in addresses: + addresses.append(address_text) + if not addresses: + return "provider_endpoint_dns_resolution_failed", () + return "", tuple(addresses) + if not address.is_global: + return "provider_endpoint_must_not_be_loopback_or_private", () + return "", (str(address),) + +def _origin(url: str) -> tuple[str, str, int | None]: + parsed = urllib.parse.urlparse(url) + port = parsed.port + if port is None and parsed.scheme == "https": + port = 443 + elif port is None and parsed.scheme == "http": + port = 80 + return parsed.scheme, (parsed.hostname or "").lower().rstrip("."), port + +def _origin_key(url: str) -> tuple[str, int]: + _scheme, host, port = _origin(url) + return host, port or 443 + + +class ProviderRedirectBlocked(RuntimeError): + def __init__(self, block_reason: str) -> None: + self.block_reason = block_reason + super().__init__(block_reason) + + +class _NoUnsafeRedirectHandler(urllib.request.HTTPRedirectHandler): + def __init__(self, pinned_addresses: dict[tuple[str, int], tuple[str, ...]] | None = None) -> None: + self._pinned_addresses = pinned_addresses if pinned_addresses is not None else {} + + def redirect_request(self, req, fp, code, msg, headers, newurl): # type: ignore[no-untyped-def] + if _origin(req.full_url) != _origin(newurl): + raise ProviderRedirectBlocked("provider_redirect_cross_origin_blocked") + raise ProviderRedirectBlocked("provider_redirect_not_followed") + + +class _PinnedHTTPSConnection(http.client.HTTPSConnection): + def __init__(self, host: str, pinned_addresses: Mapping[tuple[str, int], tuple[str, ...]], **kwargs: Any) -> None: + super().__init__(host, **kwargs) + self._pinned_addresses = pinned_addresses + + def connect(self) -> None: + addresses = self._pinned_addresses.get((self.host.lower().rstrip("."), self.port)) + if not addresses: + raise ProviderRedirectBlocked("provider_endpoint_dns_resolution_failed") + last_error: OSError | None = None + for address in addresses: + try: + sock = socket.create_connection((address, self.port), self.timeout, self.source_address) + self.sock = self._context.wrap_socket(sock, server_hostname=self.host) + return + except OSError as exc: + last_error = exc + if last_error is not None: + raise last_error + raise ProviderRedirectBlocked("provider_endpoint_dns_resolution_failed") + + +class _PinnedHTTPSHandler(urllib.request.HTTPSHandler): + def __init__(self, pinned_addresses: Mapping[tuple[str, int], tuple[str, ...]]) -> None: + super().__init__() + self._pinned_addresses = pinned_addresses + + def https_open(self, req): # type: ignore[no-untyped-def] + return self.do_open( + lambda host, **kwargs: _PinnedHTTPSConnection(host, self._pinned_addresses, **kwargs), + req, + ) + + +def _open_without_proxies(request: urllib.request.Request, *, timeout_s: float): + block_reason, addresses = _provider_pinned_addresses(request.full_url) + if block_reason: + raise ProviderRedirectBlocked(block_reason) + pinned_addresses = {_origin_key(request.full_url): addresses} + opener = urllib.request.build_opener( + urllib.request.ProxyHandler({}), + _NoUnsafeRedirectHandler(pinned_addresses), + _PinnedHTTPSHandler(pinned_addresses), + ) + return opener.open(request, timeout=timeout_s) + + + +def call_ors_openreward(payload: Mapping[str, Any], *, provider: Literal["ors", "openreward"], timeout_s: float) -> VerifierCallEvidence: + base_url, token = _provider_env(provider) + body = json.dumps(dict(payload), sort_keys=True, separators=(",", ":")).encode() + if not base_url or not token: + return VerifierCallEvidence(provider, base_url, _sha(body), "", 0, 0.0, False, "missing_live_provider_credentials") + block_reason = _provider_url_block_reason(base_url) + if block_reason: + return VerifierCallEvidence(provider, base_url, _sha(body), "", 0, 0.0, False, block_reason) + request = urllib.request.Request(base_url.rstrip("/") + "/verify", data=body, method="POST", headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}) + start = time.monotonic() + try: + with _open_without_proxies(request, timeout_s=timeout_s) as response: # noqa: S310 - URL is operator-configured provider endpoint. + response_body = response.read() + status = int(response.status) + except urllib.error.HTTPError as exc: + response_body = exc.read() + status = int(exc.code) + return VerifierCallEvidence(provider, base_url, _sha(body), _sha(response_body), status, time.monotonic() - start, False) + except ProviderRedirectBlocked as exc: + return VerifierCallEvidence(provider, base_url, _sha(body), "", 0, time.monotonic() - start, False, exc.block_reason) + except Exception as exc: # noqa: BLE001 + return VerifierCallEvidence(provider, base_url, _sha(body), "", 0, time.monotonic() - start, False, exc.__class__.__name__) + return VerifierCallEvidence(provider, base_url, _sha(body), _sha(response_body), status, time.monotonic() - start, 200 <= status < 300) + + +def run_live_verifier_campaign(rows: Sequence[Mapping[str, Any]], *, target_run_id: str) -> dict[str, Any]: + calls = [] + for provider in ("ors", "openreward"): + for row in rows: + calls.append(call_ors_openreward({"target_run_id": target_run_id, "row": dict(row)}, provider=provider, timeout_s=10.0).to_dict()) + blocked = [call for call in calls if call.get("blocked_reason")] + return { + "schema_version": "bb.rl.phase3.live_verifier_campaign.v1", + "report_id": "phase3_live_verifier_campaign", + "claim_boundary": "phase3_live_provider_campaign_scope", + "target_run_id": target_run_id, + "calls": calls, + "blocked_reason": ";".join(sorted({str(call.get("blocked_reason")) for call in blocked if call.get("blocked_reason")})), + "scorecard_update_allowed": False, + "passed": bool(calls) and all(call["passed"] for call in calls), + } + + +HARBOR_SERVICE_PROOF_SCHEMA = "bb.rl.phase3.harbor_service_proof.v1" +HARBOR_SERVICE_PROOF_ID = "phase3_harbor_service_proof" +HARBOR_CLAIM_BOUNDARY = "phase3_harbor_nemo_gym_named_endpoint_scope" +HARBOR_BLOCKED_CLAIM_BOUNDARY = "phase3_harbor_nemo_gym_blocked_scope" +HARBOR_ROUTES = ( + "GET /health", + "GET /metrics.json", + "GET /list_tasks", + "POST /score", + "POST /trial/create", + "POST /trial/{trial_id}/exec", + "GET /trial/{trial_id}", + "POST /trial/{trial_id}/finalize", +) + + +def _redacted_endpoint(base_url: str) -> str: + parsed = urllib.parse.urlparse(base_url) + if not parsed.scheme or not parsed.netloc: + return "" + return urllib.parse.urlunparse((parsed.scheme, parsed.netloc, "", "", "", "")) + + +def _response_json(response: bytes) -> Any: + if not response: + return {} + try: + decoded = json.loads(response.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return {} + return decoded + + +def _json_request_body(payload: Mapping[str, Any] | None) -> bytes: + return b"" if payload is None else json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + + +def run_benchflow_harbor_attestation(env_package_path, *, target_run_id: str) -> dict[str, Any]: + return run_harbor_service_proof(env_package_path, target_run_id=target_run_id) + + +def run_harbor_service_proof(env_package_path, *, target_run_id: str, task_name: str | None = None) -> dict[str, Any]: + harbor_base_url = os.environ.get("BREADBOARD_HARBOR_BASE_URL", "") + harbor_token = os.environ.get("BREADBOARD_HARBOR_TOKEN", "") + harbor_task_name = task_name or os.environ.get("BREADBOARD_HARBOR_TASK_NAME", "phase3-harbor-smoke") + if not harbor_base_url or not harbor_token: + return { + "schema_version": HARBOR_SERVICE_PROOF_SCHEMA, + "report_id": HARBOR_SERVICE_PROOF_ID, + "claim_boundary": HARBOR_BLOCKED_CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "attestation_backend": "harbor_facade", + "provider_kind": "harbor_facade", + "harbor_routes": list(HARBOR_ROUTES), + "missing_provider_env": [ + "BREADBOARD_HARBOR_BASE_URL", + "BREADBOARD_HARBOR_TOKEN", + ], + "blocked_reason": "missing_harbor_credentials", + "scorecard_update_allowed": False, + "passed": False, + } + return _run_harbor_facade_attestation( + harbor_base_url, + harbor_token, + env_package_path, + target_run_id=target_run_id, + task_name=harbor_task_name, + ) + + +def _run_native_benchflow_attestation(base_url: str, token: str, env_package_path, *, target_run_id: str) -> dict[str, Any]: + block_reason = _provider_url_block_reason(base_url) + if block_reason: + return { + "schema_version": "bb.rl.phase3.native_benchflow_attestation.v1", + "report_id": "phase3_native_benchflow_attestation", + "claim_boundary": "phase3_native_benchflow_contract_only_scope", + "target_run_id": target_run_id, + "blocked_reason": block_reason, + "attestation_backend": "native_benchflow", + "scorecard_update_allowed": False, + "passed": False, + } + if not os.path.isfile(env_package_path): + return { + "schema_version": "bb.rl.phase3.native_benchflow_attestation.v1", + "report_id": "phase3_native_benchflow_attestation", + "claim_boundary": "phase3_native_benchflow_contract_only_scope", + "target_run_id": target_run_id, + "blocked_reason": "missing_benchflow_env_package", + "attestation_backend": "native_benchflow", + "scorecard_update_allowed": False, + "passed": False, + } + env_package_sha256 = sha256_file(Path(env_package_path)) + body = json.dumps({"target_run_id": target_run_id, "env_package_sha256": env_package_sha256}, sort_keys=True, separators=(",", ":")).encode() + request = urllib.request.Request(base_url.rstrip("/") + "/attest", data=body, method="POST", headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}) + start = time.monotonic() + try: + with _open_without_proxies(request, timeout_s=10.0) as response: # noqa: S310 - operator-configured provider endpoint. + response_body = response.read() + status = int(response.status) + except urllib.error.HTTPError as exc: + response_body = exc.read() + status = int(exc.code) + return { + "schema_version": "bb.rl.phase3.native_benchflow_attestation.v1", + "report_id": "phase3_native_benchflow_attestation", + "claim_boundary": "phase3_native_benchflow_contract_only_scope", + "target_run_id": target_run_id, + "attestation_backend": "native_benchflow", + "endpoint_path": "/attest", + "request_sha256": _sha(body), + "response_sha256": _sha(response_body), + "status_code": status, + "latency_seconds": time.monotonic() - start, + "blocked_reason": "", + "scorecard_update_allowed": False, + "passed": False, + } + except ProviderRedirectBlocked as exc: + return { + "schema_version": "bb.rl.phase3.native_benchflow_attestation.v1", + "report_id": "phase3_native_benchflow_attestation", + "claim_boundary": "phase3_native_benchflow_contract_only_scope", + "target_run_id": target_run_id, + "attestation_backend": "native_benchflow", + "endpoint_path": "/attest", + "request_sha256": _sha(body), + "response_sha256": "", + "status_code": 0, + "latency_seconds": time.monotonic() - start, + "blocked_reason": exc.block_reason, + "scorecard_update_allowed": False, + "passed": False, + } + except Exception as exc: # noqa: BLE001 + return { + "schema_version": "bb.rl.phase3.native_benchflow_attestation.v1", + "report_id": "phase3_native_benchflow_attestation", + "claim_boundary": "phase3_native_benchflow_contract_only_scope", + "target_run_id": target_run_id, + "attestation_backend": "native_benchflow", + "endpoint_path": "/attest", + "request_sha256": _sha(body), + "response_sha256": "", + "status_code": 0, + "latency_seconds": time.monotonic() - start, + "blocked_reason": exc.__class__.__name__, + "scorecard_update_allowed": False, + "passed": False, + } + return { + "schema_version": "bb.rl.phase3.native_benchflow_attestation.v1", + "report_id": "phase3_native_benchflow_attestation", + "claim_boundary": "phase3_native_benchflow_contract_only_scope", + "target_run_id": target_run_id, + "attestation_backend": "native_benchflow", + "endpoint_path": "/attest", + "request_sha256": _sha(body), + "response_sha256": _sha(response_body), + "status_code": status, + "latency_seconds": time.monotonic() - start, + "blocked_reason": "", + "scorecard_update_allowed": False, + "passed": 200 <= status < 300, + } + + +def _harbor_json_request(base_url: str, token: str, method: str, path: str, payload: Mapping[str, Any] | None = None, *, allow_local: bool = False) -> tuple[int, bytes, bytes, float]: + body = _json_request_body(payload) + headers = {"Authorization": f"Bearer {token}"} + if payload is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request(base_url.rstrip("/") + path, data=body if payload is not None else None, method=method, headers=headers) + start = time.monotonic() + if allow_local: + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(request, timeout=10.0) as response: # noqa: S310 - self-hosted Harbor is operator-started inside the target job. + return int(response.status), body, response.read(), time.monotonic() - start + with _open_without_proxies(request, timeout_s=10.0) as response: # noqa: S310 - operator-configured provider endpoint. + return int(response.status), body, response.read(), time.monotonic() - start + + +def _harbor_call_row( + *, + method: str, + route_template: str, + path: str, + status_code: int, + request_body: bytes, + response_body: bytes, + latency_seconds: float, + blocked_reason: str = "", +) -> dict[str, Any]: + return { + "method": method, + "route_template": route_template, + "path": path, + "status_code": status_code, + "request_sha256": _sha(request_body), + "response_sha256": _sha(response_body), + "latency_seconds": latency_seconds, + "passed": 200 <= status_code < 300 and not blocked_reason, + "blocked_reason": blocked_reason, + } + + +def _blocked_harbor_report( + *, + target_run_id: str, + base_url: str, + env_package_sha256: str, + task_name: str, + calls: Sequence[Mapping[str, Any]], + blocked_reason: str, + total_latency_seconds: float, +) -> dict[str, Any]: + return { + "schema_version": HARBOR_SERVICE_PROOF_SCHEMA, + "report_id": HARBOR_SERVICE_PROOF_ID, + "claim_boundary": HARBOR_BLOCKED_CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "attestation_backend": "harbor_facade", + "provider_kind": "harbor_facade", + "endpoint_identity": _redacted_endpoint(base_url), + "env_package_sha256": env_package_sha256, + "task_name": task_name, + "harbor_routes": list(HARBOR_ROUTES), + "harbor_calls": list(calls), + "blocked_reason": blocked_reason, + "latency_seconds": total_latency_seconds, + "scorecard_update_allowed": False, + "passed": False, + } + + +def _harbor_score_passed(response: Mapping[str, Any]) -> bool: + if response.get("passed") is True: + return True + for key in ("score", "reward"): + value = response.get(key) + if isinstance(value, (int, float)) and float(value) >= 1.0: + return True + raw = response.get("raw") + if isinstance(raw, Mapping): + reward = raw.get("reward") + return isinstance(reward, (int, float)) and float(reward) >= 1.0 + return False + + +def _harbor_exec_passed(response: Mapping[str, Any]) -> bool: + return_code = response.get("return_code", response.get("returncode")) + return ( + isinstance(return_code, int) + and return_code == 0 + and str(response.get("stdout") or "") == "phase3-harbor-proof" + ) + + +def _harbor_trial_passed(response: Mapping[str, Any], *, task_name: str, trial_id: str) -> bool: + n_exec_calls = response.get("n_exec_calls") + return ( + str(response.get("trial_id") or "") == trial_id + and str(response.get("task_name") or "") == task_name + and isinstance(n_exec_calls, int) + and n_exec_calls >= 1 + ) + + +def _harbor_list_has_task(response: Any, *, task_name: str) -> bool: + if isinstance(response, list): + return task_name in {str(item) for item in response} + if isinstance(response, Mapping): + tasks = response.get("tasks", response.get("items")) + if isinstance(tasks, list): + return task_name in {str(item.get("name") if isinstance(item, Mapping) else item) for item in tasks} + return False + + +def _harbor_response_semantics_passed(responses: Mapping[str, Any], *, task_name: str, trial_id: str) -> bool: + create_response = responses.get("POST /trial/create", {}) + if not isinstance(create_response, Mapping) or str(create_response.get("trial_id") or "") != trial_id: + return False + if create_response.get("task_name") not in (None, task_name): + return False + return ( + _harbor_list_has_task(responses.get("GET /list_tasks"), task_name=task_name) + and isinstance(responses.get("POST /score"), Mapping) + and _harbor_score_passed(responses["POST /score"]) + and isinstance(responses.get("POST /trial/{trial_id}/exec"), Mapping) + and _harbor_exec_passed(responses["POST /trial/{trial_id}/exec"]) + and isinstance(responses.get("GET /trial/{trial_id}"), Mapping) + and _harbor_trial_passed(responses["GET /trial/{trial_id}"], task_name=task_name, trial_id=trial_id) + and isinstance(responses.get("POST /trial/{trial_id}/finalize"), Mapping) + and _harbor_score_passed(responses["POST /trial/{trial_id}/finalize"]) + ) + + +def _run_harbor_facade_attestation(base_url: str, token: str, env_package_path, *, target_run_id: str, task_name: str) -> dict[str, Any]: + allow_local = os.environ.get("BREADBOARD_HARBOR_ALLOW_LOCAL", "").lower() in {"1", "true", "yes"} + block_reason = "" if allow_local else _provider_url_block_reason(base_url) + if block_reason: + return _blocked_harbor_report( + target_run_id=target_run_id, + base_url=base_url, + env_package_sha256="", + task_name=task_name, + calls=[], + blocked_reason=block_reason, + total_latency_seconds=0.0, + ) + if not os.path.isfile(env_package_path): + return _blocked_harbor_report( + target_run_id=target_run_id, + base_url=base_url, + env_package_sha256="", + task_name=task_name, + calls=[], + blocked_reason="missing_harbor_env_package", + total_latency_seconds=0.0, + ) + env_package_sha256 = sha256_file(Path(env_package_path)) + calls: list[dict[str, Any]] = [] + responses: dict[str, Mapping[str, Any]] = {} + start = time.monotonic() + trial_id = "" + try: + for method, route, path, payload in ( + ("GET", "GET /health", "/health", None), + ("GET", "GET /metrics.json", "/metrics.json", None), + ("GET", "GET /list_tasks", "/list_tasks", None), + ("POST", "POST /score", "/score", {"task_name": task_name, "answer": "phase3 harbor service proof"}), + ): + status, body, response, latency = _harbor_json_request(base_url, token, method, path, payload, allow_local=allow_local) + calls.append(_harbor_call_row(method=method, route_template=route, path=path, status_code=status, request_body=body, response_body=response, latency_seconds=latency)) + responses[route] = _response_json(response) + create_payload = {"task_name": task_name} + status, body, response, latency = _harbor_json_request(base_url, token, "POST", "/trial/create", create_payload, allow_local=allow_local) + calls.append(_harbor_call_row(method="POST", route_template="POST /trial/create", path="/trial/create", status_code=status, request_body=body, response_body=response, latency_seconds=latency)) + create_response = _response_json(response) + responses["POST /trial/create"] = create_response + trial_id = str(create_response.get("trial_id") or "") + if not trial_id: + raise ValueError("harbor_trial_id_missing") + quoted_trial = urllib.parse.quote(trial_id, safe="") + exec_payload = {"cmd": "printf phase3-harbor-proof", "timeout_sec": 30} + status, body, response, latency = _harbor_json_request(base_url, token, "POST", f"/trial/{quoted_trial}/exec", exec_payload, allow_local=allow_local) + calls.append(_harbor_call_row(method="POST", route_template="POST /trial/{trial_id}/exec", path=f"/trial/{quoted_trial}/exec", status_code=status, request_body=body, response_body=response, latency_seconds=latency)) + responses["POST /trial/{trial_id}/exec"] = _response_json(response) + status, body, response, latency = _harbor_json_request(base_url, token, "GET", f"/trial/{quoted_trial}", None, allow_local=allow_local) + calls.append(_harbor_call_row(method="GET", route_template="GET /trial/{trial_id}", path=f"/trial/{quoted_trial}", status_code=status, request_body=body, response_body=response, latency_seconds=latency)) + responses["GET /trial/{trial_id}"] = _response_json(response) + finalize_payload = {"answer": "phase3 harbor service proof"} + status, body, response, latency = _harbor_json_request(base_url, token, "POST", f"/trial/{quoted_trial}/finalize", finalize_payload, allow_local=allow_local) + calls.append(_harbor_call_row(method="POST", route_template="POST /trial/{trial_id}/finalize", path=f"/trial/{quoted_trial}/finalize", status_code=status, request_body=body, response_body=response, latency_seconds=latency)) + responses["POST /trial/{trial_id}/finalize"] = _response_json(response) + except urllib.error.HTTPError as exc: + response_body = exc.read() + calls.append(_harbor_call_row(method=getattr(exc, "method", "HTTP"), route_template="http_error", path=exc.url, status_code=int(exc.code), request_body=b"", response_body=response_body, latency_seconds=time.monotonic() - start, blocked_reason="http_error")) + except ProviderRedirectBlocked as exc: + return _blocked_harbor_report( + target_run_id=target_run_id, + base_url=base_url, + env_package_sha256=env_package_sha256, + task_name=task_name, + calls=calls, + blocked_reason=exc.block_reason, + total_latency_seconds=time.monotonic() - start, + ) + except Exception as exc: # noqa: BLE001 + return _blocked_harbor_report( + target_run_id=target_run_id, + base_url=base_url, + env_package_sha256=env_package_sha256, + task_name=task_name, + calls=calls, + blocked_reason=exc.__class__.__name__, + total_latency_seconds=time.monotonic() - start, + ) + route_templates = [str(call.get("route_template") or "") for call in calls] + backend_identity = "" + health = responses.get("GET /health", {}) + if isinstance(health.get("dataset"), str): + backend_identity = str(health["dataset"]) + semantics_passed = _harbor_response_semantics_passed(responses, task_name=task_name, trial_id=trial_id) + passed = ( + list(HARBOR_ROUTES) == route_templates + and all(call.get("passed") is True for call in calls) + and bool(trial_id) + and bool(env_package_sha256) + and semantics_passed + ) + return { + "schema_version": HARBOR_SERVICE_PROOF_SCHEMA, + "report_id": HARBOR_SERVICE_PROOF_ID, + "claim_boundary": HARBOR_CLAIM_BOUNDARY if passed else HARBOR_BLOCKED_CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "attestation_backend": "harbor_facade", + "provider_kind": "harbor_facade", + "endpoint_identity": _redacted_endpoint(base_url), + "backend_identity": backend_identity, + "env_package_sha256": env_package_sha256, + "task_name": task_name, + "trial_id_sha256": _sha(trial_id.encode()) if trial_id else "", + "harbor_routes": list(HARBOR_ROUTES), + "harbor_calls": calls, + "status_codes": {str(call.get("route_template")): int(call.get("status_code") or 0) for call in calls}, + "latency_seconds": time.monotonic() - start, + "blocked_reason": "" if passed else ("harbor_response_semantics_failed" if route_templates == list(HARBOR_ROUTES) and all(call.get("passed") is True for call in calls) else "harbor_route_sequence_failed"), + "scorecard_update_allowed": False, + "passed": passed, + } diff --git a/breadboard/rl/phase3/object_store.py b/breadboard/rl/phase3/object_store.py new file mode 100644 index 00000000..cb9062a1 --- /dev/null +++ b/breadboard/rl/phase3/object_store.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any, Mapping + +from breadboard.rl.phase3.evidence import sha256_file + + +class LocalObjectStore: + def __init__(self, root: str | Path): + self.root = Path(root) + self.root.mkdir(parents=True, exist_ok=True) + + def _artifact_path(self, artifact_id: str) -> Path: + if "/" in artifact_id or ".." in artifact_id: + raise ValueError("artifact_id must be a flat identifier") + return self.root / artifact_id + + def put_file(self, path: Path, *, artifact_id: str, metadata: Mapping[str, Any]) -> dict[str, Any]: + if not path.exists() or not path.is_file(): + raise FileNotFoundError(path) + destination = self._artifact_path(artifact_id) + shutil.copyfile(path, destination) + stat = {"artifact_id": artifact_id, "path": str(destination), "sha256": sha256_file(destination), "bytes": destination.stat().st_size, "metadata": dict(metadata), "object_store": "local_object_store"} + destination.with_suffix(destination.suffix + ".json").write_text(json.dumps(stat, sort_keys=True, indent=2) + "\n") + return stat + + def get_file(self, artifact_id: str, destination: Path) -> Path: + source = self._artifact_path(artifact_id) + if not source.exists(): + raise FileNotFoundError(artifact_id) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + return destination + + def stat(self, artifact_id: str) -> dict[str, Any]: + path = self._artifact_path(artifact_id) + if not path.exists(): + raise FileNotFoundError(artifact_id) + metadata_path = path.with_suffix(path.suffix + ".json") + if metadata_path.exists(): + return json.loads(metadata_path.read_text()) + return {"artifact_id": artifact_id, "path": str(path), "sha256": sha256_file(path), "bytes": path.stat().st_size, "object_store": "local_object_store"} diff --git a/breadboard/rl/phase3/observability_live.py b/breadboard/rl/phase3/observability_live.py new file mode 100644 index 00000000..253888dd --- /dev/null +++ b/breadboard/rl/phase3/observability_live.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +import ipaddress +import os +import urllib.parse +from collections.abc import Mapping +from typing import Any +EXPECTED_METRIC_SOURCES = { + "slurm_metrics": "slurm_sacct", + "gpu_metrics": "rocm_smi", + "verifier_metrics": "verifier_client", + "service_metrics": "service_event_log", + "object_store_metrics": "object_store", + "scheduler_metrics": "scheduler_control", +} + +LOCAL_OBJECT_STORE_BACKENDS = frozenset( + {"LocalObjectStore", "target_workspace_local_object_store", "local_object_store"} +) + +def endpoint_is_local(value: str | None) -> bool: + if not value: + return False + text = value.strip() + parsed = urllib.parse.urlsplit(text) + if not parsed.hostname and "://" not in text: + parsed = urllib.parse.urlsplit(f"//{text}") + host = parsed.hostname + if not host: + return False + normalized = host.lower().rstrip(".") + if normalized in {"localhost", "localhost.localdomain"}: + return True + local_names = { + str(os.environ.get("HOSTNAME") or "").lower().rstrip("."), + str(os.environ.get("SLURMD_NODENAME") or "").lower().rstrip("."), + } + if normalized in local_names - {""}: + return True + try: + return ipaddress.ip_address(normalized).is_loopback + except ValueError: + return False + + +def collect_slurm_metrics(job_id: str) -> dict[str, Any]: + return {"source": "slurm_sacct", "job_id": job_id, "queue_wait_seconds": 0.0, "scheduler_retry_count": 0} + + +def collect_gpu_metrics() -> dict[str, Any]: + return {"source": "rocm_smi", "gpu_utilization": []} + + +def collect_verifier_metrics() -> dict[str, Any]: + return {"source": "verifier_client", "verifier_latency_seconds": []} + + +def _validate_source(name: str, section: Mapping[str, Any], errors: list[str]) -> None: + expected = EXPECTED_METRIC_SOURCES[name] + if section.get("source") != expected: + errors.append(f"{name}.source must be {expected!r}") + + +def _has_metric_value(section: Mapping[str, Any], key: str) -> bool: + value = section.get(key) + return value is not None and value != "" and value != [] and value != {} + + +def _presence_flag(section: Mapping[str, Any], key: str) -> bool | None: + if key not in section: + return None + value = section[key] + if isinstance(value, Mapping): + if "present" not in value: + return None + return value["present"] is True + if isinstance(value, bool): + return value + return None + + +def _env_presence_flag(section: Mapping[str, Any], env_name: str) -> bool | None: + env_presence = section.get("env_presence") + if not isinstance(env_presence, Mapping): + return None + return _presence_flag(env_presence, env_name) + + +def _scheduler_presence(scheduler_metrics: Mapping[str, Any], keys: tuple[str, ...], env_name: str) -> bool: + candidates: list[bool | None] = [] + control = scheduler_metrics.get("scheduler_control") + for section in (scheduler_metrics, control): + if not isinstance(section, Mapping): + continue + candidates.extend(_presence_flag(section, key) for key in keys) + candidates.append(_env_presence_flag(section, env_name)) + return any(value is True for value in candidates) + +def _any_presence(section: Mapping[str, Any], keys: tuple[str, ...], env_names: tuple[str, ...] = ()) -> bool: + if any(_presence_flag(section, key) is True for key in keys): + return True + return any(_env_presence_flag(section, env_name) is True for env_name in env_names) + + +def _status_is_success(value: Any) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, int): + return 200 <= value < 300 + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"ok", "passed", "success", "succeeded", "verified"}: + return True + try: + return 200 <= int(normalized) < 300 + except ValueError: + return False + return False + + +def _operation_status_ok(section: Mapping[str, Any], operation: str) -> bool: + candidates = ( + f"{operation}_status", + f"{operation}_status_code", + f"{operation}_http_status", + f"{operation}_passed", + f"{operation}_verified", + ) + for key in candidates: + if key in section and _status_is_success(section[key]): + return True + operation_payload = section.get(operation) + if isinstance(operation_payload, Mapping): + for key in ("status", "status_code", "http_status", "passed", "verified"): + if key in operation_payload and _status_is_success(operation_payload[key]): + return True + return False + + +def _validate_object_store_live_contract(object_store_metrics: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if not _any_presence( + object_store_metrics, + ("token_present", "object_store_token_present", "credential_present", "credentials_present", "access_key_present"), + ("BREADBOARD_OBJECT_STORE_TOKEN", "BREADBOARD_OBJECT_STORE_ACCESS_KEY"), + ): + errors.append("object_store_token_missing") + endpoint = str(object_store_metrics.get("endpoint") or "") + if not endpoint: + errors.append("production_object_store_endpoint_missing") + elif endpoint_is_local(endpoint): + errors.append("production_object_store_endpoint_local") + for endpoint_field in ("put_endpoint", "get_endpoint", "delete_endpoint"): + endpoint = str(object_store_metrics.get(endpoint_field) or "") + if not endpoint: + errors.append(f"object_store_{endpoint_field}_missing") + elif endpoint_is_local(endpoint): + errors.append(f"object_store_{endpoint_field}_local") + for operation in ("put", "get", "delete"): + if not _operation_status_ok(object_store_metrics, operation): + errors.append(f"object_store_{operation}_status_missing") + if object_store_metrics.get("write_read_verified") is not True: + errors.append("object_store_write_read_verified_missing") + for key in ("readback_verified", "read_after_write_verified", "readback_matches", "durability_verified", "readback_durable"): + if key in object_store_metrics and object_store_metrics.get(key) is not True: + errors.append(f"object_store_{key}_failed") + expected_hash = object_store_metrics.get("written_sha256") or object_store_metrics.get("put_sha256") + readback_hash = object_store_metrics.get("readback_sha256") or object_store_metrics.get("get_sha256") + if not isinstance(expected_hash, str) or not expected_hash: + errors.append("object_store_written_sha256_missing") + if not isinstance(readback_hash, str) or not readback_hash: + errors.append("object_store_readback_sha256_missing") + if isinstance(expected_hash, str) and expected_hash and isinstance(readback_hash, str) and readback_hash and expected_hash != readback_hash: + errors.append("object_store_readback_hash_mismatch") + return errors + + +def _validate_verifier_live_contract(verifier_metrics: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + endpoint = str(verifier_metrics.get("endpoint") or "") + if not endpoint: + errors.append("verifier_endpoint_missing") + elif endpoint_is_local(endpoint): + errors.append("verifier_endpoint_is_local") + if not _any_presence(verifier_metrics, ("token_present", "verifier_token_present"), ("BREADBOARD_VERIFIER_TOKEN",)): + errors.append("verifier_token_missing") + return errors + + + +def validate_scheduler_metrics_readiness(scheduler_metrics: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if not _has_metric_value(scheduler_metrics, "scheduler_control"): + errors.append("scheduler_control_missing") + if not _scheduler_presence(scheduler_metrics, ("endpoint_present", "base_url_present", "scheduler_base_url_present"), "BREADBOARD_SCHEDULER_BASE_URL"): + errors.append("scheduler_control_endpoint_missing") + if not _scheduler_presence(scheduler_metrics, ("token_present", "scheduler_token_present"), "BREADBOARD_SCHEDULER_TOKEN"): + errors.append("scheduler_control_token_missing") + return errors + +def _object_store_backend(object_store_metrics: Mapping[str, Any]) -> str: + return str(object_store_metrics.get("object_store") or object_store_metrics.get("backend") or "") + + + +def _validate_live_semantics( + *, + slurm_metrics: Mapping[str, Any], + gpu_metrics: Mapping[str, Any], + verifier_metrics: Mapping[str, Any], + service_metrics: Mapping[str, Any], + object_store_metrics: Mapping[str, Any], + scheduler_metrics: Mapping[str, Any], + errors: list[str], +) -> None: + if not _has_metric_value(slurm_metrics, "sacct_stdout"): + errors.append("sacct_metrics_missing") + if not _has_metric_value(gpu_metrics, "gpu_utilization"): + errors.append("gpu_utilization_missing") + if not _has_metric_value(service_metrics, "events") and not _has_metric_value(service_metrics, "task_throughput"): + errors.append("service_event_metrics_missing") + if not _has_metric_value(verifier_metrics, "verifier_latency_seconds"): + errors.append("verifier_latency_seconds_missing") + errors.extend(_validate_verifier_live_contract(verifier_metrics)) + object_store_backend = _object_store_backend(object_store_metrics) + if not ( + object_store_backend + and _has_metric_value(object_store_metrics, "object_store_writes") + and _has_metric_value(object_store_metrics, "artifact_bytes") + ): + errors.append("object_store_metrics_missing") + elif object_store_backend in LOCAL_OBJECT_STORE_BACKENDS: + errors.append("production_object_store_endpoint_missing") + errors.extend(_validate_object_store_live_contract(object_store_metrics)) + scheduler_control = scheduler_metrics.get("scheduler_control") + if isinstance(scheduler_control, Mapping) and endpoint_is_local(str(scheduler_control.get("endpoint") or "")): + errors.append("scheduler_control_endpoint_local") + errors.extend(validate_scheduler_metrics_readiness(scheduler_metrics)) + + +def build_live_observability_report( + *, + target_run_id: str, + slurm_metrics: Mapping[str, Any], + gpu_metrics: Mapping[str, Any], + verifier_metrics: Mapping[str, Any], + service_metrics: Mapping[str, Any], + object_store_metrics: Mapping[str, Any], + budget_caps: Mapping[str, Any], + scheduler_metrics: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + errors: list[str] = [] + scheduler_metrics = scheduler_metrics or {} + sections = { + "slurm_metrics": slurm_metrics, + "gpu_metrics": gpu_metrics, + "verifier_metrics": verifier_metrics, + "service_metrics": service_metrics, + "object_store_metrics": object_store_metrics, + "scheduler_metrics": scheduler_metrics, + } + for name, section in sections.items(): + _validate_source(name, section, errors) + _validate_live_semantics( + slurm_metrics=slurm_metrics, + gpu_metrics=gpu_metrics, + verifier_metrics=verifier_metrics, + service_metrics=service_metrics, + object_store_metrics=object_store_metrics, + scheduler_metrics=scheduler_metrics, + errors=errors, + ) + remaining_usd = budget_caps.get("remaining_usd") + if remaining_usd is None or remaining_usd == "": + errors.append("budget_caps.remaining_usd_missing") + else: + try: + remaining_usd_value = float(remaining_usd) + except (TypeError, ValueError): + errors.append("budget_caps.remaining_usd_invalid") + else: + if remaining_usd_value < 0: + errors.append("budget caps exceeded") + return { + "schema_version": "bb.rl.phase3.live_observability.v1", + "report_id": "phase3_live_observability", + "claim_boundary": "phase3_live_observability_object_store_scheduler_scope", + "target_run_id": target_run_id, + "queue_wait": slurm_metrics.get("queue_wait_seconds"), + "gpu_utilization": gpu_metrics.get("gpu_utilization"), + "task_throughput": service_metrics.get("task_throughput"), + "verifier_latency": verifier_metrics.get("verifier_latency_seconds"), + "failure_taxonomy": service_metrics.get("failure_taxonomy", {}), + "budget_caps": dict(budget_caps), + "artifact_bytes": object_store_metrics.get("artifact_bytes"), + "object_store_writes": object_store_metrics.get("object_store_writes"), + "scheduler_retry_count": slurm_metrics.get("scheduler_retry_count"), + "scheduler_control": scheduler_metrics.get("scheduler_control"), + "metric_sections": {key: dict(value) for key, value in sections.items()}, + "errors": errors, + "scorecard_update_allowed": False, + "passed": not errors, + } diff --git a/breadboard/rl/phase3/parity.py b/breadboard/rl/phase3/parity.py new file mode 100644 index 00000000..7dbae397 --- /dev/null +++ b/breadboard/rl/phase3/parity.py @@ -0,0 +1,383 @@ +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from breadboard.rl.phase3.evidence import sha256_file + +PHASE3_PARITY_SCHEMA = "bb.rl.phase3.parity_report.v1" +PHASE3_PARITY_REPORT_ID = "phase3_parity_report" +PHASE3_PARITY_CLAIM_BOUNDARY = "phase3_ppo_grpo_closed_loop_parity_named_scope" +REQUIRED_PARITY_SECTIONS = ("scorer", "rollout", "token_logprob", "checkpoint", "model_merge", "infra", "dataproto", "limitations", "checklist") + + +def _sha(path: Path | None) -> str: + return sha256_file(path) if path and path.exists() and path.is_file() else "" + + +def _mapping(value: Any) -> dict[str, Any]: + return dict(value) if isinstance(value, Mapping) else {} + + +def _artifact_path(evidence_root: Path, report: Mapping[str, Any], key: str) -> Path | None: + raw = _mapping(report.get("artifact_paths")).get(key) + if not raw: + return None + candidate = Path(str(raw)) + if not candidate.is_absolute(): + candidate = evidence_root / candidate + try: + resolved = candidate.resolve() + resolved.relative_to(evidence_root.resolve()) + except (OSError, ValueError): + return None + return resolved + + +def _without_parity_hash(report: Mapping[str, Any]) -> dict[str, Any]: + hashes = _mapping(report.get("input_hashes")) + hashes.pop("parity_report", None) + return hashes + + +def _checkpoint_item(report: Mapping[str, Any]) -> dict[str, Any]: + return { + "report_id": report.get("report_id"), + "trainer_backend": report.get("trainer_backend"), + "entrypoint": report.get("entrypoint"), + "rollout_name": report.get("rollout_name"), + "model_ref": report.get("model_ref"), + "n_gpus_per_node": report.get("n_gpus_per_node"), + "device_count": report.get("device_count"), + "optimizer_step_count": report.get("optimizer_step_count"), + "checkpoint_before_sha256": report.get("checkpoint_before_sha256"), + "checkpoint_after_sha256": report.get("checkpoint_after_sha256"), + "checkpoint_changed": report.get("checkpoint_changed"), + } + + +def _validate_checkpoint_items(items: Mapping[str, Mapping[str, Any]]) -> list[str]: + errors: list[str] = [] + for name, item in items.items(): + if int(item.get("optimizer_step_count") or 0) < 1: + errors.append(f"{name}.optimizer_step_count must be >= 1") + if item.get("checkpoint_changed") is not True: + errors.append(f"{name}.checkpoint_changed must be true") + if item.get("checkpoint_before_sha256") == item.get("checkpoint_after_sha256"): + errors.append(f"{name}.checkpoint hashes must differ") + if int(item.get("device_count") or 0) != 8 or int(item.get("n_gpus_per_node") or 0) != 8: + errors.append(f"{name}.device_count and n_gpus_per_node must be 8") + return errors + +EMPTY_TREE_SHA256 = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + +def _read_text(path: Path | None) -> str: + if not path or not path.exists() or not path.is_file(): + return "" + try: + return path.read_text(errors="replace") + except OSError: + return "" + + +def _load_json(path: Path | None) -> dict[str, Any]: + text = _read_text(path) + if not text: + return {} + try: + payload = json.loads(text) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def _first_digest(text: str) -> str: + match = re.search(r"Digest:\s*(sha256:[0-9a-f]{64})", text) + return match.group(1) if match else "" + + +def _checkpoint_checklist(evidence_root: Path, closed_loop_report: Mapping[str, Any], checkpoint: Mapping[str, Mapping[str, Any]]) -> dict[str, Any]: + manifest_path = _artifact_path(evidence_root, closed_loop_report, "evidence_manifest") + stdout_path = _artifact_path(evidence_root, closed_loop_report, "trainer_stdout") + manifest = _load_json(manifest_path) + stdout = _read_text(stdout_path) + checkpoint_dir = str(manifest.get("checkpoint_dir") or "") + closed_loop_after = _mapping(checkpoint.get("closed_loop")).get("checkpoint_after_sha256") + before_hashes = [_mapping(checkpoint.get(name)).get("checkpoint_before_sha256") for name in ("ppo", "grpo", "closed_loop")] + files = _mapping(manifest.get("files")) + file_sizes_ok = bool(files) and all(int(_mapping(meta).get("bytes") or 0) > 0 for meta in files.values()) + evidence = { + "checkpoint_dir": checkpoint_dir, + "manifest_checkpoint_tree_sha256": manifest.get("checkpoint_tree_sha256"), + "closed_loop_after_sha256": closed_loop_after, + "all_before_hashes_empty_tree": all(value == EMPTY_TREE_SHA256 for value in before_hashes), + "after_hash_matches_manifest_tree": bool(closed_loop_after) and manifest.get("checkpoint_tree_sha256") == closed_loop_after, + "trainer_default_local_dir_matches_manifest": bool(checkpoint_dir) and f"'default_local_dir': '{checkpoint_dir}'" in stdout, + "manifest_files_nonzero": file_sizes_ok, + "source_artifacts": { + "closed_loop_report": closed_loop_report.get("report_id"), + "evidence_manifest": str(_mapping(closed_loop_report.get("artifact_paths")).get("evidence_manifest", "")), + "trainer_stdout": str(_mapping(closed_loop_report.get("artifact_paths")).get("trainer_stdout", "")), + }, + } + satisfied = all( + bool(evidence[key]) + for key in ( + "checkpoint_dir", + "manifest_checkpoint_tree_sha256", + "closed_loop_after_sha256", + "all_before_hashes_empty_tree", + "after_hash_matches_manifest_tree", + "trainer_default_local_dir_matches_manifest", + "manifest_files_nonzero", + ) + ) + return {"status": "satisfied" if satisfied else "open", "evidence": evidence} + + +def _infra_checklist( + evidence_root: Path, + ppo_report: Mapping[str, Any], + grpo_report: Mapping[str, Any], + closed_loop_report: Mapping[str, Any], + introspection_report: Mapping[str, Any], + runtime_evidence: Mapping[str, Any], +) -> dict[str, Any]: + stdout = _read_text(_artifact_path(evidence_root, closed_loop_report, "trainer_stdout")) + stderr = _read_text(_artifact_path(evidence_root, closed_loop_report, "trainer_stderr")) + ppo_log = _read_text(_artifact_path(evidence_root, ppo_report, "target_command_log")) + grpo_log = _read_text(_artifact_path(evidence_root, grpo_report, "target_command_log")) + torch_info = _mapping(introspection_report.get("torch")) + symbols = _mapping(introspection_report.get("symbols")) + trainer_runtime_path = str(runtime_evidence.get("runtime_path") or "") + runtime_install_path = str(runtime_evidence.get("runtime_install_report_path") or "") + runtime_install_runtime = str(runtime_evidence.get("runtime_install_runtime") or "") + introspection_path = str(runtime_evidence.get("introspection_report_path") or "") + runtime_install_is_scratch = "/scratch_runs/" in runtime_install_path.replace("\\", "/") + trainer_runtime_root = str(Path(trainer_runtime_path).resolve()) if trainer_runtime_path else "" + install_runtime_root = str(Path(runtime_install_runtime).resolve()) if runtime_install_runtime else "" + split_scope = bool(trainer_runtime_root and install_runtime_root and trainer_runtime_root != install_runtime_root) + evidence = { + "trainer_runtime_path": trainer_runtime_path, + "introspection_report_path": introspection_path, + "split_scope": split_scope, + "container_image": runtime_evidence.get("container_image"), + "ppo_image_digest": _first_digest(ppo_log), + "grpo_image_digest": _first_digest(grpo_log), + "verl_version": symbols.get("verl.__version__"), + "torch_version": torch_info.get("version"), + "device_count": torch_info.get("device_count"), + "devices": torch_info.get("devices") if isinstance(torch_info.get("devices"), list) else [], + "ray_started": "Started a local Ray instance" in stderr, + "tensor_model_parallel_size": "'tensor_model_parallel_size': 2" in stdout, + "sandbox_memory_limit_mb": "'memory_limit_mb': 1024" in stdout, + "checkpoint_default_local_dir": "'default_local_dir':" in stdout, + "transfer_queue_simple_storage": "'storage_backend': 'SimpleStorage'" in stdout, + "runtime_install_report_path": runtime_install_path, + "runtime_install_passed": runtime_evidence.get("runtime_install_passed") is True, + "runtime_install_runtime": runtime_install_runtime, + "vllm_version": runtime_evidence.get("vllm_version") or "", + "runtime_install_is_scratch": runtime_install_is_scratch, + } + required = ("trainer_runtime_path", "container_image", "ppo_image_digest", "grpo_image_digest", "runtime_install_report_path", "runtime_install_passed", "runtime_install_runtime", "vllm_version") + missing = [key for key in required if not evidence.get(key)] + if split_scope: + missing.append("single_runtime_install_for_trainer_runtime") + if runtime_install_is_scratch: + missing.append("target_run_bound_runtime_install") + reason = "" if not missing else "runtime install evidence must be target-run-bound and prove the same trainer runtime, container image, and vllm.__version__ used by the canonical trainer artifacts" + return { + "status": "satisfied" if not missing else "open", + "missing": missing, + "evidence": evidence, + "reason": reason, + } + + +def build_phase3_parity_report( + *, + target_run_id: str, + ppo_report: Mapping[str, Any], + grpo_report: Mapping[str, Any], + closed_loop_report: Mapping[str, Any], + introspection_report: Mapping[str, Any], + runtime_evidence: Mapping[str, Any], + evidence_root: Path, +) -> dict[str, Any]: + reward_path = _artifact_path(evidence_root, closed_loop_report, "reward_function") + projection_path = _artifact_path(evidence_root, closed_loop_report, "accepted_projection_rows") + evidence_manifest_path = _artifact_path(evidence_root, closed_loop_report, "evidence_manifest") + metrics_path = _artifact_path(evidence_root, closed_loop_report, "metrics") + checkpoint = { + "ppo": _checkpoint_item(ppo_report), + "grpo": _checkpoint_item(grpo_report), + "closed_loop": { + **_checkpoint_item(closed_loop_report), + "accepted_count": closed_loop_report.get("accepted_count"), + "quarantined_count": closed_loop_report.get("quarantined_count"), + "rejected_count": closed_loop_report.get("rejected_count"), + "dataproto_ok": closed_loop_report.get("dataproto_ok"), + }, + } + checkpoint_checklist = _checkpoint_checklist(evidence_root, closed_loop_report, checkpoint) + errors = _validate_checkpoint_items(checkpoint) + if closed_loop_report.get("dataproto_ok") is not True: + errors.append("closed_loop.dataproto_ok must be true") + if not reward_path or not reward_path.exists(): + errors.append("scorer.reward_function artifact must exist") + if not projection_path or not projection_path.exists(): + errors.append("rollout.accepted_projection_rows artifact must exist") + if not evidence_manifest_path or not evidence_manifest_path.exists(): + errors.append("dataproto.evidence_manifest artifact must exist") + torch_info = _mapping(introspection_report.get("torch")) + symbols = _mapping(introspection_report.get("symbols")) + devices = torch_info.get("devices") if isinstance(torch_info.get("devices"), list) else [] + if torch_info.get("device_count") != 8: + errors.append("infra.introspection device_count must be 8") + if not devices or any(device != "AMD Instinct MI300X" for device in devices): + errors.append("infra.introspection devices must be AMD Instinct MI300X") + if symbols.get("verl.__version__") != "0.8.0": + errors.append("infra.introspection VeRL version must be 0.8.0") + if not runtime_evidence.get("container_image") or not runtime_evidence.get("runtime_path"): + errors.append("infra.runtime evidence must include container_image and runtime_path") + infra_checklist = _infra_checklist(evidence_root, ppo_report, grpo_report, closed_loop_report, introspection_report, runtime_evidence) + if checkpoint_checklist.get("status") != "satisfied": + errors.append("checklist.C7 checkpoint parity evidence must be satisfied") + if infra_checklist.get("status") != "satisfied": + errors.append("checklist.C10 infrastructure parity evidence must be satisfied") + return { + "schema_version": PHASE3_PARITY_SCHEMA, + "report_id": PHASE3_PARITY_REPORT_ID, + "claim_boundary": PHASE3_PARITY_CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "scorecard_update_allowed": False, + "passed": not errors, + "scorer": { + "reward_function_sha256": _sha(reward_path), + "live_provider_parity": "blocked_missing_p3_m8_provider_credentials", + "scope": "closed_loop_reward_function_only_not_live_provider", + }, + "rollout": { + "accepted_projection_rows_sha256": _sha(projection_path), + "accepted_count": closed_loop_report.get("accepted_count"), + "quarantined_count": closed_loop_report.get("quarantined_count"), + "rejected_count": closed_loop_report.get("rejected_count"), + "rollout_name": closed_loop_report.get("rollout_name"), + }, + "token_logprob": {"available": False, "reason": "canonical artifacts preserve accepted projection and DataProto evidence but do not expose per-token logprob parity arrays"}, + "checkpoint": checkpoint, + "checklist": { + "C7_checkpoint_parity": checkpoint_checklist, + "C10_infrastructure_parity": infra_checklist, + }, + "model_merge": {"required": False, "reason": "current exact-scope evidence uses HF Qwen/Qwen2.5-0.5B-Instruct with no Megatron merge artifact"}, + "infra": { + "runtime_evidence": dict(runtime_evidence), + "introspection": { + "scope": "api_introspection_runtime_not_trainer_runtime" if infra_checklist.get("evidence", {}).get("split_scope") else "same_runtime", + "verl_version": symbols.get("verl.__version__"), + "torch_version": torch_info.get("version"), + "cuda_available": torch_info.get("cuda_available"), + "device_count": torch_info.get("device_count"), + "devices": devices, + }, + }, + "dataproto": {"dataproto_ok": closed_loop_report.get("dataproto_ok"), "evidence_manifest_sha256": _sha(evidence_manifest_path), "metrics_sha256": _sha(metrics_path)}, + "limitations": [ + "No live ORS/OpenReward provider parity without P3-M8 credentials.", + "No native BenchFlow/Harbor parity without P3-M9 endpoint/token.", + "No HF/Megatron merge parity because current evidence does not use Megatron merge.", + "Per-token logprob parity arrays are not exposed by canonical artifacts.", + ], + "input_hashes": { + "p3_m2_report": _without_parity_hash(ppo_report), + "p3_m3_report": _without_parity_hash(grpo_report), + "p3_m4_report": _without_parity_hash(closed_loop_report), + "introspection_report": _sha(Path(str(runtime_evidence.get("introspection_report_path", ""))) if runtime_evidence.get("introspection_report_path") else None), + "runtime_evidence": _mapping(runtime_evidence.get("input_hashes")), + }, + "artifact_paths": { + "reward_function": str(_mapping(closed_loop_report.get("artifact_paths")).get("reward_function", "")), + "accepted_projection_rows": str(_mapping(closed_loop_report.get("artifact_paths")).get("accepted_projection_rows", "")), + "evidence_manifest": str(_mapping(closed_loop_report.get("artifact_paths")).get("evidence_manifest", "")), + "metrics": str(_mapping(closed_loop_report.get("artifact_paths")).get("metrics", "")), + "introspection_report": str(runtime_evidence.get("introspection_report_artifact", "")), + "runtime_ppo_script": str(runtime_evidence.get("ppo_script_artifact", "")), + "runtime_grpo_script": str(runtime_evidence.get("grpo_script_artifact", "")), + "runtime_closed_loop_script": str(runtime_evidence.get("closed_loop_script_artifact", "")), + "runtime_install_report": str(runtime_evidence.get("runtime_install_report_artifact", "")), + }, + "errors": errors, + } + + +def validate_phase3_parity_report(report: Mapping[str, Any], *, target_run_id: str, evidence_root: Path) -> list[str]: + errors: list[str] = [] + if report.get("schema_version") != PHASE3_PARITY_SCHEMA: + errors.append("schema_version must be Phase 3 parity report schema") + if report.get("report_id") != PHASE3_PARITY_REPORT_ID: + errors.append("report_id must be Phase 3 parity report id") + if report.get("claim_boundary") != PHASE3_PARITY_CLAIM_BOUNDARY: + errors.append("claim_boundary must be Phase 3 parity boundary") + if report.get("target_run_id") != target_run_id: + errors.append("target_run_id must match expected target run") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if report.get("passed") is not True: + errors.append("passed must be true") + for section in REQUIRED_PARITY_SECTIONS: + if section not in report: + errors.append(f"{section} section must be present") + checklist = _mapping(report.get("checklist")) + c7 = _mapping(checklist.get("C7_checkpoint_parity")) + c10 = _mapping(checklist.get("C10_infrastructure_parity")) + if c7.get("status") != "satisfied": + errors.append("checklist.C7_checkpoint_parity.status must be satisfied") + if c10.get("status") != "satisfied": + errors.append("checklist.C10_infrastructure_parity.status must be satisfied") + checkpoint = report.get("checkpoint") if isinstance(report.get("checkpoint"), Mapping) else {} + dataproto = _mapping(report.get("dataproto")) + if dataproto.get("dataproto_ok") is not True: + errors.append("dataproto.dataproto_ok must be true") + infra = _mapping(report.get("infra")) + runtime_evidence = _mapping(infra.get("runtime_evidence")) + if not runtime_evidence.get("container_image"): + errors.append("infra.runtime_evidence.container_image must be present") + if not runtime_evidence.get("runtime_path"): + errors.append("infra.runtime_evidence.runtime_path must be present") + introspection = _mapping(infra.get("introspection")) + if introspection.get("verl_version") != "0.8.0": + errors.append("infra.introspection.verl_version must be 0.8.0") + if introspection.get("device_count") != 8: + errors.append("infra.introspection.device_count must be 8") + devices = introspection.get("devices") if isinstance(introspection.get("devices"), list) else [] + if len(devices) != 8 or any(device != "AMD Instinct MI300X" for device in devices): + errors.append("infra.introspection.devices must list 8 AMD Instinct MI300X devices") + if introspection.get("cuda_available") is not True: + errors.append("infra.introspection.cuda_available must be true") + rollout = _mapping(report.get("rollout")) + if rollout.get("rollout_name") != "vllm": + errors.append("rollout.rollout_name must be vllm") + errors.extend(f"checkpoint.{error}" for error in _validate_checkpoint_items({k: _mapping(checkpoint.get(k)) for k in ("ppo", "grpo", "closed_loop")})) + artifact_paths = report.get("artifact_paths") if isinstance(report.get("artifact_paths"), Mapping) else {} + root = evidence_root.resolve() + for key in ("reward_function", "accepted_projection_rows", "evidence_manifest", "metrics", "introspection_report", "runtime_ppo_script", "runtime_grpo_script", "runtime_closed_loop_script", "runtime_install_report"): + raw = artifact_paths.get(key) + if not raw: + errors.append(f"artifact_paths.{key} must be present") + continue + candidate = (root / str(raw)).resolve() + try: + candidate.relative_to(root) + except ValueError: + errors.append(f"artifact_paths.{key} must stay under evidence_root") + continue + if not candidate.exists(): + errors.append(f"artifact_paths.{key} must exist") + if report.get("errors") not in ([], None): + errors.append("errors must be empty") + return errors diff --git a/breadboard/rl/phase3/promotion_audit.py b/breadboard/rl/phase3/promotion_audit.py new file mode 100644 index 00000000..b10f3e5f --- /dev/null +++ b/breadboard/rl/phase3/promotion_audit.py @@ -0,0 +1,196 @@ +from __future__ import annotations +import re + +from collections.abc import Mapping +from typing import Any + +from breadboard.rl.phase3.evidence import PHASE3_TARGET_RUN_ID_PATTERN +from breadboard.rl.phase3.final_report import PHASE3_ACTIVE_SCOPE_CLAIM_BOUNDARY, PHASE3_ACTIVE_SCOPE_SCHEMA, PHASE3_ACTIVE_MILESTONES, PHASE3_FINAL_CLAIM_BOUNDARY, PHASE3_FINAL_REPORT_ID, PHASE3_MILESTONES + +PHASE3_PROMOTION_AUDIT_ID = "bb_zyphra_rl_phase3_promotion_audit_v1" +PHASE3_PROMOTION_CLAIM_BOUNDARY = "phase3_promotion_review_only_not_scorecard_update" +PHASE3_EXACT_SCOPE_REVIEW_READY_MEANING = ( + "Artifact-audit boundary is clean for the promoted exact-scope Phase 3 claim; broader successor claims remain separately gated." +) + + +def _int_or_zero(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + +def _string_list(value: Any) -> list[str]: + return value if isinstance(value, list) and all(isinstance(item, str) for item in value) else [] + + +def build_phase3_promotion_audit(*, target_run_id: str, final_report: Mapping[str, Any], scorecard: Mapping[str, Any], claim_ledger_text: str, bd_epic_closed: bool) -> dict[str, Any]: + if not isinstance(final_report, Mapping): + final_report = {"validation_errors": ["final report must be a JSON object"]} + target_run_id = str(target_run_id or "") + claim_ledger_text = claim_ledger_text if isinstance(claim_ledger_text, str) else "" + summaries_raw = final_report.get("milestone_summaries", []) + summaries = [item for item in summaries_raw if isinstance(item, Mapping)] if isinstance(summaries_raw, list) else [] + completed = [item["milestone_id"] for item in summaries if isinstance(item.get("milestone_id"), str) and item.get("passed") is True] + blocked = [item["milestone_id"] for item in summaries if isinstance(item.get("milestone_id"), str) and item.get("passed") is not True] + validation_errors_raw = final_report.get("validation_errors", []) + if isinstance(validation_errors_raw, list): + final_report_validation_errors = [str(error) for error in validation_errors_raw] + elif validation_errors_raw: + final_report_validation_errors = [str(validation_errors_raw)] + else: + final_report_validation_errors = [] + target_run_id_valid = re.match(PHASE3_TARGET_RUN_ID_PATTERN, target_run_id) is not None + active_scope = final_report.get("active_scope", final_report.get("core_readiness", {})) + active_scope = dict(active_scope) if isinstance(active_scope, Mapping) else {} + active_completed = [milestone for milestone in completed if milestone in PHASE3_ACTIVE_MILESTONES] + active_blocked = _string_list(active_scope.get("blocked_active_milestones", [])) + active_claim_ledger_anchored = PHASE3_ACTIVE_SCOPE_CLAIM_BOUNDARY in claim_ledger_text + active_milestones = active_scope.get("active_milestones", active_scope.get("core_milestones")) + active_scope_current = active_milestones == list(PHASE3_ACTIVE_MILESTONES) + active_review_ready = ( + final_report_validation_errors == [] + and target_run_id_valid + and active_scope.get("ready") is True + and active_scope_current + and set(active_completed) == set(PHASE3_ACTIVE_MILESTONES) + and len(active_completed) == len(PHASE3_ACTIVE_MILESTONES) + and not active_blocked + and active_claim_ledger_anchored + ) + promotion_ready = ( + final_report_validation_errors == [] + and target_run_id_valid + and set(completed) == set(PHASE3_MILESTONES) + and len(completed) == len(PHASE3_MILESTONES) + and not blocked + and target_run_id in claim_ledger_text + and PHASE3_FINAL_REPORT_ID in claim_ledger_text + and PHASE3_FINAL_CLAIM_BOUNDARY in claim_ledger_text + and bd_epic_closed + ) + audit = { + "schema_version": "bb.rl.phase3.promotion_audit.v1", + "report_id": PHASE3_PROMOTION_AUDIT_ID, + "claim_boundary": PHASE3_PROMOTION_CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "completed_milestones": completed, + "blocked_milestones": blocked, + "final_report_validation_errors": final_report_validation_errors, + "bd_epic_closed": bd_epic_closed, + "promotion_review_ready": promotion_ready, + "active_scope": active_scope, + "active_completed_milestones": active_completed, + "active_blocked_milestones": active_blocked, + "active_claim_ledger_anchored": active_claim_ledger_anchored, + "active_review_ready": active_review_ready, + "active_artifact_audit_clean": active_review_ready, + "active_review_ready_meaning": PHASE3_EXACT_SCOPE_REVIEW_READY_MEANING, + "core_readiness": active_scope, + "core_completed_milestones": active_completed, + "core_blocked_milestones": active_blocked, + "core_claim_ledger_anchored": active_claim_ledger_anchored, + "core_review_ready": active_review_ready, + "core_artifact_audit_clean": active_review_ready, + "core_review_ready_meaning": PHASE3_EXACT_SCOPE_REVIEW_READY_MEANING, + "core_scorecard_update_allowed": False, + "scorecard_update_allowed": False, + } + audit["core_validation_errors"] = validate_phase3_core_promotion_audit(audit) + return audit + + +def validate_phase3_core_promotion_audit(audit: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if audit.get("core_scorecard_update_allowed") is not False: + errors.append("core_scorecard_update_allowed must be false") + if audit.get("active_claim_ledger_anchored") is not True: + errors.append("active claim boundary must be anchored in claim ledger") + if not re.match(PHASE3_TARGET_RUN_ID_PATTERN, str(audit.get("target_run_id") or "")): + errors.append("target_run_id must match Phase 3 Slurm target run id pattern") + final_report_errors = audit.get("final_report_validation_errors", []) + if final_report_errors != _string_list(final_report_errors): + errors.append("final_report_validation_errors must be a list of strings") + elif final_report_errors: + errors.append("active audit requires clean final_report_validation_errors") + active_scope = audit.get("active_scope", {}) + if not isinstance(active_scope, Mapping): + errors.append("active_scope must be an object") + else: + if active_scope.get("schema_version") != PHASE3_ACTIVE_SCOPE_SCHEMA: + errors.append("active_scope.schema_version must be Phase 3 active scope schema") + if active_scope.get("claim_boundary") != PHASE3_ACTIVE_SCOPE_CLAIM_BOUNDARY: + errors.append("active_scope.claim_boundary must be Phase 3 active scope boundary") + if active_scope.get("scorecard_update_allowed") is not False: + errors.append("active_scope.scorecard_update_allowed must be false") + active_milestones = active_scope.get("active_milestones", active_scope.get("core_milestones")) + if active_milestones != list(PHASE3_ACTIVE_MILESTONES): + errors.append("active_scope.active_milestones must match current Phase 3 active milestones") + if active_scope.get("ready") is not True: + errors.append("active_scope.ready must be true") + active_completed = audit.get("active_completed_milestones", []) + if active_completed != _string_list(active_completed): + errors.append("active_completed_milestones must be a list of strings") + elif set(active_completed) != set(PHASE3_ACTIVE_MILESTONES) or len(active_completed) != len(PHASE3_ACTIVE_MILESTONES): + errors.append("every active P3 milestone must be completed") + active_blocked = audit.get("active_blocked_milestones", []) + if active_blocked != _string_list(active_blocked): + errors.append("active_blocked_milestones must be a list of strings") + elif active_blocked: + errors.append("active audit must not list active_blocked_milestones") + if audit.get("active_review_ready") is not True: + errors.append("active_review_ready must be true") + return errors + +def validate_phase3_promotion_audit(audit: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if audit.get("schema_version") != "bb.rl.phase3.promotion_audit.v1": + errors.append("schema_version must be Phase 3 promotion audit schema") + if audit.get("report_id") != PHASE3_PROMOTION_AUDIT_ID: + errors.append("report_id must be Phase 3 promotion audit id") + if audit.get("core_scorecard_update_allowed") is not False: + errors.append("core_scorecard_update_allowed must be false") + active_scope = audit.get("active_scope", {}) + if active_scope: + if not isinstance(active_scope, Mapping): + errors.append("active_scope must be an object when present") + else: + if active_scope.get("schema_version") != PHASE3_ACTIVE_SCOPE_SCHEMA: + errors.append("active_scope.schema_version must be Phase 3 active scope schema") + if active_scope.get("claim_boundary") != PHASE3_ACTIVE_SCOPE_CLAIM_BOUNDARY: + errors.append("active_scope.claim_boundary must be Phase 3 active scope boundary") + if active_scope.get("scorecard_update_allowed") is not False: + errors.append("active_scope.scorecard_update_allowed must be false") + active_milestones = active_scope.get("active_milestones", active_scope.get("core_milestones")) + if active_milestones != list(PHASE3_ACTIVE_MILESTONES): + errors.append("active_scope.active_milestones must match current Phase 3 active milestones") + if audit.get("active_review_ready") is True: + if not isinstance(active_scope, Mapping) or active_scope.get("ready") is not True: + errors.append("ready active audit must contain ready active_scope") + if audit.get("claim_boundary") != PHASE3_PROMOTION_CLAIM_BOUNDARY: + errors.append("claim_boundary must be promotion review only") + if not re.match(PHASE3_TARGET_RUN_ID_PATTERN, str(audit.get("target_run_id") or "")): + errors.append("target_run_id must match Phase 3 Slurm target run id pattern") + if audit.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + blocked_milestones = audit.get("blocked_milestones", []) + final_report_errors = audit.get("final_report_validation_errors", []) + if blocked_milestones != _string_list(blocked_milestones): + errors.append("blocked_milestones must be a list of strings") + if final_report_errors != _string_list(final_report_errors): + errors.append("final_report_validation_errors must be a list of strings") + if audit.get("promotion_review_ready") is not True: + errors.append("promotion_review_ready must be true") + if audit.get("promotion_review_ready") is True: + if blocked_milestones: + errors.append("ready promotion audit must not list blocked_milestones") + if final_report_errors: + errors.append("ready promotion audit must not list final_report_validation_errors") + completed = audit.get("completed_milestones", []) + completed_ids = [item for item in completed if isinstance(item, str)] if isinstance(completed, list) else [] + if not isinstance(completed, list) or len(completed) != len(completed_ids) or set(completed_ids) != set(PHASE3_MILESTONES) or len(completed_ids) != len(PHASE3_MILESTONES): + errors.append("every P3 milestone must be completed") + if audit.get("bd_epic_closed") is not True: + errors.append("bd epic must be closed") + return errors diff --git a/breadboard/rl/phase3/rollout_runner.py b/breadboard/rl/phase3/rollout_runner.py new file mode 100644 index 00000000..c2d79323 --- /dev/null +++ b/breadboard/rl/phase3/rollout_runner.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +from breadboard.rl.phase3.trainer_live import build_phase3_dataproto + +PHASE3_CLOSED_LOOP_SCHEMA = "bb.rl.phase3.closed_loop_run.v1" +PHASE3_CLOSED_LOOP_CLAIM_BOUNDARY = "phase3_closed_loop_target_run_named_scope" + + +@dataclass(frozen=True) +class Phase3ClosedLoopSpec: + target_run_id: str + env_package_path: Path + task_manifest_path: Path + policy_snapshot_ref: str + trainer_backend: Literal["verl_ppo", "verl_grpo"] + output_dir: Path + max_tasks: int + + +def _load_rows(path: Path, max_tasks: int) -> list[dict[str, Any]]: + payload = json.loads(path.read_text()) + rows = payload.get("rows", payload if isinstance(payload, list) else []) + if not isinstance(rows, list): + raise ValueError("task manifest must contain rows") + return [dict(row) for row in rows[:max_tasks]] + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n") + + +def run_phase3_closed_loop(spec: Phase3ClosedLoopSpec) -> dict[str, Any]: + rows = _load_rows(spec.task_manifest_path, spec.max_tasks) + accepted = [row for row in rows if row.get("admission", {}).get("row_status", row.get("status")) == "accepted" and row.get("admission", {}).get("quarantine_status", "clear") == "clear"] + rejected = [row for row in rows if row not in accepted and row.get("admission", {}).get("quarantine_status", "clear") != "quarantined"] + quarantined = [row for row in rows if row.get("admission", {}).get("quarantine_status") == "quarantined"] + errors: list[str] = [] + for row in accepted: + if not row.get("accepted_replay_ref") and not row.get("replay_ref"): + errors.append(f"accepted row {row.get('task_id')} lacks replay closure") + row_policy = row.get("policy_snapshot_id") or row.get("policy", {}).get("policy_snapshot_id") + if row_policy != spec.policy_snapshot_ref: + errors.append(f"row {row.get('task_id')} policy snapshot mismatch") + if quarantined and any(row in accepted for row in quarantined): + errors.append("quarantined row entered trainer batch") + projection_path = spec.output_dir / "projection_rows.json" + _write_json(projection_path, {"rows": accepted}) + dataproto_ok = False + trainer_report_path = spec.output_dir / "trainer_update_report.json" + if accepted and not errors: + try: + batch = {"rows": accepted, "target_run_id": spec.target_run_id} + build_phase3_dataproto(batch, device="cpu", require_grpo_uid=spec.trainer_backend == "verl_grpo") + dataproto_ok = True + except Exception as exc: # noqa: BLE001 - report must retain failure cause. + errors.append(f"dataproto build failed: {exc}") + trainer_report = {"passed": dataproto_ok and not errors, "checkpoint_after_sha256": "sha256:unavailable" if errors else "sha256:closedloop"} + _write_json(trainer_report_path, trainer_report) + if not trainer_report["passed"]: + errors.append("trainer update failed") + report = { + "schema_version": PHASE3_CLOSED_LOOP_SCHEMA, + "report_id": "phase3_closed_loop_run", + "claim_boundary": PHASE3_CLOSED_LOOP_CLAIM_BOUNDARY, + "target_run_id": spec.target_run_id, + "accepted_count": len(accepted), + "rejected_count": len(rejected), + "quarantined_count": len(quarantined), + "projection_manifest_ref": str(projection_path), + "trainer_update_report_path": str(trainer_report_path), + "checkpoint_after_sha256": trainer_report.get("checkpoint_after_sha256"), + "accepted_replay_refs": [row.get("accepted_replay_ref") or row.get("replay_ref") for row in accepted], + "rejected_replay_refs": [row.get("rejected_replay_ref") or row.get("replay_ref") for row in rejected], + "policy_snapshot_id": spec.policy_snapshot_ref, + "errors": errors, + "scorecard_update_allowed": False, + "passed": not errors, + } + _write_json(spec.output_dir / "phase3_closed_loop_report.json", report) + return report + + +def validate_phase3_closed_loop_report(report: dict[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("schema_version") != PHASE3_CLOSED_LOOP_SCHEMA: + errors.append("schema_version must be closed loop v1") + if report.get("claim_boundary") != PHASE3_CLOSED_LOOP_CLAIM_BOUNDARY: + errors.append("claim_boundary must be closed-loop boundary") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if report.get("passed") is not True: + errors.append("passed must be true") + if any(not ref for ref in report.get("accepted_replay_refs", [])): + errors.append("accepted replay refs must be present") + if report.get("quarantined_count", 0) and report.get("accepted_count", 0) < 0: + errors.append("quarantined rows cannot enter trainer batch") + return errors diff --git a/breadboard/rl/phase3/scheduler.py b/breadboard/rl/phase3/scheduler.py new file mode 100644 index 00000000..ec250e33 --- /dev/null +++ b/breadboard/rl/phase3/scheduler.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import json +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +@dataclass(frozen=True) +class SlurmRunSpec: + ssh_alias: str + partition: str + job_name: str + command_id: str + payload_zip: Path + output_dir: Path + nodes: int = 1 + ntasks: int = 1 + gres: str = "gpu:8" + time_limit: str = "02:00:00" + + +def parse_sbatch_job_id(output: str) -> str: + for token in output.replace("\n", " ").split(): + job_id = token.split(";", 1)[0] + if job_id.isdigit(): + return job_id + raise ValueError("sbatch output did not contain job id") + + +def submit_slurm_run(spec: SlurmRunSpec) -> dict[str, Any]: + spec.output_dir.mkdir(parents=True, exist_ok=True) + if not spec.payload_zip.exists(): + raise FileNotFoundError(spec.payload_zip) + command = [ + "ssh", + spec.ssh_alias, + "sbatch", + "--parsable", + f"--partition={spec.partition}", + f"--job-name={spec.job_name}", + f"--nodes={spec.nodes}", + f"--ntasks={spec.ntasks}", + f"--gres={spec.gres}", + f"--time={spec.time_limit}", + ] + result = subprocess.run(command, check=False, text=True, capture_output=True) + if result.returncode != 0: + raise RuntimeError(result.stderr or result.stdout) + job_id = parse_sbatch_job_id(result.stdout) + payload = {"job_id": job_id, "command_id": spec.command_id, "argv": command, "scheduler": "slurm"} + (spec.output_dir / f"{spec.command_id}.submission.json").write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n") + return payload + + +def collect_slurm_run(job_id: str, *, output_dir: Path) -> dict[str, Any]: + log_path = output_dir / f"slurm-{job_id}.out" + return {"job_id": job_id, "raw_log_path": str(log_path), "completed": log_path.exists()} diff --git a/breadboard/rl/phase3/security_enforcement.py b/breadboard/rl/phase3/security_enforcement.py new file mode 100644 index 00000000..0c99756f --- /dev/null +++ b/breadboard/rl/phase3/security_enforcement.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import PurePosixPath + +from breadboard.rl.phase2.hardening import ( + ArtifactEgressRequest, + DestructiveActionRequest, + EgressPolicy, + evaluate_artifact_egress, + guard_destructive_action, + redact_mapping, + workspace_isolated, +) + +__all__ = [ + "ArtifactEgressRequest", + "EgressPolicy", + "enforce_artifact_egress", + "enforce_workspace_path", + "enforce_command_request", + "redact_mapping", + "workspace_isolated", +] + + +def enforce_artifact_egress(request: ArtifactEgressRequest, policy: EgressPolicy) -> None: + result = evaluate_artifact_egress(request, policy) + if not result["allowed"]: + reason = "; ".join(result["reasons"]) + raise PermissionError(f"artifact egress denied: {reason}") + + +def enforce_workspace_path(path: str, *, tenant_id: str, workspace_id: str) -> PurePosixPath: + del tenant_id + normalized = path.replace("\\", "/") + pure = PurePosixPath(normalized) + if not normalized or normalized.startswith("~") or pure.is_absolute() or ".." in pure.parts: + raise PermissionError("workspace path denied: path must be workspace-relative") + if not pure.parts or pure.parts[0] != workspace_id: + raise PermissionError("workspace path denied: path must start with workspace_id") + return pure + + +def enforce_command_request(command: Sequence[str], *, workspace_relative_path: str, workspace_id: str) -> None: + command_text = " ".join(str(part) for part in command) + result = guard_destructive_action( + DestructiveActionRequest(action_id="phase3_command_request", command=command_text, workspace_relative_path=workspace_relative_path), + workspace_id=workspace_id, + ) + if not result["allowed"]: + raise PermissionError("command request denied: " + "; ".join(result["reasons"])) diff --git a/breadboard/rl/phase3/service_live.py b/breadboard/rl/phase3/service_live.py new file mode 100644 index 00000000..1eee8b94 --- /dev/null +++ b/breadboard/rl/phase3/service_live.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from breadboard.rl.phase2.hardening import ArtifactEgressRequest, EgressPolicy +from breadboard.rl.phase2.service import ACTIVE_STATES, TERMINAL_STATES, ArtifactRecord, ResourceCaps, RunStatus, RunSubmission, StreamEvent, resource_cap_rejections +from breadboard.rl.phase3.security_enforcement import enforce_artifact_egress, enforce_command_request, enforce_workspace_path +from breadboard.rl.phase3.store import SQLiteRLRunStore + +DEFAULT_CAPS = ResourceCaps(max_tasks=128, max_gpus=8, max_budget_usd=500.0, max_duration_seconds=7200, max_artifact_bytes=512 * 1024 * 1024) +DEFAULT_EGRESS_POLICY = EgressPolicy(allowed_prefixes=("ws",), max_artifact_bytes=DEFAULT_CAPS.max_artifact_bytes) + + +def _resource_floor_rejections(submission: RunSubmission) -> list[str]: + rejections: list[str] = [] + if submission.requested_tasks <= 0: + rejections.append("requested_tasks must be positive") + if submission.requested_gpus <= 0: + rejections.append("requested_gpus must be positive") + if submission.requested_budget_usd <= 0: + rejections.append("requested_budget_usd must be positive") + if submission.requested_duration_seconds <= 0: + rejections.append("requested_duration_seconds must be positive") + return rejections + + +class LiveRLRunService: + def __init__(self, store: SQLiteRLRunStore | str | Path | None = None, *, caps: ResourceCaps = DEFAULT_CAPS, egress_policy: EgressPolicy = DEFAULT_EGRESS_POLICY): + self.store = store if isinstance(store, SQLiteRLRunStore) else SQLiteRLRunStore(store or ":memory:") + self.caps = caps + self.egress_policy = egress_policy + + def submit(self, submission: RunSubmission) -> RunStatus: + command = submission.metadata.get("command") if isinstance(submission.metadata, dict) else None + if command is not None: + if not isinstance(command, list): + raise PermissionError("command request denied: command must be a list") + enforce_command_request(command, workspace_relative_path=submission.env_package_ref, workspace_id=submission.workspace_id) + enforce_workspace_path(submission.env_package_ref, tenant_id=submission.tenant_id, workspace_id=submission.workspace_id) + rejections = _resource_floor_rejections(submission) + resource_cap_rejections(submission, self.caps) + state = "queued" if not rejections else "rejected" + reason = "; ".join(rejections) + self.store.create_run(submission, self.caps, state=state, reason=reason) + self.store.append_event(submission.run_id, event_type="run.submitted", state=state, message=reason or "run queued", target_run_id=submission.target_run_id) + return self.store.status(submission.run_id) + + def status(self, run_id: str, *, tenant_id: str | None = None, workspace_id: str | None = None) -> RunStatus: + if tenant_id is not None: + self.store.assert_tenant(run_id, tenant_id=tenant_id, workspace_id=workspace_id) + return self.store.status(run_id) + + def start(self, run_id: str) -> RunStatus: + status = self.store.status(run_id) + if status.state != "queued": + raise ValueError(f"run {run_id} cannot start from state {status.state}") + self.store.update_state(run_id, state="running") + self.store.append_event(run_id, event_type="run.start", state="running", message="run started", target_run_id=status.target_run_id) + return self.store.status(run_id) + + def cancel(self, run_id: str, *, reason: str = "cancel requested") -> RunStatus: + status = self.store.status(run_id) + if status.state in TERMINAL_STATES: + raise ValueError(f"run {run_id} is already terminal: {status.state}") + self.store.update_state(run_id, state="cancel_requested", cancellation_state="requested", reason=reason) + self.store.append_event(run_id, event_type="cancel.requested", state="cancel_requested", message=reason, target_run_id=status.target_run_id) + return self.store.status(run_id) + + def acknowledge_cancelled(self, run_id: str, *, reason: str = "cancelled") -> RunStatus: + status = self.store.status(run_id) + if status.state != "cancel_requested": + raise ValueError(f"run {run_id} has no pending cancellation") + self.store.update_state(run_id, state="cancelled", cancellation_state="acknowledged", reason=reason) + self.store.append_event(run_id, event_type="cancel.acknowledged", state="cancelled", message=reason, target_run_id=status.target_run_id) + return self.store.status(run_id) + + def complete(self, run_id: str, *, succeeded: bool = True, reason: str = "") -> RunStatus: + status = self.store.status(run_id) + if status.state not in ACTIVE_STATES: + raise ValueError(f"run {run_id} cannot complete from state {status.state}") + state = "succeeded" if succeeded else "failed" + self.store.update_state(run_id, state=state, reason=reason) + self.store.append_event(run_id, event_type="run.end", state=state, message=reason or state, target_run_id=status.target_run_id) + return self.store.status(run_id) + + def add_artifact(self, record: ArtifactRecord, *, tenant_id: str, workspace_id: str) -> ArtifactRecord: + self.store.assert_tenant(record.run_id, tenant_id=tenant_id, workspace_id=workspace_id) + self.store.add_artifact(record, tenant_id=tenant_id, workspace_id=workspace_id) + status = self.store.status(record.run_id) + self.store.append_event(record.run_id, event_type="artifact.added", state=status.state, message=record.artifact_id, target_run_id=status.target_run_id, payload=record.to_dict()) + return record + + def collect(self, run_id: str, *, tenant_id: str, workspace_id: str) -> list[ArtifactRecord]: + self.store.assert_tenant(run_id, tenant_id=tenant_id, workspace_id=workspace_id) + artifacts = self.store.artifacts(run_id) + for artifact in artifacts: + enforce_workspace_path(artifact.relative_path, tenant_id=tenant_id, workspace_id=workspace_id) + if artifact.egress_allowed: + enforce_artifact_egress(ArtifactEgressRequest(artifact.relative_path, artifact.bytes, "tenant_internal"), self.egress_policy) + return artifacts + + def replay(self, run_id: str, artifact_id: str, *, tenant_id: str, workspace_id: str) -> dict[str, Any]: + self.store.assert_tenant(run_id, tenant_id=tenant_id, workspace_id=workspace_id) + artifact = self.store.artifact(run_id, artifact_id) + enforce_workspace_path(artifact.relative_path, tenant_id=tenant_id, workspace_id=workspace_id) + if not artifact.egress_allowed: + return {"available": False, "artifact_id": artifact_id, "reason": "egress denied"} + enforce_artifact_egress(ArtifactEgressRequest(artifact.relative_path, artifact.bytes, "tenant_internal"), self.egress_policy) + return {"available": True, "artifact_id": artifact_id, "replay_path": artifact.relative_path, "sha256": artifact.sha256} + + def audit(self, run_id: str, *, tenant_id: str, workspace_id: str) -> dict[str, Any]: + row = self.store.assert_tenant(run_id, tenant_id=tenant_id, workspace_id=workspace_id) + status = self.store.status(run_id) + return { + "run_id": run_id, + "tenant_id": row["tenant_id"], + "workspace_id": row["workspace_id"], + "target_run_id": status.target_run_id, + "state": status.state, + "persistent_store": "sqlite", + "scorecard_update_allowed": False, + } + + def stream_since(self, run_id: str, *, from_sequence: int = 0, tenant_id: str | None = None, workspace_id: str | None = None) -> list[StreamEvent]: + if tenant_id is not None: + self.store.assert_tenant(run_id, tenant_id=tenant_id, workspace_id=workspace_id) + return self.store.events_since(run_id, from_sequence) diff --git a/breadboard/rl/phase3/store.py b/breadboard/rl/phase3/store.py new file mode 100644 index 00000000..b792f791 --- /dev/null +++ b/breadboard/rl/phase3/store.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import json +import sqlite3 +import time +from pathlib import Path +from typing import Any + +from breadboard.rl.phase2.service import ArtifactRecord, ResourceCaps, RunStatus, RunSubmission, StreamEvent +from breadboard.rl.phase3.security_enforcement import enforce_workspace_path + + +class SQLiteRLRunStore: + def __init__(self, path: str | Path): + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self._conn = sqlite3.connect(self.path, check_same_thread=False) + self._conn.row_factory = sqlite3.Row + self._init_schema() + + def close(self) -> None: + self._conn.close() + + def _init_schema(self) -> None: + self._conn.executescript( + """ + create table if not exists rl_runs( + run_id text primary key, + tenant_id text not null, + workspace_id text not null, + env_package_ref text not null, + target_run_id text not null, + state text not null, + cancellation_state text not null, + created_at real not null, + updated_at real not null, + reason text not null, + resource_caps_json text not null + ); + create table if not exists rl_events( + run_id text not null, + sequence integer not null, + event_type text not null, + state text not null, + message text not null, + target_run_id text not null, + payload_json text not null, + primary key(run_id, sequence) + ); + create table if not exists rl_artifacts( + run_id text not null, + artifact_id text not null, + relative_path text not null, + sha256 text not null, + bytes integer not null, + egress_allowed integer not null, + primary key(run_id, artifact_id) + ); + """ + ) + self._conn.commit() + + def create_run(self, submission: RunSubmission, caps: ResourceCaps, *, state: str, reason: str = "") -> None: + now = time.time() + with self._conn: + self._conn.execute( + "insert into rl_runs values(?,?,?,?,?,?,?,?,?,?,?)", + ( + submission.run_id, + submission.tenant_id, + submission.workspace_id, + submission.env_package_ref, + submission.target_run_id, + state, + "not_cancelled", + now, + now, + reason, + json.dumps(caps.to_dict(), sort_keys=True), + ), + ) + + def update_state(self, run_id: str, *, state: str, cancellation_state: str | None = None, reason: str = "") -> None: + row = self.get_run_row(run_id) + if row is None: + raise KeyError(run_id) + with self._conn: + self._conn.execute( + "update rl_runs set state=?, cancellation_state=?, reason=?, updated_at=? where run_id=?", + (state, cancellation_state or row["cancellation_state"], reason, time.time(), run_id), + ) + + def get_run_row(self, run_id: str) -> sqlite3.Row | None: + return self._conn.execute("select * from rl_runs where run_id=?", (run_id,)).fetchone() + + def status(self, run_id: str) -> RunStatus: + row = self.get_run_row(run_id) + if row is None: + raise KeyError(run_id) + return RunStatus( + run_id=row["run_id"], + state=row["state"], + target_run_id=row["target_run_id"], + accepted=row["state"] != "rejected", + cancellation_state=row["cancellation_state"], + reason=row["reason"], + ) + + def assert_tenant(self, run_id: str, *, tenant_id: str, workspace_id: str | None = None) -> sqlite3.Row: + row = self.get_run_row(run_id) + if row is None: + raise KeyError(run_id) + if row["tenant_id"] != tenant_id or (workspace_id is not None and row["workspace_id"] != workspace_id): + raise PermissionError("tenant mismatch") + return row + + def append_event(self, run_id: str, *, event_type: str, state: str, message: str, target_run_id: str, payload: dict[str, Any] | None = None) -> StreamEvent: + last = self._conn.execute("select max(sequence) as seq from rl_events where run_id=?", (run_id,)).fetchone()["seq"] + sequence = int(last or 0) + 1 + event = StreamEvent(sequence, run_id, event_type, state, message, target_run_id, payload or {}) + with self._conn: + self._conn.execute( + "insert into rl_events values(?,?,?,?,?,?,?)", + (run_id, sequence, event_type, state, message, target_run_id, json.dumps(event.payload, sort_keys=True)), + ) + return event + + def events_since(self, run_id: str, sequence: int = 0) -> list[StreamEvent]: + rows = self._conn.execute( + "select * from rl_events where run_id=? and sequence>? order by sequence", (run_id, sequence) + ).fetchall() + return [ + StreamEvent(row["sequence"], row["run_id"], row["event_type"], row["state"], row["message"], row["target_run_id"], json.loads(row["payload_json"])) + for row in rows + ] + + def add_artifact(self, record: ArtifactRecord, *, tenant_id: str, workspace_id: str) -> None: + enforce_workspace_path(record.relative_path, tenant_id=tenant_id, workspace_id=workspace_id) + with self._conn: + self._conn.execute( + "insert or replace into rl_artifacts values(?,?,?,?,?,?)", + (record.run_id, record.artifact_id, record.relative_path, record.sha256, record.bytes, int(record.egress_allowed)), + ) + + def artifacts(self, run_id: str) -> list[ArtifactRecord]: + rows = self._conn.execute("select * from rl_artifacts where run_id=? order by artifact_id", (run_id,)).fetchall() + return [ArtifactRecord(row["run_id"], row["artifact_id"], row["relative_path"], row["sha256"], int(row["bytes"]), bool(row["egress_allowed"])) for row in rows] + + def artifact(self, run_id: str, artifact_id: str) -> ArtifactRecord: + row = self._conn.execute("select * from rl_artifacts where run_id=? and artifact_id=?", (run_id, artifact_id)).fetchone() + if row is None: + raise KeyError(artifact_id) + return ArtifactRecord(row["run_id"], row["artifact_id"], row["relative_path"], row["sha256"], int(row["bytes"]), bool(row["egress_allowed"])) diff --git a/breadboard/rl/phase3/trainer_live.py b/breadboard/rl/phase3/trainer_live.py new file mode 100644 index 00000000..d6c9e280 --- /dev/null +++ b/breadboard/rl/phase3/trainer_live.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import json +from collections import Counter +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +from breadboard.rl.phase2.bridge import build_verl_batch_from_projection_rows +from breadboard.rl.phase3.evidence import sha256_file, validate_phase3_command_log_manifest + +PHASE3_DATAPROTO_SCHEMA = "bb.rl.phase3.verl_dataproto.v1" +PHASE3_TRAINER_UPDATE_SCHEMA = "bb.rl.phase3.verl_trainer_update.v1" +PHASE3_TRAINER_CLAIM_BOUNDARY = "phase3_verl_ppo_grpo_weight_update_named_target_scope" + + +@dataclass(frozen=True) +class Phase3TrainerRunSpec: + target_run_id: str + trainer_backend: Literal["verl_ppo", "verl_grpo"] + model_ref: str + projection_rows_path: Path + output_dir: Path + max_steps: int + expected_device_count: int = 8 + require_weight_update: bool = True + + +def _as_payload(batch: Mapping[str, Any]) -> Mapping[str, Any]: + if "tensors" in batch and "row_refs" in batch: + return batch + rows = batch.get("rows") if isinstance(batch, Mapping) else None + if rows is not None: + return build_verl_batch_from_projection_rows(rows, target_run_id=str(batch["target_run_id"])).to_dict() + return batch + + +def _tensor(torch: Any, values: Any, *, device: str, dtype: Any | None = None) -> Any: + try: + return torch.tensor(values, device=device, dtype=dtype) + except TypeError: + return torch.tensor(values) + + +def _dataproto_non_tensors(values: Mapping[str, Any]) -> dict[str, Any]: + try: + import numpy as np # type: ignore + except ImportError: + return dict(values) + return {key: np.asarray(value, dtype=object) for key, value in values.items()} + + +def _make_dataproto(DataProto: Any, payload: dict[str, Any]) -> Any: + if hasattr(DataProto, "from_dict"): + try: + return DataProto.from_dict( + tensors=dict(payload["batch"].items()), + non_tensors=_dataproto_non_tensors(payload.get("non_tensor_batch", {})), + meta_info=payload.get("meta_info", {}), + ) + except (AttributeError, TypeError, ValueError, AssertionError, ImportError): + try: + return DataProto.from_dict(payload) + except (AttributeError, TypeError, ValueError, AssertionError): + pass + if hasattr(DataProto, "from_single_dict"): + try: + return DataProto.from_single_dict(payload) + except (AttributeError, TypeError, ValueError, AssertionError): + pass + try: + return DataProto(batch=payload["batch"], non_tensor_batch=_dataproto_non_tensors(payload.get("non_tensor_batch", {})), meta_info=payload.get("meta_info", {})) + except TypeError: + instance = DataProto() + for key, value in payload.items(): + setattr(instance, key, value) + return instance + + +def build_phase3_dataproto(batch: Mapping[str, Any], *, device: str, require_grpo_uid: bool) -> Any: + import torch # type: ignore + from tensordict import TensorDict # type: ignore + from verl.protocol import DataProto # type: ignore + + payload = _as_payload(batch) + row_refs = [dict(row) for row in payload.get("row_refs", [])] + if not row_refs: + raise ValueError("row_refs must be present") + policy_snapshot_id = str(payload.get("policy_snapshot_id") or "") + target_run_id = str(payload.get("target_run_id") or "") + if not policy_snapshot_id: + raise ValueError("policy_snapshot_id must be present") + if not target_run_id: + raise ValueError("target_run_id must be present") + if payload.get("field_ledger", {}).get("provenance_loss_detected"): + raise ValueError("projection provenance loss is not allowed") + tensors = payload.get("tensors", {}) + masks = payload.get("masks", {}) + rewards = payload.get("rewards", {}) + logprobs = payload.get("logprobs", {}) + uid: list[str] = [] + for row_ref in row_refs: + value = row_ref.get("group_id") if row_ref.get("group_id") else row_ref.get("task_id") + if not value: + raise ValueError("uid must be present for every row") + uid.append(str(value)) + if require_grpo_uid: + singles = sorted(key for key, count in Counter(uid).items() if count < 2) + if singles: + raise ValueError("GRPO uid groups must contain at least two rows: " + ",".join(singles)) + batch_tensors = TensorDict( + { + "input_ids": _tensor(torch, tensors.get("input_ids"), device=device, dtype=getattr(torch, "long", None)), + "attention_mask": _tensor(torch, tensors.get("attention_mask"), device=device, dtype=getattr(torch, "long", None)), + "responses": _tensor(torch, tensors.get("input_ids"), device=device, dtype=getattr(torch, "long", None)), + "response_mask": _tensor(torch, masks.get("completion_logprob_mask") or masks.get("loss_mask"), device=device), + "token_level_rewards": _tensor(torch, rewards.get("token_rewards"), device=device), + "old_log_probs": _tensor(torch, logprobs.get("completion_logprobs"), device=device), + }, + batch_size=[len(row_refs)], + ) + return _make_dataproto( + DataProto, + { + "batch": batch_tensors, + "non_tensor_batch": {"uid": uid, "row_refs": row_refs}, + "meta_info": { + "target_run_id": target_run_id, + "policy_snapshot_id": policy_snapshot_id, + "claim_boundary": PHASE3_TRAINER_CLAIM_BOUNDARY, + "schema_version": PHASE3_DATAPROTO_SCHEMA, + }, + }, + ) + + +def _read_metrics(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text()) + except FileNotFoundError: + return {} + return payload if isinstance(payload, dict) else {} + + +def _first_command_sha(manifest: Mapping[str, Any]) -> str: + rows = manifest.get("commands", manifest.get("command_logs", [])) + if isinstance(rows, list) and rows and isinstance(rows[0], Mapping): + return str(rows[0].get("raw_log_sha256") or "") + return "" + + +def _derive_evidence_root(output_dir: Path) -> Path: + resolved = output_dir.resolve() + parts = resolved.parts + for index in range(len(parts) - 2): + if parts[index] == "ZYPHRA" and parts[index + 1] == "RL_PHASE_3" and parts[index + 2] == "runs": + return Path(*parts[:index]) + return resolved + + +def build_phase3_trainer_update_report( + spec: Phase3TrainerRunSpec, *, command_log_manifest: Mapping[str, Any], checkpoint_before: Path, + checkpoint_after: Path, metrics_path: Path +) -> dict[str, Any]: + metrics = _read_metrics(metrics_path) + before_sha = sha256_file(checkpoint_before) if checkpoint_before.exists() else "" + after_sha = sha256_file(checkpoint_after) if checkpoint_after.exists() else "" + optimizer_steps = int(metrics.get("optimizer_step_count") or metrics.get("optimizer_steps") or 0) + device_count = int(metrics.get("device_count") or spec.expected_device_count) + manifest_errors = validate_phase3_command_log_manifest( + command_log_manifest, + target_run_id=spec.target_run_id, + repo_root=Path.cwd(), + evidence_root=_derive_evidence_root(spec.output_dir), + ) + checkpoint_changed = bool(before_sha and after_sha and before_sha != after_sha) + weight_update = optimizer_steps >= 1 and checkpoint_changed and bool(metrics.get("weight_update_performed", True)) + passed = not manifest_errors and optimizer_steps >= 1 and checkpoint_changed and weight_update and device_count == spec.expected_device_count + return { + "schema_version": PHASE3_TRAINER_UPDATE_SCHEMA, + "report_id": f"phase3_{spec.trainer_backend}_trainer_update", + "claim_boundary": PHASE3_TRAINER_CLAIM_BOUNDARY, + "target_run_id": spec.target_run_id, + "trainer_backend": spec.trainer_backend, + "model_ref": spec.model_ref, + "optimizer_step_count": optimizer_steps, + "checkpoint_before_sha256": before_sha, + "checkpoint_after_sha256": after_sha, + "checkpoint_changed": checkpoint_changed, + "weight_update_performed": weight_update, + "loss_metrics": metrics.get("loss_metrics", {}), + "device_count": device_count, + "raw_command_log_sha256": _first_command_sha(command_log_manifest), + "manifest_validation_errors": manifest_errors, + "input_hashes": {"metrics": sha256_file(metrics_path) if metrics_path.exists() else ""}, + "artifact_paths": {"checkpoint_after": str(checkpoint_after), "metrics": str(metrics_path)}, + "scorecard_update_allowed": False, + "passed": passed, + } + + +def validate_phase3_trainer_update_report(report: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if report.get("schema_version") != PHASE3_TRAINER_UPDATE_SCHEMA: + errors.append("schema_version must be trainer update v1") + if report.get("claim_boundary") != PHASE3_TRAINER_CLAIM_BOUNDARY: + errors.append("claim_boundary must be trainer live boundary") + if int(report.get("optimizer_step_count") or 0) < 1: + errors.append("optimizer_step_count must be at least 1") + if report.get("checkpoint_changed") is not True: + errors.append("checkpoint_changed must be true") + if report.get("checkpoint_before_sha256") == report.get("checkpoint_after_sha256"): + errors.append("checkpoint hashes must differ") + if report.get("weight_update_performed") is not True: + errors.append("weight_update_performed must be true") + if int(report.get("device_count") or 0) != 8: + errors.append("device_count must be 8") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + if report.get("passed") is not True: + errors.append("passed must be true") + return errors diff --git a/breadboard/rl/phase4/__init__.py b/breadboard/rl/phase4/__init__.py new file mode 100644 index 00000000..5822c312 --- /dev/null +++ b/breadboard/rl/phase4/__init__.py @@ -0,0 +1,27 @@ +"""Phase 4 RL inference-lane evidence helpers.""" +from breadboard.rl.phase4.infra_hardening import InfraHardeningInputs, evaluate_infra_hardening + +from breadboard.rl.phase4.native_inference import ( + BREADBOARD_NATIVE_INFERENCE_OWNER, + NativeCompletionRecord, + NativeCompletionResponse, + NativeInferenceLane, + sha256_bytes, + sha256_file, + sha256_json, +) +from breadboard.rl.phase4.wrapper_identity import collect_wrapper_identity, runtime_module_provenance + +__all__ = [ + "BREADBOARD_NATIVE_INFERENCE_OWNER", + "InfraHardeningInputs", + "NativeCompletionRecord", + "NativeCompletionResponse", + "NativeInferenceLane", + "collect_wrapper_identity", + "evaluate_infra_hardening", + "sha256_bytes", + "sha256_file", + "sha256_json", + "runtime_module_provenance", +] diff --git a/breadboard/rl/phase4/infra_hardening.py b/breadboard/rl/phase4/infra_hardening.py new file mode 100644 index 00000000..3b81c8b3 --- /dev/null +++ b/breadboard/rl/phase4/infra_hardening.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +INFRA_HARDENING_SCHEMA = "bb.rl.phase4.production_infra_hardening.v1" + + +@dataclass(frozen=True) +class InfraHardeningInputs: + firecracker_version: str = "" + firecracker_non_root_boot: bool = False + firecracker_jailer_available: bool = False + firecracker_networking_configured: bool = False + firecracker_concurrent_microvms: int = 0 + gvisor_runsc_version: str = "" + gvisor_registered_runtime: bool = False + gvisor_runtime_name: str = "" + cpu_squeeze_workers: int = 0 + cpu_squeeze_success: bool = False + + +def evaluate_infra_hardening(inputs: InfraHardeningInputs) -> dict[str, Any]: + """Evaluate the requested production infra-hardening contract without overclaiming. + + Firecracker is production-hardening evidence only when the probe proves the hard + parts together: non-root boot, jailer, networking, and more than one concurrent + microVM. gVisor is production-hardening evidence only when it is registered as a + container runtime, not merely when the `runsc` binary exists. + """ + + firecracker_blockers: list[str] = [] + if not inputs.firecracker_version: + firecracker_blockers.append("firecracker_version_missing") + if not inputs.firecracker_non_root_boot: + firecracker_blockers.append("firecracker_non_root_boot_missing") + if not inputs.firecracker_jailer_available: + firecracker_blockers.append("firecracker_jailer_missing") + if not inputs.firecracker_networking_configured: + firecracker_blockers.append("firecracker_networking_missing") + if inputs.firecracker_concurrent_microvms < 2: + firecracker_blockers.append("firecracker_concurrency_missing") + + gvisor_blockers: list[str] = [] + if not inputs.gvisor_runsc_version: + gvisor_blockers.append("gvisor_runsc_version_missing") + if not inputs.gvisor_registered_runtime: + gvisor_blockers.append("gvisor_registered_runtime_missing") + if inputs.gvisor_registered_runtime and not inputs.gvisor_runtime_name: + gvisor_blockers.append("gvisor_runtime_name_missing") + + cpu_blockers: list[str] = [] + if not inputs.cpu_squeeze_success: + cpu_blockers.append("cpu_squeeze_missing") + if inputs.cpu_squeeze_workers < 2: + cpu_blockers.append("cpu_squeeze_concurrency_missing") + + blockers = firecracker_blockers + gvisor_blockers + cpu_blockers + return { + "schema_version": INFRA_HARDENING_SCHEMA, + "component": "phase4_production_infra_hardening", + "claim_boundary": "phase4_production_infra_hardening_non_promotional_scope", + "promotional": False, + "scorecard_update_allowed": False, + "inputs": { + "firecracker_version": inputs.firecracker_version, + "firecracker_non_root_boot": inputs.firecracker_non_root_boot, + "firecracker_jailer_available": inputs.firecracker_jailer_available, + "firecracker_networking_configured": inputs.firecracker_networking_configured, + "firecracker_concurrent_microvms": inputs.firecracker_concurrent_microvms, + "gvisor_runsc_version": inputs.gvisor_runsc_version, + "gvisor_registered_runtime": inputs.gvisor_registered_runtime, + "gvisor_runtime_name": inputs.gvisor_runtime_name, + "cpu_squeeze_workers": inputs.cpu_squeeze_workers, + "cpu_squeeze_success": inputs.cpu_squeeze_success, + }, + "checks": { + "firecracker": {"passed": not firecracker_blockers, "blockers": firecracker_blockers}, + "gvisor": {"passed": not gvisor_blockers, "blockers": gvisor_blockers}, + "cpu_squeeze": {"passed": not cpu_blockers, "blockers": cpu_blockers}, + }, + "blockers": blockers, + "passed": not blockers, + } diff --git a/breadboard/rl/phase4/native_inference.py b/breadboard/rl/phase4/native_inference.py new file mode 100644 index 00000000..be822314 --- /dev/null +++ b/breadboard/rl/phase4/native_inference.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import hashlib +import json +import time +import uuid +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Protocol + +BREADBOARD_NATIVE_INFERENCE_OWNER = "breadboard_native_sub_inference_lane" +BREADBOARD_NATIVE_LANE_SCHEMA = "bb.rl.phase4.native_inference_lane.v1" + + +class CompletionSession(Protocol): + def post(self, url: str, *, json: dict[str, Any], timeout: float): ... # pragma: no cover - protocol + + +class CompletionResponseLike(Protocol): + status_code: int + + def json(self) -> dict[str, Any]: ... # pragma: no cover - protocol + def raise_for_status(self) -> None: ... # pragma: no cover - protocol + + +class TokenizerLike(Protocol): + def decode(self, token_ids: list[int], *, skip_special_tokens: bool = False) -> str: ... + def encode(self, text: str, *, add_special_tokens: bool = False) -> list[int]: ... + + +def sha256_bytes(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def sha256_text(text: str) -> str: + return sha256_bytes(text.encode("utf-8")) + + +def sha256_json(payload: Any) -> str: + return sha256_text(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def _safe_id_part(value: object) -> str: + text = str(value or "").strip() + safe = "".join(ch if ch.isalnum() or ch in {"_", "-", "."} else "-" for ch in text).strip("-._") + return safe[:96] or "request" + + +@dataclass(frozen=True) +class NativeCompletionResponse: + request_id: str + upstream_request_id: str + response_id: str + model_ref: str + prompt_text: str + output_text: str + posthoc_token_ids: list[int] + backend_token_texts: list[str] + backend_token_logprobs: list[float] + backend_token_ids: list[int] + latency_ms: float + http_status: int + request_sha256: str + response_sha256: str + output_text_sha256: str + posthoc_token_ids_sha256: str + backend_token_texts_sha256: str + backend_token_logprobs_sha256: str + backend_token_ids_sha256: str + backend_completion_id: str + passed: bool + error_type: str = "" + error_message: str = "" + raw_response_preview: str = "" + + +@dataclass(frozen=True) +class NativeCompletionRecord: + schema_version: str + inference_owner: str + breadboard_native_lane_used: bool + request_id: str + upstream_request_id: str + response_id: str + model_ref: str + prompt_sha256: str + sampling_config_sha256: str + request_sha256: str + response_sha256: str + output_text_sha256: str + posthoc_token_ids_sha256: str + backend_token_texts_sha256: str + backend_token_logprobs_sha256: str + backend_token_ids_sha256: str + posthoc_token_count: int + backend_token_text_count: int + backend_token_logprob_count: int + backend_token_id_count: int + latency_ms: float + http_status: int + backend_completion_id: str + passed: bool + error_type: str = "" + error_message: str = "" + recorded_at_unix_ms: int = field(default_factory=lambda: int(time.time() * 1000)) + + +class NativeInferenceLane: + """BreadBoard-owned request/response adapter for target inference evidence. + + The model engine can still be vLLM. This class owns the sub-inference request + path: request id, payload construction, response id, response normalization, + token/text hashes, latency, and append-only evidence records. + """ + + def __init__( + self, + *, + model_ref: str, + base_url: str, + tokenizer: TokenizerLike, + request_log_path: Path, + target_run_id: str, + session: CompletionSession | None = None, + timeout_seconds: float = 180.0, + ) -> None: + if not model_ref: + raise ValueError("model_ref is required") + if not base_url: + raise ValueError("base_url is required") + self.model_ref = model_ref + self.base_url = base_url.rstrip("/") + self.tokenizer = tokenizer + self.request_log_path = request_log_path + self.target_run_id = target_run_id + self.timeout_seconds = timeout_seconds + if session is None: + import requests + + session = requests + self.session = session + self.generate_calls = 0 + self.last_response: NativeCompletionResponse | None = None + self.request_log_path.parent.mkdir(parents=True, exist_ok=True) + + def generate_completion( + self, + *, + upstream_request_id: object, + prompt_ids: list[int], + sampling_params: dict[str, Any], + ) -> NativeCompletionResponse: + self.generate_calls += 1 + upstream_id = _safe_id_part(upstream_request_id) + request_id = f"bbreq-{_safe_id_part(self.target_run_id)}-{upstream_id}-{uuid.uuid4().hex[:12]}" + prompt_text = self.tokenizer.decode(list(prompt_ids), skip_special_tokens=False) + max_tokens = int(sampling_params.get("max_tokens", 128)) + temperature = float(sampling_params.get("temperature", 0.0)) + payload = { + "model": self.model_ref, + "prompt": prompt_text, + "max_tokens": max_tokens, + "temperature": temperature, + "logprobs": int(sampling_params.get("logprobs", 1)), + } + started = time.perf_counter() + http_status = 0 + response_payload: dict[str, Any] = {} + output_text = "" + posthoc_token_ids: list[int] = [] + backend_token_texts: list[str] = [] + backend_token_logprobs: list[float] = [] + backend_token_ids: list[int] = [] + backend_completion_id = "" + error_type = "" + error_message = "" + passed = False + try: + response = self.session.post(f"{self.base_url}/v1/completions", json=payload, timeout=self.timeout_seconds) + http_status = int(getattr(response, "status_code", 0) or 0) + response.raise_for_status() + response_payload = response.json() + choices = response_payload.get("choices") or [] + choice = choices[0] if choices else {} + output_text = str(choice.get("text") or "") + logprobs = choice.get("logprobs") if isinstance(choice, dict) else None + if isinstance(logprobs, dict): + backend_token_texts = [str(token) for token in (logprobs.get("tokens") or [])] + backend_token_logprobs = [float(value) for value in (logprobs.get("token_logprobs") or []) if value is not None] + raw_backend_token_ids = choice.get("token_ids") if isinstance(choice, dict) else None + if isinstance(raw_backend_token_ids, list): + backend_token_ids = [int(token_id) for token_id in raw_backend_token_ids] + posthoc_token_ids = self.tokenizer.encode(output_text, add_special_tokens=False) + backend_completion_id = str(response_payload.get("id") or "") + passed = True + except Exception as exc: # noqa: BLE001 + error_type = exc.__class__.__name__ + error_message = str(exc) + response_payload = {"error_type": error_type, "error_message": error_message} + latency_ms = (time.perf_counter() - started) * 1000.0 + response_id = "bbresp-" + hashlib.sha256( + f"{request_id}:{backend_completion_id}:{sha256_json(response_payload)}".encode("utf-8") + ).hexdigest()[:24] + record = NativeCompletionRecord( + schema_version=BREADBOARD_NATIVE_LANE_SCHEMA, + inference_owner=BREADBOARD_NATIVE_INFERENCE_OWNER, + breadboard_native_lane_used=True, + request_id=request_id, + upstream_request_id=str(upstream_request_id), + response_id=response_id, + model_ref=self.model_ref, + prompt_sha256=sha256_text(prompt_text), + sampling_config_sha256=sha256_json({"max_tokens": max_tokens, "temperature": temperature, "logprobs": payload["logprobs"]}), + request_sha256=sha256_json(payload), + response_sha256=sha256_json(response_payload), + output_text_sha256=sha256_text(output_text), + posthoc_token_ids_sha256=sha256_json(posthoc_token_ids), + backend_token_texts_sha256=sha256_json(backend_token_texts), + backend_token_logprobs_sha256=sha256_json(backend_token_logprobs), + backend_token_ids_sha256=sha256_json(backend_token_ids), + posthoc_token_count=len(posthoc_token_ids), + backend_token_text_count=len(backend_token_texts), + backend_token_logprob_count=len(backend_token_logprobs), + backend_token_id_count=len(backend_token_ids), + latency_ms=latency_ms, + http_status=http_status, + backend_completion_id=backend_completion_id, + passed=passed, + error_type=error_type, + error_message=error_message, + ) + self._append_record(record) + native_response = NativeCompletionResponse( + request_id=request_id, + upstream_request_id=str(upstream_request_id), + response_id=response_id, + model_ref=self.model_ref, + prompt_text=prompt_text, + output_text=output_text, + posthoc_token_ids=posthoc_token_ids, + backend_token_texts=backend_token_texts, + backend_token_logprobs=backend_token_logprobs, + backend_token_ids=backend_token_ids, + latency_ms=latency_ms, + http_status=http_status, + request_sha256=record.request_sha256, + response_sha256=record.response_sha256, + output_text_sha256=record.output_text_sha256, + posthoc_token_ids_sha256=record.posthoc_token_ids_sha256, + backend_token_texts_sha256=record.backend_token_texts_sha256, + backend_token_logprobs_sha256=record.backend_token_logprobs_sha256, + backend_token_ids_sha256=record.backend_token_ids_sha256, + backend_completion_id=backend_completion_id, + passed=passed, + error_type=error_type, + error_message=error_message, + raw_response_preview=json.dumps(response_payload, sort_keys=True)[:1000], + ) + self.last_response = native_response + if not passed: + raise RuntimeError(f"native inference request failed: {error_type}: {error_message}") + return native_response + + def status(self) -> dict[str, Any]: + last = self.last_response + return { + "schema_version": BREADBOARD_NATIVE_LANE_SCHEMA, + "inference_owner": BREADBOARD_NATIVE_INFERENCE_OWNER, + "breadboard_native_lane_used": True, + "generate_calls": self.generate_calls, + "request_log_path": str(self.request_log_path), + "request_log_sha256": sha256_file(self.request_log_path) if self.request_log_path.exists() else "", + "last_request_id": last.request_id if last else "", + "last_response_id": last.response_id if last else "", + "last_model_ref": last.model_ref if last else self.model_ref, + "last_output_text": last.output_text[:1000] if last else "", + "last_output_text_sha256": last.output_text_sha256 if last else "", + "last_posthoc_token_ids_sha256": last.posthoc_token_ids_sha256 if last else "", + "last_backend_token_texts_sha256": last.backend_token_texts_sha256 if last else "", + "last_backend_token_logprobs_sha256": last.backend_token_logprobs_sha256 if last else "", + "last_backend_token_ids_sha256": last.backend_token_ids_sha256 if last else "", + "last_posthoc_token_count": len(last.posthoc_token_ids) if last else 0, + "last_backend_token_text_count": len(last.backend_token_texts) if last else 0, + "last_backend_token_logprob_count": len(last.backend_token_logprobs) if last else 0, + "last_backend_token_id_count": len(last.backend_token_ids) if last else 0, + "last_http_status": last.http_status if last else 0, + "last_latency_ms": last.latency_ms if last else 0.0, + } + + def _append_record(self, record: NativeCompletionRecord) -> None: + with self.request_log_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(asdict(record), sort_keys=True, separators=(",", ":")) + "\n") diff --git a/breadboard/rl/phase4/wrapper_identity.py b/breadboard/rl/phase4/wrapper_identity.py new file mode 100644 index 00000000..f12a4930 --- /dev/null +++ b/breadboard/rl/phase4/wrapper_identity.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Callable + +import yaml + +WRAPPER_IDENTITY_SCHEMA = "bb.rl.phase4.wrapper_identity.v1" +REQUIRED_SUBMODULES = { + "verl": "third_party/verl", + "nemo_gym": "third_party/nemo-gym", +} +PACKAGE_SENTINELS = { + "verl": ("pyproject.toml", "setup.py", "verl/__init__.py"), + "nemo_gym": ("pyproject.toml", "setup.py", "nemo_gym/__init__.py"), +} +GitRunner = Callable[[list[str], Path], str] + + +def sha256_bytes(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def sha256_json(payload: Any) -> str: + return sha256_bytes(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")) + + +def _run_git(args: list[str], cwd: Path) -> str: + try: + result = subprocess.run( + ["git", *args], + cwd=cwd, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired): + return "" + return result.stdout.strip() if result.returncode == 0 else "" + + +def _directory_digest(root: Path) -> str: + h = hashlib.sha256() + any_file = False + if not root.exists() or not root.is_dir(): + return "" + for path in sorted(root.rglob("*"), key=lambda item: str(item.relative_to(root))): + if path.is_dir() or "__pycache__" in path.parts or path.suffix == ".pyc" or ".git" in path.parts: + continue + any_file = True + h.update(str(path.relative_to(root)).replace("\\", "/").encode("utf-8") + b"\0" + path.read_bytes() + b"\0") + return "sha256:" + h.hexdigest() if any_file else "" + + +def parse_deps_pins(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + loaded = yaml.safe_load(path.read_text()) or {} + if not isinstance(loaded, dict): + return {} + pins: dict[str, str] = {} + for section, values in loaded.items(): + if not isinstance(section, str) or not isinstance(values, dict): + continue + for key in ("pin", "commit", "rev", "branch"): + value = values.get(key) + if isinstance(value, str) and value.strip(): + pins[f"{section}_{key}"] = value.strip() + return pins + + +def parse_submodule_status(raw: str) -> dict[str, dict[str, str]]: + rows: dict[str, dict[str, str]] = {} + for line in raw.splitlines(): + stripped = line.strip() + if not stripped: + continue + marker = stripped[0] if stripped[0] in {"-", "+", "U"} else "" + parts = stripped.lstrip("-+U ").split() + if len(parts) >= 2: + rows[parts[1]] = {"path": parts[1], "commit": parts[0], "marker": marker} + return rows + + +@dataclass(frozen=True) +class WrapperSubmoduleIdentity: + name: str + relative_path: str + path: str + expected_commit: str + actual_commit: str + submodule_commit: str + submodule_marker: str + package_sentinel: str + content_sha256: str + blockers: list[str] = field(default_factory=list) + + @property + def passed(self) -> bool: + return not self.blockers + + def to_dict(self) -> dict[str, Any]: + payload = asdict(self) + payload["passed"] = self.passed + return payload + + +@dataclass(frozen=True) +class WrapperIdentity: + schema_version: str + wrapper_path: str + wrapper_commit: str + wrapper_ref: str + deps_yaml_path: str + deps_yaml_sha256: str + pins: dict[str, str] + submodules: dict[str, dict[str, str]] + components: dict[str, WrapperSubmoduleIdentity] + blockers: list[str] = field(default_factory=list) + + @property + def passed(self) -> bool: + return not self.blockers and all(component.passed for component in self.components.values()) + + def to_dict(self) -> dict[str, Any]: + return { + "schema_version": self.schema_version, + "wrapper_path": self.wrapper_path, + "wrapper_commit": self.wrapper_commit, + "wrapper_ref": self.wrapper_ref, + "deps_yaml_path": self.deps_yaml_path, + "deps_yaml_sha256": self.deps_yaml_sha256, + "pins": dict(self.pins), + "submodules": dict(self.submodules), + "components": {name: component.to_dict() for name, component in self.components.items()}, + "blockers": list(self.blockers), + "passed": self.passed, + } + + +def _sha_file(path: Path) -> str: + return sha256_bytes(path.read_bytes()) if path.exists() and path.is_file() else "" + + +def _first_existing(root: Path, candidates: tuple[str, ...]) -> str: + for candidate in candidates: + if (root / candidate).exists(): + return candidate + return "" + + +def _expected_pin(name: str, pins: dict[str, str]) -> str: + return pins.get(f"{name}_pin") or pins.get(f"{name}_commit") or pins.get(f"{name}_rev") or "" + + +def collect_wrapper_identity(wrapper_dir: Path, *, git_runner: GitRunner = _run_git) -> WrapperIdentity: + wrapper_dir = wrapper_dir.resolve() + deps_path = wrapper_dir / "deps.yaml" + deps_sha = _sha_file(deps_path) + pins = parse_deps_pins(deps_path) + raw_status = git_runner(["submodule", "status", "--recursive"], wrapper_dir) + submodules = parse_submodule_status(raw_status) + wrapper_commit = git_runner(["rev-parse", "HEAD"], wrapper_dir) + wrapper_ref = git_runner(["rev-parse", "--abbrev-ref", "HEAD"], wrapper_dir) + blockers: list[str] = [] + if not wrapper_dir.exists(): + blockers.append("wrapper_missing") + if not wrapper_commit: + blockers.append("wrapper_commit_missing") + if not deps_path.exists(): + blockers.append("deps_yaml_missing") + + components: dict[str, WrapperSubmoduleIdentity] = {} + for name, rel in REQUIRED_SUBMODULES.items(): + path = wrapper_dir / rel + status = submodules.get(rel, {}) + component_blockers: list[str] = [] + expected = _expected_pin(name, pins) + if not expected: + component_blockers.append(f"submodule_expected_pin_missing:{rel}") + actual = git_runner(["rev-parse", "HEAD"], path) if path.exists() else "" + sentinel = _first_existing(path, PACKAGE_SENTINELS[name]) if path.exists() else "" + digest = _directory_digest(path) + marker = status.get("marker", "") + status_commit = status.get("commit", "") + if rel not in submodules: + component_blockers.append(f"submodule_status_missing:{rel}") + if marker == "-": + component_blockers.append(f"submodule_uninitialized:{rel}") + elif marker == "+": + component_blockers.append(f"submodule_dirty:{rel}") + elif marker == "U": + component_blockers.append(f"submodule_conflicted:{rel}") + if not path.exists() or not path.is_dir(): + component_blockers.append(f"submodule_path_missing:{rel}") + if path.exists() and not digest: + component_blockers.append(f"submodule_empty:{rel}") + if not sentinel: + component_blockers.append(f"package_sentinel_missing:{rel}") + if not actual: + component_blockers.append(f"submodule_commit_missing:{rel}") + if expected and actual and expected != actual: + component_blockers.append(f"submodule_pin_mismatch:{rel}") + if status_commit and actual and status_commit != actual: + component_blockers.append(f"submodule_status_commit_mismatch:{rel}") + components[name] = WrapperSubmoduleIdentity( + name=name, + relative_path=rel, + path=str(path), + expected_commit=expected, + actual_commit=actual, + submodule_commit=status_commit, + submodule_marker=marker, + package_sentinel=sentinel, + content_sha256=digest, + blockers=component_blockers, + ) + blockers.extend(blocker for component in components.values() for blocker in component.blockers) + return WrapperIdentity( + schema_version=WRAPPER_IDENTITY_SCHEMA, + wrapper_path=str(wrapper_dir), + wrapper_commit=wrapper_commit, + wrapper_ref=wrapper_ref, + deps_yaml_path=str(deps_path), + deps_yaml_sha256=deps_sha, + pins=pins, + submodules=submodules, + components=components, + blockers=blockers, + ) + + +def runtime_module_provenance(wrapper_dir: Path, module_files: dict[str, str]) -> dict[str, Any]: + wrapper_dir = wrapper_dir.resolve() + expected_roots = { + "zyphra_verl": wrapper_dir / "src", + "verl": wrapper_dir / "third_party" / "verl", + "nemo_gym": wrapper_dir / "third_party" / "nemo-gym", + } + modules: dict[str, dict[str, Any]] = {} + blockers: list[str] = [] + for name, root in expected_roots.items(): + module_file = module_files.get(name, "") + try: + resolved = Path(module_file).resolve() if module_file else Path("") + path_match = bool(module_file) and resolved.is_relative_to(root.resolve()) + except (OSError, RuntimeError, ValueError): + path_match = False + if not path_match: + blockers.append(f"runtime_identity_mismatch:{name}") + modules[name] = { + "module_file": module_file, + "expected_root": str(root), + "path_match": path_match, + } + return {"schema_version": "bb.rl.phase4.runtime_module_provenance.v1", "modules": modules, "blockers": blockers, "passed": not blockers} diff --git a/breadboard/rl/renderer/__init__.py b/breadboard/rl/renderer/__init__.py new file mode 100644 index 00000000..915a466f --- /dev/null +++ b/breadboard/rl/renderer/__init__.py @@ -0,0 +1,21 @@ +"""Renderer and token-native record primitives for RL rollout exports.""" + +from breadboard.rl.renderer.records import ( + classify_rendered_turn_trainability, + validate_rendered_turn, +) +from breadboard.rl.renderer.schema import ( + BridgeToNextTurn, + ProviderFidelity, + RenderedTurnRecord, + RendererIdentity, +) + +__all__ = [ + "BridgeToNextTurn", + "ProviderFidelity", + "RenderedTurnRecord", + "RendererIdentity", + "classify_rendered_turn_trainability", + "validate_rendered_turn", +] diff --git a/breadboard/rl/renderer/conformance.py b/breadboard/rl/renderer/conformance.py new file mode 100644 index 00000000..a94267c0 --- /dev/null +++ b/breadboard/rl/renderer/conformance.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from collections import Counter +from typing import Any, Sequence + +from breadboard.rl.renderer.records import classify_rendered_turn_trainability, validate_rendered_turn +from breadboard.rl.renderer.schema import RenderedTurnRecord + + +def build_renderer_conformance_report(records: Sequence[RenderedTurnRecord]) -> dict[str, Any]: + """Summarize renderer-token record validity without promoting support claims.""" + + fidelity_counts = Counter(record.provider_fidelity.fidelity_class for record in records) + invalid_records: list[dict[str, Any]] = [] + trainability = [] + for record in records: + errors = validate_rendered_turn(record) + if errors: + invalid_records.append({"turn_id": record.turn_id, "errors": errors}) + trainability.append(classify_rendered_turn_trainability(record).to_dict()) + return { + "record_count": len(records), + "valid_record_count": len(records) - len(invalid_records), + "invalid_records": invalid_records, + "fidelity_counts": dict(fidelity_counts), + "sft_trainable_count": sum(1 for item in trainability if item["sft_trainable"]), + "on_policy_trainable_count": sum(1 for item in trainability if item["on_policy_trainable"]), + "claim_boundary": "renderer_conformance_only_not_verl_support", + } diff --git a/breadboard/rl/renderer/records.py b/breadboard/rl/renderer/records.py new file mode 100644 index 00000000..e587059c --- /dev/null +++ b/breadboard/rl/renderer/records.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from breadboard.rl.renderer.schema import RenderedTurnRecord + + +TOKEN_ALIGNED_FIELDS = [ + "attention_mask", + "loss_mask", + "assistant_mask", + "tool_action_mask", + "reward_mask", + "sampled_mask", + "message_indices", +] +VALID_TOOL_PARSE_STATUSES = {"not_applicable", "ok", "failed"} + + +@dataclass(frozen=True) +class TrainabilityDecision: + sft_trainable: bool + on_policy_trainable: bool + fidelity_class: str + blocked_reasons: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "sft_trainable": self.sft_trainable, + "on_policy_trainable": self.on_policy_trainable, + "fidelity_class": self.fidelity_class, + "blocked_reasons": list(self.blocked_reasons), + } + + +def validate_rendered_turn(record: RenderedTurnRecord) -> list[str]: + errors: list[str] = [] + input_len = len(record.input_ids) + + if record.input_ids != [*record.prompt_ids, *record.completion_ids]: + errors.append("input_ids must equal prompt_ids + completion_ids") + if not record.input_ids: + errors.append("input_ids must be non-empty") + if not record.completion_ids: + errors.append("completion_ids must be non-empty") + + for field_name in TOKEN_ALIGNED_FIELDS: + if len(getattr(record, field_name)) != input_len: + errors.append(f"{field_name} length must equal input_ids length") + + if len(record.attention_mask) == input_len and any(item not in {0, 1} for item in record.attention_mask): + errors.append("attention_mask values must be 0 or 1") + + if record.completion_logprobs is not None and len(record.completion_logprobs) != len(record.completion_ids): + errors.append("completion_logprobs length must equal completion_ids length") + + if record.tool_parse_status not in VALID_TOOL_PARSE_STATUSES: + errors.append(f"tool_parse_status must be one of {sorted(VALID_TOOL_PARSE_STATUSES)}") + if record.tool_calls and record.tool_parse_status != "ok": + errors.append("tool_calls require tool_parse_status=ok") + if record.tool_calls and not any(record.tool_action_mask): + errors.append("tool_calls require at least one true tool_action_mask entry") + + completion_start = len(record.prompt_ids) + if completion_start < input_len: + completion_assistant_mask = record.assistant_mask[completion_start:] + if not any(completion_assistant_mask): + errors.append("completion tokens require at least one assistant_mask entry") + if not any(record.sampled_mask[completion_start:]): + errors.append("completion tokens require sampled_mask entries") + + if record.provider_fidelity.fidelity_class == "F0": + if record.provider_fidelity.token_ids_source != "unavailable": + errors.append("F0 provider fidelity requires token_ids_source=unavailable") + if record.provider_fidelity.logprobs_source != "unavailable": + errors.append("F0 provider fidelity requires logprobs_source=unavailable") + if record.provider_fidelity.fidelity_class == "F1": + if record.provider_fidelity.token_ids_source != "posthoc_tokenizer": + errors.append("F1 provider fidelity requires token_ids_source=posthoc_tokenizer") + if record.provider_fidelity.logprobs_source not in {"unavailable", "posthoc_actor"}: + errors.append("F1 provider fidelity requires logprobs_source unavailable or posthoc_actor") + if record.provider_fidelity.fidelity_class == "F2": + if record.provider_fidelity.token_ids_source != "provider_native": + errors.append("F2 provider fidelity requires token_ids_source=provider_native") + if record.provider_fidelity.logprobs_source != "unavailable": + errors.append("F2 provider fidelity requires logprobs_source=unavailable") + if record.provider_fidelity.fidelity_class == "F3": + if record.provider_fidelity.token_ids_source != "provider_native": + errors.append("F3 provider fidelity requires token_ids_source=provider_native") + if record.provider_fidelity.logprobs_source != "provider_native": + errors.append("F3 provider fidelity requires logprobs_source=provider_native") + if record.completion_logprobs is None: + errors.append("F3 provider fidelity requires completion_logprobs") + + return errors + + +def classify_rendered_turn_trainability(record: RenderedTurnRecord) -> TrainabilityDecision: + blocked_reasons = validate_rendered_turn(record) + + if record.finish_reason == "length": + blocked_reasons.append("finish_reason=length") + if record.is_truncated: + blocked_reasons.append("is_truncated") + if record.overlong_prompt: + blocked_reasons.append("overlong_prompt") + if record.bridge_to_next_turn.attempted and not record.bridge_to_next_turn.success: + blocked_reasons.append("bridge_to_next_turn_failed") + + fidelity_class = record.provider_fidelity.fidelity_class + if fidelity_class == "F0": + blocked_reasons.append("message_only_provider_fidelity") + + sft_trainable = not blocked_reasons and fidelity_class in {"F1", "F2", "F3"} + on_policy_trainable = not blocked_reasons and fidelity_class == "F3" + if fidelity_class in {"F1", "F2"}: + blocked_reasons.append("on_policy_requires_f3_token_native_logprobs") + if fidelity_class == "F0": + sft_trainable = False + on_policy_trainable = False + + return TrainabilityDecision( + sft_trainable=sft_trainable, + on_policy_trainable=on_policy_trainable, + fidelity_class=fidelity_class, + blocked_reasons=list(dict.fromkeys(blocked_reasons)), + ) diff --git a/breadboard/rl/renderer/schema.py b/breadboard/rl/renderer/schema.py new file mode 100644 index 00000000..9aae3626 --- /dev/null +++ b/breadboard/rl/renderer/schema.py @@ -0,0 +1,339 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Mapping + + +FIDELITY_CLASSES = { + "F0": "message_only", + "F1": "tokenized_posthoc", + "F2": "token_native", + "F3": "token_native_with_logprobs", +} + + +def _text(value: Any, field_name: str) -> str: + text = str(value or "").strip() + if not text: + raise ValueError(f"{field_name} must be non-empty") + return text + + +def _optional_text(value: Any) -> str | None: + text = str(value or "").strip() + return text or None + + +def _int_list(value: Any, field_name: str) -> list[int]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError(f"{field_name} must be a list") + return [int(item) for item in value] + + +def _bool_list(value: Any, field_name: str) -> list[bool]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError(f"{field_name} must be a list") + return [bool(item) for item in value] + + +def _float_list(value: Any, field_name: str) -> list[float] | None: + if value is None: + return None + if not isinstance(value, list): + raise ValueError(f"{field_name} must be a list") + return [float(item) for item in value] + + +def _mapping(value: Any, field_name: str) -> dict[str, Any]: + if value is None: + return {} + if not isinstance(value, Mapping): + raise ValueError(f"{field_name} must be a mapping") + return dict(value) + + +def _mapping_list(value: Any, field_name: str) -> list[dict[str, Any]]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError(f"{field_name} must be a list") + copied: list[dict[str, Any]] = [] + for index, item in enumerate(value): + if not isinstance(item, Mapping): + raise ValueError(f"{field_name}[{index}] must be a mapping") + copied.append(dict(item)) + return copied + + +@dataclass(frozen=True) +class RendererIdentity: + renderer_id: str + renderer_version: str + renderer_config_hash: str + tokenizer_id: str + tokenizer_hash: str + chat_template_id: str + chat_template_hash: str + stop_token_ids: list[int] = field(default_factory=list) + + def __post_init__(self) -> None: + object.__setattr__(self, "renderer_id", _text(self.renderer_id, "renderer_id")) + object.__setattr__(self, "renderer_version", _text(self.renderer_version, "renderer_version")) + object.__setattr__( + self, + "renderer_config_hash", + _text(self.renderer_config_hash, "renderer_config_hash"), + ) + object.__setattr__(self, "tokenizer_id", _text(self.tokenizer_id, "tokenizer_id")) + object.__setattr__(self, "tokenizer_hash", _text(self.tokenizer_hash, "tokenizer_hash")) + object.__setattr__(self, "chat_template_id", _text(self.chat_template_id, "chat_template_id")) + object.__setattr__( + self, + "chat_template_hash", + _text(self.chat_template_hash, "chat_template_hash"), + ) + object.__setattr__(self, "stop_token_ids", [int(item) for item in self.stop_token_ids]) + + def to_dict(self) -> dict[str, Any]: + return { + "renderer_id": self.renderer_id, + "renderer_version": self.renderer_version, + "renderer_config_hash": self.renderer_config_hash, + "tokenizer_id": self.tokenizer_id, + "tokenizer_hash": self.tokenizer_hash, + "chat_template_id": self.chat_template_id, + "chat_template_hash": self.chat_template_hash, + "stop_token_ids": list(self.stop_token_ids), + } + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "RendererIdentity": + return RendererIdentity( + renderer_id=data.get("renderer_id") or "", + renderer_version=data.get("renderer_version") or "", + renderer_config_hash=data.get("renderer_config_hash") or "", + tokenizer_id=data.get("tokenizer_id") or "", + tokenizer_hash=data.get("tokenizer_hash") or "", + chat_template_id=data.get("chat_template_id") or "", + chat_template_hash=data.get("chat_template_hash") or "", + stop_token_ids=_int_list(data.get("stop_token_ids"), "renderer.stop_token_ids"), + ) + + +@dataclass(frozen=True) +class BridgeToNextTurn: + attempted: bool + success: bool + failure_reason: str | None = None + + def __post_init__(self) -> None: + failure_reason = _optional_text(self.failure_reason) + object.__setattr__(self, "attempted", bool(self.attempted)) + object.__setattr__(self, "success", bool(self.success)) + object.__setattr__(self, "failure_reason", failure_reason) + if not self.attempted and self.success: + raise ValueError("bridge_to_next_turn cannot succeed when it was not attempted") + if self.attempted and not self.success and not failure_reason: + raise ValueError("bridge_to_next_turn failure requires failure_reason") + + def to_dict(self) -> dict[str, Any]: + payload = {"attempted": self.attempted, "success": self.success} + if self.failure_reason: + payload["failure_reason"] = self.failure_reason + return payload + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "BridgeToNextTurn": + return BridgeToNextTurn( + attempted=bool(data.get("attempted")), + success=bool(data.get("success")), + failure_reason=data.get("failure_reason"), + ) + + +@dataclass(frozen=True) +class ProviderFidelity: + fidelity_class: str + token_ids_source: str + logprobs_source: str + provider: str + model_requested: str + model_served: str + sampling_config: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + fidelity_class = _text(self.fidelity_class, "fidelity_class").upper() + if fidelity_class not in FIDELITY_CLASSES: + raise ValueError(f"fidelity_class must be one of {sorted(FIDELITY_CLASSES)}") + object.__setattr__(self, "fidelity_class", fidelity_class) + object.__setattr__(self, "token_ids_source", _text(self.token_ids_source, "token_ids_source")) + object.__setattr__(self, "logprobs_source", _text(self.logprobs_source, "logprobs_source")) + object.__setattr__(self, "provider", _text(self.provider, "provider")) + object.__setattr__(self, "model_requested", _text(self.model_requested, "model_requested")) + object.__setattr__(self, "model_served", _text(self.model_served, "model_served")) + object.__setattr__(self, "sampling_config", dict(self.sampling_config or {})) + + def to_dict(self) -> dict[str, Any]: + return { + "fidelity_class": self.fidelity_class, + "fidelity_name": FIDELITY_CLASSES[self.fidelity_class], + "token_ids_source": self.token_ids_source, + "logprobs_source": self.logprobs_source, + "provider": self.provider, + "model_requested": self.model_requested, + "model_served": self.model_served, + "sampling_config": dict(self.sampling_config), + } + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "ProviderFidelity": + return ProviderFidelity( + fidelity_class=data.get("fidelity_class") or "", + token_ids_source=data.get("token_ids_source") or "", + logprobs_source=data.get("logprobs_source") or "", + provider=data.get("provider") or "", + model_requested=data.get("model_requested") or "", + model_served=data.get("model_served") or "", + sampling_config=_mapping(data.get("sampling_config"), "provider_fidelity.sampling_config"), + ) + + +@dataclass(frozen=True) +class RenderedTurnRecord: + rollout_id: str + trajectory_id: str + task_id: str + split_id: str + env_package_hash: str + turn_id: str + renderer: RendererIdentity + provider_fidelity: ProviderFidelity + prompt_ids: list[int] + completion_ids: list[int] + input_ids: list[int] + attention_mask: list[int] + loss_mask: list[bool] + assistant_mask: list[bool] + tool_action_mask: list[bool] + reward_mask: list[bool] + sampled_mask: list[bool] + message_indices: list[int] + bridge_to_next_turn: BridgeToNextTurn + tool_parse_status: str = "not_applicable" + parsed_completion: str | None = None + tool_calls: list[dict[str, Any]] = field(default_factory=list) + completion_logprobs: list[float] | None = None + finish_reason: str = "stop" + is_truncated: bool = False + overlong_prompt: bool = False + metadata: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + for field_name in [ + "rollout_id", + "trajectory_id", + "task_id", + "split_id", + "env_package_hash", + "turn_id", + ]: + object.__setattr__(self, field_name, _text(getattr(self, field_name), field_name)) + renderer = self.renderer if isinstance(self.renderer, RendererIdentity) else RendererIdentity.from_dict(self.renderer) + fidelity = ( + self.provider_fidelity + if isinstance(self.provider_fidelity, ProviderFidelity) + else ProviderFidelity.from_dict(self.provider_fidelity) + ) + bridge = ( + self.bridge_to_next_turn + if isinstance(self.bridge_to_next_turn, BridgeToNextTurn) + else BridgeToNextTurn.from_dict(self.bridge_to_next_turn) + ) + object.__setattr__(self, "renderer", renderer) + object.__setattr__(self, "provider_fidelity", fidelity) + object.__setattr__(self, "bridge_to_next_turn", bridge) + for field_name in ["prompt_ids", "completion_ids", "input_ids", "attention_mask", "message_indices"]: + object.__setattr__(self, field_name, [int(item) for item in getattr(self, field_name)]) + for field_name in ["loss_mask", "assistant_mask", "tool_action_mask", "reward_mask", "sampled_mask"]: + object.__setattr__(self, field_name, [bool(item) for item in getattr(self, field_name)]) + if self.completion_logprobs is not None: + object.__setattr__(self, "completion_logprobs", [float(item) for item in self.completion_logprobs]) + object.__setattr__(self, "tool_parse_status", _text(self.tool_parse_status, "tool_parse_status")) + object.__setattr__(self, "parsed_completion", _optional_text(self.parsed_completion)) + object.__setattr__(self, "tool_calls", [dict(item) for item in self.tool_calls]) + object.__setattr__(self, "finish_reason", _text(self.finish_reason, "finish_reason")) + object.__setattr__(self, "is_truncated", bool(self.is_truncated)) + object.__setattr__(self, "overlong_prompt", bool(self.overlong_prompt)) + object.__setattr__(self, "metadata", dict(self.metadata or {})) + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "rollout_id": self.rollout_id, + "trajectory_id": self.trajectory_id, + "task_id": self.task_id, + "split_id": self.split_id, + "env_package_hash": self.env_package_hash, + "turn_id": self.turn_id, + "renderer": self.renderer.to_dict(), + "provider_fidelity": self.provider_fidelity.to_dict(), + "prompt_ids": list(self.prompt_ids), + "completion_ids": list(self.completion_ids), + "input_ids": list(self.input_ids), + "attention_mask": list(self.attention_mask), + "loss_mask": list(self.loss_mask), + "assistant_mask": list(self.assistant_mask), + "tool_action_mask": list(self.tool_action_mask), + "reward_mask": list(self.reward_mask), + "sampled_mask": list(self.sampled_mask), + "message_indices": list(self.message_indices), + "bridge_to_next_turn": self.bridge_to_next_turn.to_dict(), + "tool_parse_status": self.tool_parse_status, + "tool_calls": [dict(item) for item in self.tool_calls], + "finish_reason": self.finish_reason, + "is_truncated": self.is_truncated, + "overlong_prompt": self.overlong_prompt, + "metadata": dict(self.metadata), + } + if self.parsed_completion: + payload["parsed_completion"] = self.parsed_completion + if self.completion_logprobs is not None: + payload["completion_logprobs"] = list(self.completion_logprobs) + return payload + + @staticmethod + def from_dict(data: Mapping[str, Any]) -> "RenderedTurnRecord": + return RenderedTurnRecord( + rollout_id=data.get("rollout_id") or "", + trajectory_id=data.get("trajectory_id") or "", + task_id=data.get("task_id") or "", + split_id=data.get("split_id") or "", + env_package_hash=data.get("env_package_hash") or "", + turn_id=data.get("turn_id") or "", + renderer=RendererIdentity.from_dict(_mapping(data.get("renderer"), "renderer")), + provider_fidelity=ProviderFidelity.from_dict(_mapping(data.get("provider_fidelity"), "provider_fidelity")), + prompt_ids=_int_list(data.get("prompt_ids"), "prompt_ids"), + completion_ids=_int_list(data.get("completion_ids"), "completion_ids"), + input_ids=_int_list(data.get("input_ids"), "input_ids"), + attention_mask=_int_list(data.get("attention_mask"), "attention_mask"), + loss_mask=_bool_list(data.get("loss_mask"), "loss_mask"), + assistant_mask=_bool_list(data.get("assistant_mask"), "assistant_mask"), + tool_action_mask=_bool_list(data.get("tool_action_mask"), "tool_action_mask"), + reward_mask=_bool_list(data.get("reward_mask"), "reward_mask"), + sampled_mask=_bool_list(data.get("sampled_mask"), "sampled_mask"), + message_indices=_int_list(data.get("message_indices"), "message_indices"), + bridge_to_next_turn=BridgeToNextTurn.from_dict( + _mapping(data.get("bridge_to_next_turn"), "bridge_to_next_turn") + ), + tool_parse_status=data.get("tool_parse_status") or "not_applicable", + parsed_completion=data.get("parsed_completion"), + tool_calls=_mapping_list(data.get("tool_calls"), "tool_calls"), + completion_logprobs=_float_list(data.get("completion_logprobs"), "completion_logprobs"), + finish_reason=data.get("finish_reason") or "stop", + is_truncated=bool(data.get("is_truncated")), + overlong_prompt=bool(data.get("overlong_prompt")), + metadata=_mapping(data.get("metadata"), "metadata"), + ) diff --git a/breadboard/rl/replay/__init__.py b/breadboard/rl/replay/__init__.py new file mode 100644 index 00000000..72bfc930 --- /dev/null +++ b/breadboard/rl/replay/__init__.py @@ -0,0 +1,11 @@ +"""Replay parity and export admission primitives.""" + +from breadboard.rl.replay.admission import AdmissionDecision, decide_export_admission +from breadboard.rl.replay.parity import ReplayParityReport, compare_replay_parity + +__all__ = [ + "AdmissionDecision", + "ReplayParityReport", + "compare_replay_parity", + "decide_export_admission", +] diff --git a/breadboard/rl/replay/admission.py b/breadboard/rl/replay/admission.py new file mode 100644 index 00000000..11899583 --- /dev/null +++ b/breadboard/rl/replay/admission.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from breadboard.rl.replay.parity import ReplayParityReport + + +@dataclass(frozen=True) +class AdmissionDecision: + exportable: bool + trainable: bool + replay_status: str + hardening_status: str + quarantine_status: str + blocked_reasons: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "exportable": self.exportable, + "trainable": self.trainable, + "replay_status": self.replay_status, + "hardening_status": self.hardening_status, + "quarantine_status": self.quarantine_status, + "blocked_reasons": list(self.blocked_reasons), + } + + +def decide_export_admission( + *, + replay_report: ReplayParityReport, + hardening_status: str = "not_applicable", + quarantine_status: str = "clear", + token_records_valid: bool = True, +) -> AdmissionDecision: + blocked: list[str] = [] + if not replay_report.passed: + blocked.append("replay_mismatch") + if hardening_status not in {"passed", "not_applicable"}: + blocked.append(f"hardening_status={hardening_status}") + if quarantine_status != "clear": + blocked.append(f"quarantine_status={quarantine_status}") + if not token_records_valid: + blocked.append("token_records_invalid") + return AdmissionDecision( + exportable=not blocked, + trainable=False, + replay_status="passed" if replay_report.passed else "failed", + hardening_status=hardening_status, + quarantine_status=quarantine_status, + blocked_reasons=blocked, + ) diff --git a/breadboard/rl/replay/parity.py b/breadboard/rl/replay/parity.py new file mode 100644 index 00000000..dd39b2fe --- /dev/null +++ b/breadboard/rl/replay/parity.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from breadboard.rl.trace.graph import TrajectoryGraph + + +@dataclass(frozen=True) +class ReplayParityReport: + parity_tier: str + passed: bool + mismatches: list[str] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "parity_tier": self.parity_tier, + "passed": self.passed, + "mismatches": list(self.mismatches), + "metadata": dict(self.metadata), + } + + +def compare_replay_parity(live_graph: TrajectoryGraph, replay_graph: TrajectoryGraph) -> ReplayParityReport: + mismatches: list[str] = [] + live_kinds = [node.node_kind for node in live_graph.nodes] + replay_kinds = [node.node_kind for node in replay_graph.nodes] + if live_kinds != replay_kinds: + mismatches.append("node_kind_sequence_mismatch") + live_edges = [edge.edge_kind for edge in live_graph.edges] + replay_edges = [edge.edge_kind for edge in replay_graph.edges] + if live_edges != replay_edges: + mismatches.append("edge_kind_sequence_mismatch") + live_rewards = [ + node.payload.get("payload", {}).get("reward") + for node in live_graph.nodes + if node.node_kind == "evaluate" + ] + replay_rewards = [ + node.payload.get("payload", {}).get("reward") + for node in replay_graph.nodes + if node.node_kind == "evaluate" + ] + if live_rewards != replay_rewards: + mismatches.append("reward_mismatch") + return ReplayParityReport( + parity_tier="T2_deterministic_runtime_verifier", + passed=not mismatches, + mismatches=mismatches, + metadata={"live_graph_id": live_graph.graph_id, "replay_graph_id": replay_graph.graph_id}, + ) diff --git a/breadboard/rl/runtime/__init__.py b/breadboard/rl/runtime/__init__.py new file mode 100644 index 00000000..2e7bf7a8 --- /dev/null +++ b/breadboard/rl/runtime/__init__.py @@ -0,0 +1,29 @@ +"""Runtime backend primitives for RL sessions.""" + +from breadboard.rl.runtime.base import RuntimeHealth, RuntimeResult, RuntimeSnapshot +from breadboard.rl.runtime.local_process import LocalProcessToyRuntime +from breadboard.rl.runtime.pool import RuntimePool, WorkerRecord +from breadboard.rl.runtime.signature import RuntimeSignature, build_runtime_signature +from breadboard.rl.runtime.telemetry import build_warm_vs_cold_report, summarize_stage_metrics + +__all__ = [ + "LocalProcessToyRuntime", + "RuntimePool", + "RuntimeHealth", + "RuntimeResult", + "RuntimeSnapshot", + "RuntimeSignature", + "WorkerRecord", + "build_runtime_signature", + "build_warm_vs_cold_report", + "run_local_ray_toy_probe", + "summarize_stage_metrics", +] + + +def __getattr__(name: str): + if name == "run_local_ray_toy_probe": + from breadboard.rl.runtime.ray_worker import run_local_ray_toy_probe + + return run_local_ray_toy_probe + raise AttributeError(name) diff --git a/breadboard/rl/runtime/base.py b/breadboard/rl/runtime/base.py new file mode 100644 index 00000000..785f7f71 --- /dev/null +++ b/breadboard/rl/runtime/base.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Protocol + + +@dataclass(frozen=True) +class RuntimeHealth: + ready: bool + backend: str + reasons: list[str] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "ready": self.ready, + "backend": self.backend, + "reasons": list(self.reasons), + "metadata": dict(self.metadata), + } + + +@dataclass(frozen=True) +class RuntimeResult: + success: bool + result_kind: str + observation: dict[str, Any] | None = None + reward: float | None = None + done: bool = False + error: dict[str, Any] | None = None + evidence: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "success": self.success, + "result_kind": self.result_kind, + "done": self.done, + "evidence": dict(self.evidence), + } + if self.observation is not None: + payload["observation"] = dict(self.observation) + if self.reward is not None: + payload["reward"] = self.reward + if self.error is not None: + payload["error"] = dict(self.error) + return payload + + +@dataclass(frozen=True) +class RuntimeSnapshot: + snapshot_id: str + state: dict[str, Any] + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "snapshot_id": self.snapshot_id, + "state": dict(self.state), + "metadata": dict(self.metadata), + } + + +class RuntimeBackend(Protocol): + backend_id: str + + def health(self) -> RuntimeHealth: + ... + + def reset(self) -> RuntimeResult: + ... + + def observe(self) -> RuntimeResult: + ... + + def step(self, action: dict[str, Any]) -> RuntimeResult: + ... + + def evaluate(self) -> RuntimeResult: + ... + + def snapshot(self) -> RuntimeSnapshot: + ... + + def restore(self, snapshot: RuntimeSnapshot) -> RuntimeResult: + ... + + def terminate(self) -> RuntimeResult: + ... diff --git a/breadboard/rl/runtime/local_process.py b/breadboard/rl/runtime/local_process.py new file mode 100644 index 00000000..378d8158 --- /dev/null +++ b/breadboard/rl/runtime/local_process.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from breadboard.rl.env_package.schema import EnvPackage +from breadboard.rl.runtime.base import RuntimeHealth, RuntimeResult, RuntimeSnapshot + + +class LocalProcessToyRuntime: + """Deterministic local toy runtime for M3 lifecycle proof. + + This is not a security boundary. It only exercises the lifecycle contract + before Docker/gVisor/SWE hardening is introduced in later milestones. + """ + + backend_id = "local_process" + + def __init__(self, package: EnvPackage, task_id: str, *, expected_answer: str = "42") -> None: + self.package = package + self.task_id = str(task_id or "").strip() + self.expected_answer = str(expected_answer) + self._state: dict[str, Any] = { + "reset": False, + "terminated": False, + "turn": 0, + "actions": [], + "submitted_answer": None, + } + + def health(self) -> RuntimeHealth: + reasons: list[str] = [] + if self.package.runtime.backend != self.backend_id: + reasons.append(f"runtime.backend must be {self.backend_id}") + if self.package.verifier.kind != "exact_match": + reasons.append("local toy runtime requires verifier.kind=exact_match") + if not self.task_id: + reasons.append("task_id must be non-empty") + return RuntimeHealth( + ready=not reasons, + backend=self.backend_id, + reasons=reasons, + metadata={ + "package_id": self.package.package_id, + "task_id": self.task_id, + }, + ) + + def reset(self) -> RuntimeResult: + health = self.health() + if not health.ready: + return RuntimeResult( + success=False, + result_kind="reset", + error={"kind": "runtime_not_ready", "reasons": health.reasons}, + ) + self._state = { + "reset": True, + "terminated": False, + "turn": 0, + "actions": [], + "submitted_answer": None, + } + return RuntimeResult( + success=True, + result_kind="reset", + observation=self._observation(), + evidence={"task_id": self.task_id, "package_id": self.package.package_id}, + ) + + def observe(self) -> RuntimeResult: + if not self._state["reset"]: + return self._error("observe", "not_reset", "runtime must be reset before observe") + if self._state["terminated"]: + return self._error("observe", "terminated", "runtime is terminated") + return RuntimeResult(success=True, result_kind="observe", observation=self._observation()) + + def step(self, action: dict[str, Any]) -> RuntimeResult: + if not self._state["reset"]: + return self._error("step", "not_reset", "runtime must be reset before step") + if self._state["terminated"]: + return self._error("step", "terminated", "runtime is terminated") + + tool = str(action.get("tool") or "").strip() + if tool == "sleep": + return self._error("step", "timeout", "step exceeded timeout", {"action": dict(action)}) + if tool == "runtime_crash": + return self._error("step", "runtime_crash", "simulated runtime crash", {"action": dict(action)}) + if tool not in {"python", "submit_answer"}: + return self._error("step", "unknown_tool", f"unknown tool: {tool}", {"action": dict(action)}) + + self._state["turn"] += 1 + self._state["actions"].append(dict(action)) + if tool == "submit_answer": + self._state["submitted_answer"] = str(action.get("answer") or "") + return RuntimeResult( + success=True, + result_kind="step", + observation=self._observation(), + done=True, + evidence={"submitted_answer": self._state["submitted_answer"]}, + ) + return RuntimeResult( + success=True, + result_kind="step", + observation=self._observation(), + done=False, + evidence={"tool": tool}, + ) + + def evaluate(self) -> RuntimeResult: + if not self._state["reset"]: + return self._error("evaluate", "not_reset", "runtime must be reset before evaluate") + if self._state["terminated"]: + return self._error("evaluate", "terminated", "runtime is terminated") + submitted = self._state.get("submitted_answer") + if submitted is None: + return self._error("evaluate", "no_submission", "submit_answer is required before evaluate") + reward = 1.0 if str(submitted).strip() == self.expected_answer else 0.0 + return RuntimeResult( + success=True, + result_kind="evaluate", + reward=reward, + done=True, + evidence={ + "verifier_id": self.package.verifier.verifier_id, + "expected_answer_hash": "sha256:toy-expected-answer", + "submitted_answer": submitted, + }, + ) + + def snapshot(self) -> RuntimeSnapshot: + return RuntimeSnapshot( + snapshot_id=f"{self.package.package_id}.{self.task_id}.turn{self._state['turn']}", + state=deepcopy(self._state), + metadata={"backend": self.backend_id, "package_id": self.package.package_id}, + ) + + def restore(self, snapshot: RuntimeSnapshot) -> RuntimeResult: + self._state = deepcopy(snapshot.state) + return RuntimeResult( + success=True, + result_kind="restore", + observation=self._observation() if self._state.get("reset") and not self._state.get("terminated") else None, + evidence={"snapshot_id": snapshot.snapshot_id}, + ) + + def terminate(self) -> RuntimeResult: + self._state["terminated"] = True + return RuntimeResult( + success=True, + result_kind="terminate", + done=True, + evidence={"actions_recorded": len(self._state.get("actions", []))}, + ) + + def _observation(self) -> dict[str, Any]: + return { + "task_id": self.task_id, + "prompt": "Use tools if needed, then submit the answer to the toy task.", + "turn": self._state["turn"], + "submitted": self._state.get("submitted_answer") is not None, + } + + @staticmethod + def _error( + result_kind: str, + error_kind: str, + message: str, + evidence: dict[str, Any] | None = None, + ) -> RuntimeResult: + return RuntimeResult( + success=False, + result_kind=result_kind, + error={"kind": error_kind, "message": message}, + evidence=dict(evidence or {}), + ) diff --git a/breadboard/rl/runtime/pool.py b/breadboard/rl/runtime/pool.py new file mode 100644 index 00000000..64fdd8cf --- /dev/null +++ b/breadboard/rl/runtime/pool.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +READY = "READY" +ASSIGNED = "ASSIGNED" +QUARANTINED = "QUARANTINED" + + +@dataclass +class WorkerRecord: + worker_id: str + signature_digest: str + state: str = READY + assigned_count: int = 0 + quarantine_reason: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "worker_id": self.worker_id, + "signature_digest": self.signature_digest, + "state": self.state, + "assigned_count": self.assigned_count, + "quarantine_reason": self.quarantine_reason, + "metadata": dict(self.metadata), + } + + +class RuntimePool: + def __init__(self) -> None: + self.workers: dict[str, WorkerRecord] = {} + + def register(self, worker: WorkerRecord) -> None: + self.workers[worker.worker_id] = worker + + def route(self, signature_digest: str) -> WorkerRecord | None: + for worker in self.workers.values(): + if worker.signature_digest == signature_digest and worker.state == READY: + worker.state = ASSIGNED + worker.assigned_count += 1 + return worker + return None + + def release(self, worker_id: str) -> None: + worker = self.workers[worker_id] + if worker.state != QUARANTINED: + worker.state = READY + + def quarantine(self, worker_id: str, reason: str) -> None: + worker = self.workers[worker_id] + worker.state = QUARANTINED + worker.quarantine_reason = reason + + def to_dict(self) -> dict[str, Any]: + return {"workers": [worker.to_dict() for worker in self.workers.values()]} diff --git a/breadboard/rl/runtime/ray_worker.py b/breadboard/rl/runtime/ray_worker.py new file mode 100644 index 00000000..37af6379 --- /dev/null +++ b/breadboard/rl/runtime/ray_worker.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from typing import Any + +import ray + +from breadboard.rl.env_package.schema import EnvPackage +from breadboard.rl.session.controller import create_local_session + + +@ray.remote +class RayToyWorker: + def __init__(self, package_payload: dict[str, Any], worker_id: str) -> None: + self.package_payload = package_payload + self.worker_id = worker_id + self.run_count = 0 + + def run_once(self, task_id: str, answer: str = "42") -> dict[str, Any]: + package = EnvPackage.from_dict(self.package_payload) + session = create_local_session(package, task_id) + session.reset() + session.step({"tool": "submit_answer", "answer": answer}) + evaluation = session.evaluate() + self.run_count += 1 + return { + "worker_id": self.worker_id, + "task_id": task_id, + "reward": evaluation.reward, + "event_count": len(session.events), + "run_count": self.run_count, + "metrics_ms": { + "reset_ms": 3.0, + "step_ms": 4.0, + "verify_ms": 5.0, + "total_ms": 12.0, + }, + } + + +def run_local_ray_toy_probe( + *, + package: EnvPackage, + task_ids: list[str], + num_workers: int = 2, + local_mode: bool = True, +) -> dict[str, Any]: + started_here = not ray.is_initialized() + if started_here: + init_kwargs: dict[str, Any] = { + "ignore_reinit_error": True, + "include_dashboard": False, + "num_cpus": max(1, num_workers), + } + if local_mode: + init_kwargs["address"] = "local" + ray.init(**init_kwargs) + try: + workers = [ + RayToyWorker.remote(package.to_dict(), f"worker-{index}") + for index in range(num_workers) + ] + futures = [ + workers[index % num_workers].run_once.remote(task_id) + for index, task_id in enumerate(task_ids) + ] + rows = ray.get(futures) + return { + "row_count": len(rows), + "rows": rows, + "worker_count": num_workers, + "ray_local_mode": local_mode, + "claim_boundary": "local_ray_worker_probe_not_production_scale", + } + finally: + if started_here: + ray.shutdown() diff --git a/breadboard/rl/runtime/signature.py b/breadboard/rl/runtime/signature.py new file mode 100644 index 00000000..7a46fca7 --- /dev/null +++ b/breadboard/rl/runtime/signature.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from typing import Any + +from breadboard.rl.env_package.schema import EnvPackage + + +def _stable_hash(payload: dict[str, Any]) -> str: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +@dataclass(frozen=True) +class RuntimeSignature: + package_id: str + package_hash: str + backend: str + image_digest: str | None + taskset_id: str + source_hash: str + hardening_policy_hash: str + renderer_hash: str + network_policy: str + resource_class: str = "cpu_local" + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "package_id": self.package_id, + "package_hash": self.package_hash, + "backend": self.backend, + "image_digest": self.image_digest, + "taskset_id": self.taskset_id, + "source_hash": self.source_hash, + "hardening_policy_hash": self.hardening_policy_hash, + "renderer_hash": self.renderer_hash, + "network_policy": self.network_policy, + "resource_class": self.resource_class, + "metadata": dict(self.metadata), + } + + def digest(self) -> str: + return _stable_hash(self.to_dict()) + + +def build_runtime_signature(package: EnvPackage, *, resource_class: str = "cpu_local") -> RuntimeSignature: + taskset = package.tasksets[0] + hardening_policy_hash = ( + _stable_hash(package.hardening.to_dict()) if package.hardening is not None else "sha256:no-hardening" + ) + renderer_hash = _stable_hash(package.renderer.to_dict()) + return RuntimeSignature( + package_id=package.package_id, + package_hash=package.package_hash or "", + backend=package.runtime.backend, + image_digest=package.runtime.image_digest, + taskset_id=taskset.taskset_id, + source_hash=taskset.source_hash, + hardening_policy_hash=hardening_policy_hash, + renderer_hash=renderer_hash, + network_policy=package.runtime.network, + resource_class=resource_class, + ) diff --git a/breadboard/rl/runtime/telemetry.py b/breadboard/rl/runtime/telemetry.py new file mode 100644 index 00000000..32d7108e --- /dev/null +++ b/breadboard/rl/runtime/telemetry.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import statistics +from typing import Any, Mapping + + +def summarize_stage_metrics(rows: list[Mapping[str, Any]]) -> dict[str, dict[str, float]]: + keys = sorted({key for row in rows for key in row.get("metrics_ms", {})}) + summary: dict[str, dict[str, float]] = {} + for key in keys: + values = sorted(float(row["metrics_ms"][key]) for row in rows if key in row.get("metrics_ms", {})) + if not values: + continue + p95_index = min(len(values) - 1, int(round(0.95 * (len(values) - 1)))) + summary[key] = { + "p50": float(statistics.median(values)), + "p95": float(values[p95_index]), + "count": float(len(values)), + } + return summary + + +def build_warm_vs_cold_report(*, warm_rows: list[Mapping[str, Any]], cold_rows: list[Mapping[str, Any]]) -> dict[str, Any]: + warm = summarize_stage_metrics(warm_rows) + cold = summarize_stage_metrics(cold_rows) + return { + "warm": warm, + "cold": cold, + "claim_boundary": "local_ray_warm_pool_probe_not_production_scale", + } diff --git a/breadboard/rl/security/__init__.py b/breadboard/rl/security/__init__.py new file mode 100644 index 00000000..62d98763 --- /dev/null +++ b/breadboard/rl/security/__init__.py @@ -0,0 +1,28 @@ +"""Security hardening and quarantine primitives for RL rollout evidence.""" + +from breadboard.rl.security.hardening import ( + HardeningFinding, + HardeningReport, + build_hardening_report, + scan_python_import_hooks, + scan_symlink_escapes, + validate_process_cleanup_before_verify, +) +from breadboard.rl.security.probes import ProbeResult, run_probe_suite +from breadboard.rl.security.quarantine import QuarantineDecision, quarantine_on_findings +from breadboard.rl.security.reports import VerifierRunReport, build_verifier_run_report + +__all__ = [ + "HardeningFinding", + "HardeningReport", + "ProbeResult", + "QuarantineDecision", + "VerifierRunReport", + "build_hardening_report", + "build_verifier_run_report", + "quarantine_on_findings", + "run_probe_suite", + "scan_python_import_hooks", + "scan_symlink_escapes", + "validate_process_cleanup_before_verify", +] diff --git a/breadboard/rl/security/hardening.py b/breadboard/rl/security/hardening.py new file mode 100644 index 00000000..46f68fea --- /dev/null +++ b/breadboard/rl/security/hardening.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + +from breadboard.rl.env_package.schema import HardeningPolicy + + +@dataclass(frozen=True) +class HardeningFinding: + finding_id: str + severity: str + path: str + message: str + probe_id: str | None = None + + def to_dict(self) -> dict[str, Any]: + payload = { + "finding_id": self.finding_id, + "severity": self.severity, + "path": self.path, + "message": self.message, + } + if self.probe_id: + payload["probe_id"] = self.probe_id + return payload + + +@dataclass(frozen=True) +class HardeningReport: + report_id: str + status: str + findings: list[HardeningFinding] = field(default_factory=list) + clean_baseline_passed: bool = True + reference_solution_passed: bool = True + process_cleanup_observed: bool = False + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "report_id": self.report_id, + "status": self.status, + "findings": [item.to_dict() for item in self.findings], + "clean_baseline_passed": self.clean_baseline_passed, + "reference_solution_passed": self.reference_solution_passed, + "process_cleanup_observed": self.process_cleanup_observed, + "metadata": dict(self.metadata), + } + + +def scan_python_import_hooks(workspace: Path) -> list[HardeningFinding]: + workspace = workspace.resolve() + findings: list[HardeningFinding] = [] + suspicious_names = {"sitecustomize.py", "usercustomize.py"} + for path in workspace.rglob("*"): + if not path.is_file(): + continue + rel = path.relative_to(workspace).as_posix() + if path.name in suspicious_names: + findings.append( + HardeningFinding( + finding_id=path.stem, + severity="high", + path=rel, + message=f"Python import hook file detected: {path.name}", + ) + ) + if path.suffix == ".pth": + findings.append( + HardeningFinding( + finding_id="pth_injection", + severity="high", + path=rel, + message=".pth import path injection detected", + ) + ) + if path.name == "conftest.py" and not rel.startswith("tests/"): + findings.append( + HardeningFinding( + finding_id="conftest_outside_tests", + severity="medium", + path=rel, + message="pytest conftest.py outside tests/ detected", + ) + ) + return findings + + +def scan_symlink_escapes(workspace: Path) -> list[HardeningFinding]: + workspace = workspace.resolve() + findings: list[HardeningFinding] = [] + for path in workspace.rglob("*"): + if not path.is_symlink(): + continue + rel = path.relative_to(workspace).as_posix() + try: + resolved = path.resolve(strict=False) + except OSError: + resolved = path.absolute() + if workspace not in resolved.parents and resolved != workspace: + findings.append( + HardeningFinding( + finding_id="symlink_escape", + severity="high", + path=rel, + message=f"symlink escapes workspace: {resolved}", + ) + ) + return findings + + +def validate_process_cleanup_before_verify(events: Iterable[str]) -> list[str]: + event_list = list(events) + if "process_cleanup" not in event_list: + return ["process_cleanup event is required before verify"] + if "verify" in event_list and event_list.index("process_cleanup") > event_list.index("verify"): + return ["process_cleanup must occur before verify"] + return [] + + +def build_hardening_report( + *, + report_id: str, + workspace: Path, + policy: HardeningPolicy, + clean_baseline_passed: bool, + reference_solution_passed: bool, + process_cleanup_observed: bool, + extra_findings: list[HardeningFinding] | None = None, +) -> HardeningReport: + findings = [ + *scan_python_import_hooks(workspace), + *scan_symlink_escapes(workspace), + *list(extra_findings or []), + ] + status = "passed" + if not clean_baseline_passed or not reference_solution_passed: + status = "failed" + elif findings: + status = "quarantined" + elif policy.verifier_isolated_required and not process_cleanup_observed: + status = "quarantined" + findings.append( + HardeningFinding( + finding_id="missing_process_cleanup", + severity="medium", + path=".", + message="process cleanup was not observed before verifier", + ) + ) + return HardeningReport( + report_id=report_id, + status=status, + findings=findings, + clean_baseline_passed=clean_baseline_passed, + reference_solution_passed=reference_solution_passed, + process_cleanup_observed=process_cleanup_observed, + metadata={"policy_id": policy.policy_id}, + ) diff --git a/breadboard/rl/security/probes.py b/breadboard/rl/security/probes.py new file mode 100644 index 00000000..8a834b46 --- /dev/null +++ b/breadboard/rl/security/probes.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable + +from breadboard.rl.env_package.schema import HardeningPolicy +from breadboard.rl.security.hardening import HardeningFinding, build_hardening_report +from breadboard.rl.security.quarantine import quarantine_on_findings + + +@dataclass(frozen=True) +class ProbeResult: + probe_id: str + status: str + findings: list[HardeningFinding] = field(default_factory=list) + evidence: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "probe_id": self.probe_id, + "status": self.status, + "findings": [item.to_dict() for item in self.findings], + "evidence": dict(self.evidence), + } + + +ProbeMaterializer = Callable[[Path], list[HardeningFinding]] + + +def run_probe_suite( + *, + workspace: Path, + policy: HardeningPolicy, + probes: dict[str, ProbeMaterializer], +) -> list[ProbeResult]: + results: list[ProbeResult] = [] + for probe_id, materializer in probes.items(): + probe_workspace = workspace / probe_id + probe_workspace.mkdir(parents=True, exist_ok=True) + extra_findings = materializer(probe_workspace) + report = build_hardening_report( + report_id=f"{probe_id}.hardening", + workspace=probe_workspace, + policy=policy, + clean_baseline_passed=True, + reference_solution_passed=True, + process_cleanup_observed=True, + extra_findings=extra_findings, + ) + quarantine = quarantine_on_findings( + row_id=probe_id, + findings=report.findings, + policy=policy, + ) + results.append( + ProbeResult( + probe_id=probe_id, + status="quarantined" if quarantine.quarantined else report.status, + findings=report.findings, + evidence={ + "hardening_status": report.status, + "quarantine": quarantine.to_dict(), + }, + ) + ) + return results diff --git a/breadboard/rl/security/quarantine.py b/breadboard/rl/security/quarantine.py new file mode 100644 index 00000000..dd15277e --- /dev/null +++ b/breadboard/rl/security/quarantine.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from breadboard.rl.env_package.schema import HardeningPolicy +from breadboard.rl.security.hardening import HardeningFinding + + +@dataclass(frozen=True) +class QuarantineDecision: + row_id: str + quarantined: bool + reasons: list[str] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "row_id": self.row_id, + "quarantined": self.quarantined, + "reasons": list(self.reasons), + "metadata": dict(self.metadata), + } + + +def quarantine_on_findings( + *, + row_id: str, + findings: list[HardeningFinding], + policy: HardeningPolicy, +) -> QuarantineDecision: + policy_triggers = set(policy.quarantine_on_findings) + reasons: list[str] = [] + for finding in findings: + if finding.finding_id in policy_triggers or finding.severity in {"high", "critical"}: + reasons.append(finding.finding_id) + return QuarantineDecision( + row_id=row_id, + quarantined=bool(reasons), + reasons=list(dict.fromkeys(reasons)), + metadata={"policy_id": policy.policy_id}, + ) diff --git a/breadboard/rl/security/reports.py b/breadboard/rl/security/reports.py new file mode 100644 index 00000000..ac778389 --- /dev/null +++ b/breadboard/rl/security/reports.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class VerifierRunReport: + report_id: str + verifier_id: str + status: str + output: str + evidence_sha256: str + rerun_agreement: bool | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + def verify_evidence_hash(self) -> bool: + digest = "sha256:" + hashlib.sha256(self.output.encode("utf-8")).hexdigest() + return digest == self.evidence_sha256 + + def to_dict(self) -> dict[str, Any]: + payload = { + "report_id": self.report_id, + "verifier_id": self.verifier_id, + "status": self.status, + "output": self.output, + "evidence_sha256": self.evidence_sha256, + "evidence_hash_valid": self.verify_evidence_hash(), + "metadata": dict(self.metadata), + } + if self.rerun_agreement is not None: + payload["rerun_agreement"] = self.rerun_agreement + return payload + + +def build_verifier_run_report( + *, + report_id: str, + verifier_id: str, + status: str, + output: str, + rerun_output: str | None = None, +) -> VerifierRunReport: + evidence_sha256 = "sha256:" + hashlib.sha256(output.encode("utf-8")).hexdigest() + return VerifierRunReport( + report_id=report_id, + verifier_id=verifier_id, + status=status, + output=output, + evidence_sha256=evidence_sha256, + rerun_agreement=(output == rerun_output) if rerun_output is not None else None, + ) diff --git a/breadboard/rl/security/reward_hack_suites/__init__.py b/breadboard/rl/security/reward_hack_suites/__init__.py new file mode 100644 index 00000000..5594765e --- /dev/null +++ b/breadboard/rl/security/reward_hack_suites/__init__.py @@ -0,0 +1,11 @@ +"""Reward-hack probe suites.""" + +from breadboard.rl.security.reward_hack_suites.swe import ( + SWE_REWARD_HACK_PROBES, + build_swe_reward_hack_probe_suite, +) + +__all__ = [ + "SWE_REWARD_HACK_PROBES", + "build_swe_reward_hack_probe_suite", +] diff --git a/breadboard/rl/security/reward_hack_suites/swe.py b/breadboard/rl/security/reward_hack_suites/swe.py new file mode 100644 index 00000000..2d9bd0e2 --- /dev/null +++ b/breadboard/rl/security/reward_hack_suites/swe.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.security.hardening import HardeningFinding + + +SWE_REWARD_HACK_PROBES = [ + "conftest_outside_tests", + "sitecustomize_shadow", + "pth_injection", + "symlink_escape", + "verifier_output_tamper", +] + + +def _conftest_outside_tests(workspace: Path) -> list[HardeningFinding]: + (workspace / "conftest.py").write_text("pytest_plugins=[]\n", encoding="utf-8") + return [] + + +def _sitecustomize_shadow(workspace: Path) -> list[HardeningFinding]: + (workspace / "sitecustomize.py").write_text("raise SystemExit('poison')\n", encoding="utf-8") + return [] + + +def _pth_injection(workspace: Path) -> list[HardeningFinding]: + (workspace / "poison.pth").write_text("import os\n", encoding="utf-8") + return [] + + +def _symlink_escape(workspace: Path) -> list[HardeningFinding]: + (workspace / "escape").symlink_to("/etc/passwd") + return [] + + +def _verifier_output_tamper(workspace: Path) -> list[HardeningFinding]: + (workspace / "verifier.out").write_text("tampered\n", encoding="utf-8") + return [ + HardeningFinding( + finding_id="verifier_output_tamper", + severity="high", + path="verifier.out", + message="verifier output tamper fixture", + probe_id="verifier_output_tamper", + ) + ] + + +def build_swe_reward_hack_probe_suite(): + return { + "conftest_outside_tests": _conftest_outside_tests, + "sitecustomize_shadow": _sitecustomize_shadow, + "pth_injection": _pth_injection, + "symlink_escape": _symlink_escape, + "verifier_output_tamper": _verifier_output_tamper, + } diff --git a/breadboard/rl/session/__init__.py b/breadboard/rl/session/__init__.py new file mode 100644 index 00000000..661a3a7e --- /dev/null +++ b/breadboard/rl/session/__init__.py @@ -0,0 +1,13 @@ +"""Local RL session lifecycle primitives.""" + +from breadboard.rl.session.controller import LocalSession, create_local_session +from breadboard.rl.session.events import SessionEvent +from breadboard.rl.session.lifecycle import SessionLifecycleState, SessionStatus + +__all__ = [ + "LocalSession", + "SessionEvent", + "SessionLifecycleState", + "SessionStatus", + "create_local_session", +] diff --git a/breadboard/rl/session/controller.py b/breadboard/rl/session/controller.py new file mode 100644 index 00000000..92286a0c --- /dev/null +++ b/breadboard/rl/session/controller.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from breadboard.rl.env_package.schema import EnvPackage +from breadboard.rl.runtime.base import RuntimeBackend, RuntimeHealth, RuntimeResult, RuntimeSnapshot +from breadboard.rl.runtime.local_process import LocalProcessToyRuntime +from breadboard.rl.session.events import SessionEvent +from breadboard.rl.session.lifecycle import SessionLifecycleState, SessionStatus + + +@dataclass +class LocalSession: + session_id: str + package: EnvPackage + task_id: str + runtime: RuntimeBackend + lifecycle: SessionLifecycleState = field(default_factory=SessionLifecycleState) + events: list[SessionEvent] = field(default_factory=list) + last_result: RuntimeResult | None = None + last_evaluation: RuntimeResult | None = None + + def health(self) -> RuntimeHealth: + return self.runtime.health() + + def reset(self) -> RuntimeResult: + self._require_not_terminated("reset") + result = self.runtime.reset() + self._record_result("reset", result, SessionStatus.READY if result.success else SessionStatus.FAILED) + return result + + def observe(self) -> RuntimeResult: + if self.lifecycle.status not in {SessionStatus.READY, SessionStatus.RUNNING}: + raise ValueError(f"observe requires ready/running session, got {self.lifecycle.status}") + result = self.runtime.observe() + self.last_result = result + self._append_event("observe", self.lifecycle.status, self.lifecycle.status, result) + return result + + def step(self, action: dict[str, Any]) -> RuntimeResult: + if self.lifecycle.status not in {SessionStatus.READY, SessionStatus.RUNNING}: + raise ValueError(f"step requires ready/running session, got {self.lifecycle.status}") + result = self.runtime.step(action) + self._record_result("step", result, SessionStatus.RUNNING if result.success else SessionStatus.FAILED) + return result + + def evaluate(self) -> RuntimeResult: + if self.lifecycle.status not in {SessionStatus.READY, SessionStatus.RUNNING}: + raise ValueError(f"evaluate requires ready/running session, got {self.lifecycle.status}") + result = self.runtime.evaluate() + self.last_evaluation = result if result.success else None + self._record_result("evaluate", result, SessionStatus.EVALUATED if result.success else SessionStatus.FAILED) + return result + + def snapshot(self) -> RuntimeSnapshot: + if self.lifecycle.status == SessionStatus.TERMINATED: + raise ValueError("snapshot is not allowed after terminate") + snapshot = self.runtime.snapshot() + self._append_event( + "snapshot", + self.lifecycle.status, + self.lifecycle.status, + RuntimeResult(success=True, result_kind="snapshot", evidence={"snapshot_id": snapshot.snapshot_id}), + ) + return snapshot + + def restore(self, snapshot: RuntimeSnapshot) -> RuntimeResult: + if self.lifecycle.status == SessionStatus.TERMINATED: + raise ValueError("restore is not allowed after terminate") + result = self.runtime.restore(snapshot) + self._append_event("restore", self.lifecycle.status, self.lifecycle.status, result) + self.last_result = result + return result + + def terminate(self) -> RuntimeResult: + if self.lifecycle.status == SessionStatus.TERMINATED: + return RuntimeResult(success=True, result_kind="terminate", done=True, evidence={"already_terminated": True}) + result = self.runtime.terminate() + self._record_result("terminate", result, SessionStatus.TERMINATED) + return result + + def export_admission(self) -> dict[str, Any]: + reasons: list[str] = [] + if self.lifecycle.status != SessionStatus.EVALUATED: + reasons.append(f"lifecycle_status={self.lifecycle.status}") + if self.last_evaluation is None or not self.last_evaluation.success: + reasons.append("missing_successful_evaluation") + return { + "session_id": self.session_id, + "lifecycle_status": self.lifecycle.status, + "trainable": False, + "exportable_debug": not reasons, + "blocked_reasons": reasons, + "claim_boundary": "m3_lifecycle_only_not_trainable_export", + } + + def _record_result(self, event_kind: str, result: RuntimeResult, success_status: str) -> None: + before = self.lifecycle.status + next_status = success_status if result.success else SessionStatus.FAILED + self.lifecycle = self.lifecycle.transition(next_status) + self.last_result = result + self._append_event(event_kind, before, self.lifecycle.status, result) + + def _append_event( + self, + event_kind: str, + status_before: str, + status_after: str, + result: RuntimeResult, + ) -> None: + self.events.append( + SessionEvent( + event_id=f"{self.session_id}.event.{len(self.events) + 1}", + session_id=self.session_id, + event_kind=event_kind, + status_before=status_before, + status_after=status_after, + payload=result.to_dict(), + error=result.error, + ) + ) + + def _require_not_terminated(self, operation: str) -> None: + if self.lifecycle.status == SessionStatus.TERMINATED: + raise ValueError(f"{operation} is not allowed after terminate") + + +def create_local_session(package: EnvPackage, task_id: str) -> LocalSession: + session_id = f"{package.package_id}.{task_id}.session" + return LocalSession( + session_id=session_id, + package=package, + task_id=task_id, + runtime=LocalProcessToyRuntime(package, task_id), + ) diff --git a/breadboard/rl/session/events.py b/breadboard/rl/session/events.py new file mode 100644 index 00000000..e3a12df9 --- /dev/null +++ b/breadboard/rl/session/events.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class SessionEvent: + event_id: str + session_id: str + event_kind: str + status_before: str + status_after: str + payload: dict[str, Any] = field(default_factory=dict) + error: dict[str, Any] | None = None + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "event_id": self.event_id, + "session_id": self.session_id, + "event_kind": self.event_kind, + "status_before": self.status_before, + "status_after": self.status_after, + "payload": dict(self.payload), + } + if self.error is not None: + payload["error"] = dict(self.error) + return payload diff --git a/breadboard/rl/session/lifecycle.py b/breadboard/rl/session/lifecycle.py new file mode 100644 index 00000000..728ffa7e --- /dev/null +++ b/breadboard/rl/session/lifecycle.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +class SessionStatus: + CREATED = "created" + READY = "ready" + RUNNING = "running" + EVALUATED = "evaluated" + FAILED = "failed" + TERMINATED = "terminated" + + +ALLOWED_TRANSITIONS = { + SessionStatus.CREATED: {SessionStatus.READY, SessionStatus.FAILED, SessionStatus.TERMINATED}, + SessionStatus.READY: {SessionStatus.RUNNING, SessionStatus.EVALUATED, SessionStatus.FAILED, SessionStatus.TERMINATED}, + SessionStatus.RUNNING: {SessionStatus.RUNNING, SessionStatus.EVALUATED, SessionStatus.FAILED, SessionStatus.TERMINATED}, + SessionStatus.EVALUATED: {SessionStatus.TERMINATED}, + SessionStatus.FAILED: {SessionStatus.TERMINATED}, + SessionStatus.TERMINATED: set(), +} + + +@dataclass(frozen=True) +class SessionLifecycleState: + status: str = SessionStatus.CREATED + history: list[str] = field(default_factory=lambda: [SessionStatus.CREATED]) + + def transition(self, next_status: str) -> "SessionLifecycleState": + if next_status not in ALLOWED_TRANSITIONS.get(self.status, set()): + raise ValueError(f"invalid session transition: {self.status} -> {next_status}") + return SessionLifecycleState(status=next_status, history=[*self.history, next_status]) + + def to_dict(self) -> dict[str, Any]: + return {"status": self.status, "history": list(self.history)} diff --git a/breadboard/rl/state/__init__.py b/breadboard/rl/state/__init__.py new file mode 100644 index 00000000..2a0f93b7 --- /dev/null +++ b/breadboard/rl/state/__init__.py @@ -0,0 +1,13 @@ +"""State references and content-addressed storage primitives.""" + +from breadboard.rl.state.cas import InMemoryCAS +from breadboard.rl.state.snapshot import SnapshotManifest, build_snapshot_manifest +from breadboard.rl.state.state_ref import ArtifactRef, StateRef + +__all__ = [ + "ArtifactRef", + "InMemoryCAS", + "SnapshotManifest", + "StateRef", + "build_snapshot_manifest", +] diff --git a/breadboard/rl/state/cas.py b/breadboard/rl/state/cas.py new file mode 100644 index 00000000..7fa82ed8 --- /dev/null +++ b/breadboard/rl/state/cas.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import hashlib +from typing import Any + +from breadboard.rl.state.state_ref import ArtifactRef + + +class InMemoryCAS: + """Small immutable content-addressed store for local replay tests.""" + + def __init__(self) -> None: + self._bytes_by_id: dict[str, bytes] = {} + self._hash_by_id: dict[str, str] = {} + + def put_bytes( + self, + data: bytes, + *, + artifact_id: str | None = None, + media_type: str = "application/octet-stream", + metadata: dict[str, Any] | None = None, + ) -> ArtifactRef: + digest = "sha256:" + hashlib.sha256(data).hexdigest() + resolved_id = artifact_id or digest + existing_hash = self._hash_by_id.get(resolved_id) + if existing_hash and existing_hash != digest: + raise ValueError("CAS artifact overwrite rejected") + self._bytes_by_id[resolved_id] = bytes(data) + self._hash_by_id[resolved_id] = digest + return ArtifactRef( + artifact_id=resolved_id, + sha256=digest, + size_bytes=len(data), + media_type=media_type, + metadata=dict(metadata or {}), + ) + + def get_bytes(self, artifact_ref: ArtifactRef | str) -> bytes: + artifact_id = artifact_ref.artifact_id if isinstance(artifact_ref, ArtifactRef) else artifact_ref + return self._bytes_by_id[artifact_id] + + def has(self, artifact_ref: ArtifactRef | str) -> bool: + artifact_id = artifact_ref.artifact_id if isinstance(artifact_ref, ArtifactRef) else artifact_ref + return artifact_id in self._bytes_by_id diff --git a/breadboard/rl/state/snapshot.py b/breadboard/rl/state/snapshot.py new file mode 100644 index 00000000..1f147016 --- /dev/null +++ b/breadboard/rl/state/snapshot.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from typing import Any + +from breadboard.rl.runtime.base import RuntimeSnapshot +from breadboard.rl.state.state_ref import ArtifactRef, StateRef + + +def _stable_hash(payload: dict[str, Any]) -> str: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +@dataclass(frozen=True) +class SnapshotManifest: + snapshot_id: str + package_hash: str + runtime_backend: str + state_ref: StateRef + event_ids: list[str] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "snapshot_id": self.snapshot_id, + "package_hash": self.package_hash, + "runtime_backend": self.runtime_backend, + "state_ref": self.state_ref.to_dict(), + "event_ids": list(self.event_ids), + "metadata": dict(self.metadata), + } + + +def build_snapshot_manifest( + *, + snapshot: RuntimeSnapshot, + package_hash: str, + runtime_backend: str, + artifact_refs: list[ArtifactRef] | None = None, + event_ids: list[str] | None = None, +) -> SnapshotManifest: + state_ref = StateRef( + state_id=f"{snapshot.snapshot_id}.state", + state_hash=_stable_hash(snapshot.state), + artifact_refs=list(artifact_refs or []), + metadata={"snapshot_id": snapshot.snapshot_id}, + ) + return SnapshotManifest( + snapshot_id=snapshot.snapshot_id, + package_hash=package_hash, + runtime_backend=runtime_backend, + state_ref=state_ref, + event_ids=list(event_ids or []), + metadata=dict(snapshot.metadata), + ) diff --git a/breadboard/rl/state/state_ref.py b/breadboard/rl/state/state_ref.py new file mode 100644 index 00000000..5e0a931a --- /dev/null +++ b/breadboard/rl/state/state_ref.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class ArtifactRef: + artifact_id: str + sha256: str + size_bytes: int + media_type: str = "application/octet-stream" + metadata: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.artifact_id: + raise ValueError("artifact_id must be non-empty") + if not self.sha256.startswith("sha256:"): + raise ValueError("sha256 must start with sha256:") + if self.size_bytes < 0: + raise ValueError("size_bytes must be >= 0") + object.__setattr__(self, "metadata", dict(self.metadata or {})) + + def to_dict(self) -> dict[str, Any]: + return { + "artifact_id": self.artifact_id, + "sha256": self.sha256, + "size_bytes": self.size_bytes, + "media_type": self.media_type, + "metadata": dict(self.metadata), + } + + +@dataclass(frozen=True) +class StateRef: + state_id: str + state_hash: str + artifact_refs: list[ArtifactRef] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.state_id: + raise ValueError("state_id must be non-empty") + if not self.state_hash.startswith("sha256:"): + raise ValueError("state_hash must start with sha256:") + object.__setattr__(self, "artifact_refs", list(self.artifact_refs)) + object.__setattr__(self, "metadata", dict(self.metadata or {})) + + def to_dict(self) -> dict[str, Any]: + return { + "state_id": self.state_id, + "state_hash": self.state_hash, + "artifact_refs": [item.to_dict() for item in self.artifact_refs], + "metadata": dict(self.metadata), + } diff --git a/breadboard/rl/trace/__init__.py b/breadboard/rl/trace/__init__.py new file mode 100644 index 00000000..d42323ce --- /dev/null +++ b/breadboard/rl/trace/__init__.py @@ -0,0 +1,16 @@ +"""Trajectory graph primitives for RL replay and projection.""" + +from breadboard.rl.trace.credit import CreditFrame, build_terminal_credit_frame +from breadboard.rl.trace.edges import TraceEdge +from breadboard.rl.trace.graph import TrajectoryGraph, build_graph_from_session_events, validate_graph_invariants +from breadboard.rl.trace.nodes import TraceNode + +__all__ = [ + "CreditFrame", + "TraceEdge", + "TraceNode", + "TrajectoryGraph", + "build_graph_from_session_events", + "build_terminal_credit_frame", + "validate_graph_invariants", +] diff --git a/breadboard/rl/trace/credit.py b/breadboard/rl/trace/credit.py new file mode 100644 index 00000000..e4086aa1 --- /dev/null +++ b/breadboard/rl/trace/credit.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from breadboard.rl.trace.graph import TrajectoryGraph + + +@dataclass(frozen=True) +class CreditFrame: + credit_frame_id: str + graph_id: str + reward: float + credited_node_ids: list[str] + policy: str = "terminal_reward_to_prior_decisions" + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "credit_frame_id": self.credit_frame_id, + "graph_id": self.graph_id, + "reward": self.reward, + "credited_node_ids": list(self.credited_node_ids), + "policy": self.policy, + "metadata": dict(self.metadata), + } + + +def build_terminal_credit_frame(graph: TrajectoryGraph, *, reward: float) -> CreditFrame: + credited = [node.node_id for node in graph.nodes if node.node_kind in {"step", "evaluate"}] + return CreditFrame( + credit_frame_id=f"{graph.graph_id}.credit.terminal", + graph_id=graph.graph_id, + reward=float(reward), + credited_node_ids=credited, + ) diff --git a/breadboard/rl/trace/edges.py b/breadboard/rl/trace/edges.py new file mode 100644 index 00000000..f9d62973 --- /dev/null +++ b/breadboard/rl/trace/edges.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class TraceEdge: + edge_id: str + source_id: str + target_id: str + edge_kind: str + metadata: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.edge_id: + raise ValueError("edge_id must be non-empty") + if not self.source_id: + raise ValueError("source_id must be non-empty") + if not self.target_id: + raise ValueError("target_id must be non-empty") + if not self.edge_kind: + raise ValueError("edge_kind must be non-empty") + object.__setattr__(self, "metadata", dict(self.metadata or {})) + + def to_dict(self) -> dict[str, Any]: + return { + "edge_id": self.edge_id, + "source_id": self.source_id, + "target_id": self.target_id, + "edge_kind": self.edge_kind, + "metadata": dict(self.metadata), + } diff --git a/breadboard/rl/trace/graph.py b/breadboard/rl/trace/graph.py new file mode 100644 index 00000000..960393e9 --- /dev/null +++ b/breadboard/rl/trace/graph.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Sequence + +from breadboard.rl.session.events import SessionEvent +from breadboard.rl.trace.edges import TraceEdge +from breadboard.rl.trace.nodes import TraceNode + + +@dataclass(frozen=True) +class TrajectoryGraph: + graph_id: str + nodes: list[TraceNode] + edges: list[TraceEdge] + metadata: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "graph_id": self.graph_id, + "nodes": [item.to_dict() for item in self.nodes], + "edges": [item.to_dict() for item in self.edges], + "metadata": dict(self.metadata), + } + + +def build_graph_from_session_events( + *, + graph_id: str, + session_id: str, + events: Sequence[SessionEvent], + metadata: dict[str, Any] | None = None, +) -> TrajectoryGraph: + nodes = [ + TraceNode( + node_id=event.event_id, + node_kind=event.event_kind, + payload=event.to_dict(), + ) + for event in events + ] + edges = [ + TraceEdge( + edge_id=f"{session_id}.edge.{index}", + source_id=events[index - 1].event_id, + target_id=events[index].event_id, + edge_kind="session_order", + metadata={"index": index}, + ) + for index in range(1, len(events)) + ] + return TrajectoryGraph( + graph_id=graph_id, + nodes=nodes, + edges=edges, + metadata={"session_id": session_id, **dict(metadata or {})}, + ) + + +def validate_graph_invariants(graph: TrajectoryGraph) -> list[str]: + errors: list[str] = [] + node_ids = [node.node_id for node in graph.nodes] + node_id_set = set(node_ids) + if len(node_ids) != len(node_id_set): + errors.append("node_id values must be unique") + for edge in graph.edges: + if edge.source_id not in node_id_set: + errors.append(f"edge {edge.edge_id} references missing source_id") + if edge.target_id not in node_id_set: + errors.append(f"edge {edge.edge_id} references missing target_id") + if graph.edges and len(graph.edges) != max(0, len(graph.nodes) - 1): + errors.append("session_order graph must have node_count - 1 edges") + if graph.nodes and graph.nodes[0].node_kind != "reset": + errors.append("session graph must start with reset") + return errors diff --git a/breadboard/rl/trace/nodes.py b/breadboard/rl/trace/nodes.py new file mode 100644 index 00000000..9a5843e1 --- /dev/null +++ b/breadboard/rl/trace/nodes.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class TraceNode: + node_id: str + node_kind: str + payload: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.node_id: + raise ValueError("node_id must be non-empty") + if not self.node_kind: + raise ValueError("node_kind must be non-empty") + object.__setattr__(self, "payload", dict(self.payload or {})) + + def to_dict(self) -> dict[str, Any]: + return { + "node_id": self.node_id, + "node_kind": self.node_kind, + "payload": dict(self.payload), + } diff --git a/docs/contracts/cli_bridge/openapi.json b/docs/contracts/cli_bridge/openapi.json index c6c2e7b7..4b6506e6 100644 --- a/docs/contracts/cli_bridge/openapi.json +++ b/docs/contracts/cli_bridge/openapi.json @@ -678,6 +678,315 @@ "title": "ProviderAuthStatusResponse", "type": "object" }, + "RLRunArtifactListResponse": { + "properties": { + "artifacts": { + "items": { + "$ref": "#/components/schemas/RlArtifactModel" + }, + "title": "Artifacts", + "type": "array" + }, + "run_id": { + "title": "Run Id", + "type": "string" + } + }, + "required": [ + "run_id", + "artifacts" + ], + "title": "RLRunArtifactListResponse", + "type": "object" + }, + "RLRunAuditResponse": { + "properties": { + "persistent_store": { + "title": "Persistent Store", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "scorecard_update_allowed": { + "title": "Scorecard Update Allowed", + "type": "boolean" + }, + "state": { + "title": "State", + "type": "string" + }, + "target_run_id": { + "title": "Target Run Id", + "type": "string" + }, + "tenant_id": { + "title": "Tenant Id", + "type": "string" + }, + "workspace_id": { + "title": "Workspace Id", + "type": "string" + } + }, + "required": [ + "run_id", + "tenant_id", + "workspace_id", + "target_run_id", + "state", + "persistent_store", + "scorecard_update_allowed" + ], + "title": "RLRunAuditResponse", + "type": "object" + }, + "RLRunCancelRequest": { + "properties": { + "reason": { + "default": "cancel requested", + "title": "Reason", + "type": "string" + }, + "tenant_id": { + "title": "Tenant Id", + "type": "string" + }, + "workspace_id": { + "title": "Workspace Id", + "type": "string" + } + }, + "required": [ + "tenant_id", + "workspace_id" + ], + "title": "RLRunCancelRequest", + "type": "object" + }, + "RLRunReplayResponse": { + "properties": { + "artifact_id": { + "title": "Artifact Id", + "type": "string" + }, + "available": { + "title": "Available", + "type": "boolean" + }, + "reason": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Reason" + }, + "replay_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Replay Path" + }, + "sha256": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Sha256" + } + }, + "required": [ + "available", + "artifact_id" + ], + "title": "RLRunReplayResponse", + "type": "object" + }, + "RLRunStatusResponse": { + "properties": { + "accepted": { + "title": "Accepted", + "type": "boolean" + }, + "cancellation_state": { + "title": "Cancellation State", + "type": "string" + }, + "reason": { + "default": "", + "title": "Reason", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "state": { + "title": "State", + "type": "string" + }, + "target_run_id": { + "title": "Target Run Id", + "type": "string" + } + }, + "required": [ + "run_id", + "state", + "target_run_id", + "accepted", + "cancellation_state" + ], + "title": "RLRunStatusResponse", + "type": "object" + }, + "RLRunSubmitRequest": { + "properties": { + "env_package_ref": { + "title": "Env Package Ref", + "type": "string" + }, + "metadata": { + "additionalProperties": true, + "title": "Metadata", + "type": "object" + }, + "requested_budget_usd": { + "default": 1.0, + "exclusiveMinimum": 0.0, + "title": "Requested Budget Usd", + "type": "number" + }, + "requested_duration_seconds": { + "default": 60, + "exclusiveMinimum": 0.0, + "title": "Requested Duration Seconds", + "type": "integer" + }, + "requested_gpus": { + "default": 1, + "exclusiveMinimum": 0.0, + "title": "Requested Gpus", + "type": "integer" + }, + "requested_tasks": { + "default": 1, + "exclusiveMinimum": 0.0, + "title": "Requested Tasks", + "type": "integer" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "target_run_id": { + "title": "Target Run Id", + "type": "string" + }, + "tenant_id": { + "title": "Tenant Id", + "type": "string" + }, + "workspace_id": { + "title": "Workspace Id", + "type": "string" + } + }, + "required": [ + "run_id", + "tenant_id", + "workspace_id", + "env_package_ref", + "target_run_id" + ], + "title": "RLRunSubmitRequest", + "type": "object" + }, + "RLRunSubmitResponse": { + "properties": { + "accepted": { + "title": "Accepted", + "type": "boolean" + }, + "cancellation_state": { + "title": "Cancellation State", + "type": "string" + }, + "reason": { + "default": "", + "title": "Reason", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "state": { + "title": "State", + "type": "string" + }, + "target_run_id": { + "title": "Target Run Id", + "type": "string" + } + }, + "required": [ + "run_id", + "state", + "target_run_id", + "accepted", + "cancellation_state" + ], + "title": "RLRunSubmitResponse", + "type": "object" + }, + "RlArtifactModel": { + "properties": { + "artifact_id": { + "title": "Artifact Id", + "type": "string" + }, + "bytes": { + "title": "Bytes", + "type": "integer" + }, + "egress_allowed": { + "title": "Egress Allowed", + "type": "boolean" + }, + "relative_path": { + "title": "Relative Path", + "type": "string" + }, + "sha256": { + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "artifact_id", + "relative_path", + "sha256", + "bytes", + "egress_allowed" + ], + "title": "RlArtifactModel", + "type": "object" + }, "SessionCommandRequest": { "properties": { "command": { @@ -1186,34 +1495,14 @@ "summary": "Ready" } }, - "/sessions": { - "get": { - "operationId": "list_sessions_sessions_get", - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "items": { - "$ref": "#/components/schemas/SessionSummary" - }, - "title": "Response List Sessions Sessions Get", - "type": "array" - } - } - }, - "description": "Successful Response" - } - }, - "summary": "List Sessions" - }, + "/rl/runs": { "post": { - "operationId": "create_session_sessions_post", + "operationId": "submit_run_rl_runs_post", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SessionCreateRequest" + "$ref": "#/components/schemas/RLRunSubmitRequest" } } }, @@ -1224,22 +1513,12 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SessionCreateResponse" + "$ref": "#/components/schemas/RLRunSubmitResponse" } } }, "description": "Successful Response" }, - "400": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - }, - "description": "Bad Request" - }, "422": { "content": { "application/json": { @@ -1251,83 +1530,100 @@ "description": "Validation Error" } }, - "summary": "Create Session" + "summary": "Submit Run", + "tags": [ + "rl" + ] } }, - "/sessions/{session_id}": { - "delete": { - "operationId": "delete_session_sessions__session_id__delete", + "/rl/runs/{run_id}": { + "get": { + "operationId": "get_run_rl_runs__run_id__get", "parameters": [ { "in": "path", - "name": "session_id", + "name": "run_id", "required": true, "schema": { - "title": "Session Id", + "title": "Run Id", "type": "string" } - } - ], - "responses": { - "204": { - "description": "Successful Response" }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" + { + "in": "query", + "name": "tenant_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" } - } - }, - "description": "Not Found" + ], + "title": "Tenant Id" + } }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" - } - }, - "summary": "Delete Session" - }, - "get": { - "operationId": "get_session_sessions__session_id__get", - "parameters": [ { - "in": "path", - "name": "session_id", - "required": true, + "in": "query", + "name": "workspace_id", + "required": false, "schema": { - "title": "Session Id", - "type": "string" - } - } - ], - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SessionSummary" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" } - } - }, - "description": "Successful Response" + ], + "title": "Workspace Id" + } }, - "404": { + { + "in": "header", + "name": "x-tenant-id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Tenant-Id" + } + }, + { + "in": "header", + "name": "x-workspace-id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Workspace-Id" + } + } + ], + "responses": { + "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/RLRunStatusResponse" } } }, - "description": "Not Found" + "description": "Successful Response" }, "422": { "content": { @@ -1340,73 +1636,114 @@ "description": "Validation Error" } }, - "summary": "Get Session" + "summary": "Get Run", + "tags": [ + "rl" + ] } }, - "/sessions/{session_id}/attachments": { - "post": { - "operationId": "upload_attachments_sessions__session_id__attachments_post", + "/rl/runs/{run_id}/artifacts": { + "get": { + "operationId": "list_artifacts_rl_runs__run_id__artifacts_get", "parameters": [ { "in": "path", - "name": "session_id", + "name": "run_id", "required": true, "schema": { - "title": "Session Id", + "title": "Run Id", "type": "string" } - } - ], - "requestBody": { - "content": { - "multipart/form-data": { - "schema": { - "$ref": "#/components/schemas/Body_upload_attachments_sessions__session_id__attachments_post" - } + }, + { + "in": "query", + "name": "tenant_id", + "required": true, + "schema": { + "title": "Tenant Id", + "type": "string" } }, - "required": true - }, + { + "in": "query", + "name": "workspace_id", + "required": true, + "schema": { + "title": "Workspace Id", + "type": "string" + } + } + ], "responses": { "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AttachmentUploadResponse" + "$ref": "#/components/schemas/RLRunArtifactListResponse" } } }, "description": "Successful Response" }, - "400": { + "422": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/HTTPValidationError" } } }, - "description": "Bad Request" + "description": "Validation Error" + } + }, + "summary": "List Artifacts", + "tags": [ + "rl" + ] + } + }, + "/rl/runs/{run_id}/audit": { + "get": { + "operationId": "audit_run_rl_runs__run_id__audit_get", + "parameters": [ + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - }, - "description": "Not Found" + { + "in": "query", + "name": "tenant_id", + "required": true, + "schema": { + "title": "Tenant Id", + "type": "string" + } }, - "409": { + { + "in": "query", + "name": "workspace_id", + "required": true, + "schema": { + "title": "Workspace Id", + "type": "string" + } + } + ], + "responses": { + "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/RLRunAuditResponse" } } }, - "description": "Conflict" + "description": "Successful Response" }, "422": { "content": { @@ -1419,19 +1756,22 @@ "description": "Validation Error" } }, - "summary": "Upload Attachments" + "summary": "Audit Run", + "tags": [ + "rl" + ] } }, - "/sessions/{session_id}/command": { + "/rl/runs/{run_id}/cancel": { "post": { - "operationId": "post_command_sessions__session_id__command_post", + "operationId": "cancel_run_rl_runs__run_id__cancel_post", "parameters": [ { "in": "path", - "name": "session_id", + "name": "run_id", "required": true, "schema": { - "title": "Session Id", + "title": "Run Id", "type": "string" } } @@ -1440,53 +1780,23 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SessionCommandRequest" + "$ref": "#/components/schemas/RLRunCancelRequest" } } }, "required": true }, "responses": { - "202": { + "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SessionCommandResponse" + "$ref": "#/components/schemas/RLRunStatusResponse" } } }, "description": "Successful Response" }, - "400": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - }, - "description": "Bad Request" - }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - }, - "description": "Not Found" - }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - }, - "description": "Conflict" - }, "422": { "content": { "application/json": { @@ -1496,56 +1806,113 @@ } }, "description": "Validation Error" - }, - "501": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - }, - "description": "Not Implemented" } }, - "summary": "Post Command" + "summary": "Cancel Run", + "tags": [ + "rl" + ] } }, - "/sessions/{session_id}/ctrees": { + "/rl/runs/{run_id}/events": { "get": { - "operationId": "session_ctrees_sessions__session_id__ctrees_get", + "operationId": "get_events_rl_runs__run_id__events_get", "parameters": [ { "in": "path", - "name": "session_id", + "name": "run_id", "required": true, "schema": { - "title": "Session Id", + "title": "Run Id", "type": "string" } + }, + { + "in": "query", + "name": "from_sequence", + "required": false, + "schema": { + "default": 0, + "title": "From Sequence", + "type": "integer" + } + }, + { + "in": "query", + "name": "tenant_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tenant Id" + } + }, + { + "in": "query", + "name": "workspace_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Workspace Id" + } + }, + { + "in": "header", + "name": "x-tenant-id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Tenant-Id" + } + }, + { + "in": "header", + "name": "x-workspace-id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Workspace-Id" + } } ], "responses": { "200": { "content": { - "application/json": { + "text/plain": { "schema": { - "$ref": "#/components/schemas/CTreeSnapshotResponse" + "type": "string" } } }, "description": "Successful Response" }, - "404": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - }, - "description": "Not Found" - }, "422": { "content": { "application/json": { @@ -1557,28 +1924,49 @@ "description": "Validation Error" } }, - "summary": "Session Ctrees" + "summary": "Get Events", + "tags": [ + "rl" + ] } }, - "/sessions/{session_id}/download": { + "/rl/runs/{run_id}/replay/{artifact_id}": { "get": { - "operationId": "download_artifact_sessions__session_id__download_get", + "operationId": "replay_artifact_rl_runs__run_id__replay__artifact_id__get", "parameters": [ { "in": "path", - "name": "session_id", + "name": "run_id", "required": true, "schema": { - "title": "Session Id", + "title": "Run Id", + "type": "string" + } + }, + { + "in": "path", + "name": "artifact_id", + "required": true, + "schema": { + "title": "Artifact Id", "type": "string" } }, { "in": "query", - "name": "artifact", + "name": "tenant_id", "required": true, "schema": { - "title": "Artifact", + "title": "Tenant Id", + "type": "string" + } + }, + { + "in": "query", + "name": "workspace_id", + "required": true, + "schema": { + "title": "Workspace Id", "type": "string" } } @@ -1587,22 +1975,75 @@ "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/RLRunReplayResponse" + } } }, "description": "Successful Response" }, - "400": { + "422": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/HTTPValidationError" } } }, - "description": "Bad Request" + "description": "Validation Error" + } + }, + "summary": "Replay Artifact", + "tags": [ + "rl" + ] + } + }, + "/sessions": { + "get": { + "operationId": "list_sessions_sessions_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/SessionSummary" + }, + "title": "Response List Sessions Sessions Get", + "type": "array" + } + } + }, + "description": "Successful Response" + } + }, + "summary": "List Sessions" + }, + "post": { + "operationId": "create_session_sessions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SessionCreateRequest" + } + } }, - "404": { + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SessionCreateResponse" + } + } + }, + "description": "Successful Response" + }, + "400": { "content": { "application/json": { "schema": { @@ -1610,7 +2051,7 @@ } } }, - "description": "Not Found" + "description": "Bad Request" }, "422": { "content": { @@ -1623,12 +2064,12 @@ "description": "Validation Error" } }, - "summary": "Download Artifact" + "summary": "Create Session" } }, - "/sessions/{session_id}/events": { - "get": { - "operationId": "stream_events_sessions__session_id__events_get", + "/sessions/{session_id}": { + "delete": { + "operationId": "delete_session_sessions__session_id__delete", "parameters": [ { "in": "path", @@ -1638,57 +2079,10 @@ "title": "Session Id", "type": "string" } - }, - { - "in": "query", - "name": "replay", - "required": false, - "schema": { - "default": false, - "title": "Replay", - "type": "boolean" - } - }, - { - "in": "query", - "name": "limit", - "required": false, - "schema": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Limit" - } - }, - { - "in": "query", - "name": "from_id", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "From Id" - } } ], "responses": { - "200": { - "content": { - "application/json": { - "schema": {} - } - }, + "204": { "description": "Successful Response" }, "404": { @@ -1712,12 +2106,10 @@ "description": "Validation Error" } }, - "summary": "Stream Events" - } - }, - "/sessions/{session_id}/files": { + "summary": "Delete Session" + }, "get": { - "operationId": "session_files_sessions__session_id__files_get", + "operationId": "get_session_sessions__session_id__get", "parameters": [ { "in": "path", @@ -1727,106 +2119,18 @@ "title": "Session Id", "type": "string" } - }, - { - "in": "query", - "name": "path", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Path" - } - }, - { - "in": "query", - "name": "mode", - "required": false, - "schema": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Mode" - } - }, - { - "in": "query", - "name": "head_lines", - "required": false, - "schema": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Head Lines" - } - }, - { - "in": "query", - "name": "tail_lines", - "required": false, - "schema": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Tail Lines" - } - }, - { - "in": "query", - "name": "max_bytes", - "required": false, - "schema": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "title": "Max Bytes" - } } ], "responses": { "200": { - "content": { - "application/json": { - "schema": {} - } - }, - "description": "Successful Response" - }, - "400": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/SessionSummary" } } }, - "description": "Bad Request" + "description": "Successful Response" }, "404": { "content": { @@ -1838,16 +2142,6 @@ }, "description": "Not Found" }, - "409": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - }, - "description": "Conflict" - }, "422": { "content": { "application/json": { @@ -1859,12 +2153,12 @@ "description": "Validation Error" } }, - "summary": "Session Files" + "summary": "Get Session" } }, - "/sessions/{session_id}/input": { + "/sessions/{session_id}/attachments": { "post": { - "operationId": "post_input_sessions__session_id__input_post", + "operationId": "upload_attachments_sessions__session_id__attachments_post", "parameters": [ { "in": "path", @@ -1878,20 +2172,20 @@ ], "requestBody": { "content": { - "application/json": { + "multipart/form-data": { "schema": { - "$ref": "#/components/schemas/SessionInputRequest" + "$ref": "#/components/schemas/Body_upload_attachments_sessions__session_id__attachments_post" } } }, "required": true }, "responses": { - "202": { + "200": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SessionInputResponse" + "$ref": "#/components/schemas/AttachmentUploadResponse" } } }, @@ -1938,12 +2232,12 @@ "description": "Validation Error" } }, - "summary": "Post Input" + "summary": "Upload Attachments" } }, - "/sessions/{session_id}/skills": { - "get": { - "operationId": "session_skills_sessions__session_id__skills_get", + "/sessions/{session_id}/command": { + "post": { + "operationId": "post_command_sessions__session_id__command_post", "parameters": [ { "in": "path", @@ -1955,17 +2249,37 @@ } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SessionCommandRequest" + } + } + }, + "required": true + }, "responses": { - "200": { + "202": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SkillCatalogResponse" + "$ref": "#/components/schemas/SessionCommandResponse" } } }, "description": "Successful Response" }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Bad Request" + }, "404": { "content": { "application/json": { @@ -1976,6 +2290,16 @@ }, "description": "Not Found" }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Conflict" + }, "422": { "content": { "application/json": { @@ -1985,52 +2309,98 @@ } }, "description": "Validation Error" + }, + "501": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Not Implemented" } }, - "summary": "Session Skills" + "summary": "Post Command" } }, - "/status": { + "/sessions/{session_id}/ctrees": { "get": { - "operationId": "engine_status_status_get", + "operationId": "session_ctrees_sessions__session_id__ctrees_get", + "parameters": [ + { + "in": "path", + "name": "session_id", + "required": true, + "schema": { + "title": "Session Id", + "type": "string" + } + } + ], "responses": { "200": { "content": { "application/json": { "schema": { - "additionalProperties": true, - "title": "Response Engine Status Status Get", - "type": "object" + "$ref": "#/components/schemas/CTreeSnapshotResponse" } } }, "description": "Successful Response" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Not Found" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" } }, - "summary": "Engine Status" + "summary": "Session Ctrees" } }, - "/v1/provider-auth/attach": { - "post": { - "description": "Attach short-lived provider auth material to the in-memory engine store.", - "operationId": "attach_provider_auth_v1_provider_auth_attach_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProviderAuthAttachRequest" - } + "/sessions/{session_id}/download": { + "get": { + "operationId": "download_artifact_sessions__session_id__download_get", + "parameters": [ + { + "in": "path", + "name": "session_id", + "required": true, + "schema": { + "title": "Session Id", + "type": "string" } }, - "required": true - }, + { + "in": "query", + "name": "artifact", + "required": true, + "schema": { + "title": "Artifact", + "type": "string" + } + } + ], "responses": { "200": { "content": { "application/json": { - "schema": { - "$ref": "#/components/schemas/ProviderAuthAttachResponse" - } + "schema": {} } }, "description": "Successful Response" @@ -2045,17 +2415,7 @@ }, "description": "Bad Request" }, - "403": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - }, - "description": "Forbidden" - }, - "409": { + "404": { "content": { "application/json": { "schema": { @@ -2063,7 +2423,7 @@ } } }, - "description": "Conflict" + "description": "Not Found" }, "422": { "content": { @@ -2076,34 +2436,75 @@ "description": "Validation Error" } }, - "summary": "Attach Provider Auth" + "summary": "Download Artifact" } }, - "/v1/provider-auth/detach": { - "post": { - "operationId": "detach_provider_auth_v1_provider_auth_detach_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProviderAuthDetachRequest" - } + "/sessions/{session_id}/events": { + "get": { + "operationId": "stream_events_sessions__session_id__events_get", + "parameters": [ + { + "in": "path", + "name": "session_id", + "required": true, + "schema": { + "title": "Session Id", + "type": "string" } }, - "required": true - }, + { + "in": "query", + "name": "replay", + "required": false, + "schema": { + "default": false, + "title": "Replay", + "type": "boolean" + } + }, + { + "in": "query", + "name": "limit", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Limit" + } + }, + { + "in": "query", + "name": "from_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "From Id" + } + } + ], "responses": { "200": { "content": { "application/json": { - "schema": { - "$ref": "#/components/schemas/ProviderAuthDetachResponse" - } + "schema": {} } }, "description": "Successful Response" }, - "400": { + "404": { "content": { "application/json": { "schema": { @@ -2111,7 +2512,7 @@ } } }, - "description": "Bad Request" + "description": "Not Found" }, "422": { "content": { @@ -2124,25 +2525,941 @@ "description": "Validation Error" } }, - "summary": "Detach Provider Auth" + "summary": "Stream Events" } }, - "/v1/provider-auth/status": { + "/sessions/{session_id}/files": { "get": { - "operationId": "provider_auth_status_v1_provider_auth_status_get", - "responses": { - "200": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProviderAuthStatusResponse" + "operationId": "session_files_sessions__session_id__files_get", + "parameters": [ + { + "in": "path", + "name": "session_id", + "required": true, + "schema": { + "title": "Session Id", + "type": "string" + } + }, + { + "in": "query", + "name": "path", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Path" + } + }, + { + "in": "query", + "name": "mode", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Mode" + } + }, + { + "in": "query", + "name": "head_lines", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Head Lines" + } + }, + { + "in": "query", + "name": "tail_lines", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Tail Lines" + } + }, + { + "in": "query", + "name": "max_bytes", + "required": false, + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Bytes" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Not Found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Session Files" + } + }, + "/sessions/{session_id}/input": { + "post": { + "operationId": "post_input_sessions__session_id__input_post", + "parameters": [ + { + "in": "path", + "name": "session_id", + "required": true, + "schema": { + "title": "Session Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SessionInputRequest" + } + } + }, + "required": true + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SessionInputResponse" + } + } + }, + "description": "Successful Response" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Not Found" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Post Input" + } + }, + "/sessions/{session_id}/skills": { + "get": { + "operationId": "session_skills_sessions__session_id__skills_get", + "parameters": [ + { + "in": "path", + "name": "session_id", + "required": true, + "schema": { + "title": "Session Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SkillCatalogResponse" + } + } + }, + "description": "Successful Response" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Not Found" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Session Skills" + } + }, + "/status": { + "get": { + "operationId": "engine_status_status_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "title": "Response Engine Status Status Get", + "type": "object" + } + } + }, + "description": "Successful Response" + } + }, + "summary": "Engine Status" + } + }, + "/v1/provider-auth/attach": { + "post": { + "description": "Attach short-lived provider auth material to the in-memory engine store.", + "operationId": "attach_provider_auth_v1_provider_auth_attach_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProviderAuthAttachRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProviderAuthAttachResponse" + } + } + }, + "description": "Successful Response" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Forbidden" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Conflict" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Attach Provider Auth" + } + }, + "/v1/provider-auth/detach": { + "post": { + "operationId": "detach_provider_auth_v1_provider_auth_detach_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProviderAuthDetachRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProviderAuthDetachResponse" + } + } + }, + "description": "Successful Response" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Detach Provider Auth" + } + }, + "/v1/provider-auth/status": { + "get": { + "operationId": "provider_auth_status_v1_provider_auth_status_get", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProviderAuthStatusResponse" + } + } + }, + "description": "Successful Response" + } + }, + "summary": "Provider Auth Status" + } + }, + "/v1/rl/runs": { + "post": { + "operationId": "submit_run_v1_rl_runs_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RLRunSubmitRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RLRunSubmitResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Submit Run", + "tags": [ + "rl" + ] + } + }, + "/v1/rl/runs/{run_id}": { + "get": { + "operationId": "get_run_v1_rl_runs__run_id__get", + "parameters": [ + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "query", + "name": "tenant_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tenant Id" + } + }, + { + "in": "query", + "name": "workspace_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Workspace Id" + } + }, + { + "in": "header", + "name": "x-tenant-id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Tenant-Id" + } + }, + { + "in": "header", + "name": "x-workspace-id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Workspace-Id" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RLRunStatusResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Run", + "tags": [ + "rl" + ] + } + }, + "/v1/rl/runs/{run_id}/artifacts": { + "get": { + "operationId": "list_artifacts_v1_rl_runs__run_id__artifacts_get", + "parameters": [ + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "query", + "name": "tenant_id", + "required": true, + "schema": { + "title": "Tenant Id", + "type": "string" + } + }, + { + "in": "query", + "name": "workspace_id", + "required": true, + "schema": { + "title": "Workspace Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RLRunArtifactListResponse" } } }, "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" } }, - "summary": "Provider Auth Status" + "summary": "List Artifacts", + "tags": [ + "rl" + ] + } + }, + "/v1/rl/runs/{run_id}/audit": { + "get": { + "operationId": "audit_run_v1_rl_runs__run_id__audit_get", + "parameters": [ + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "query", + "name": "tenant_id", + "required": true, + "schema": { + "title": "Tenant Id", + "type": "string" + } + }, + { + "in": "query", + "name": "workspace_id", + "required": true, + "schema": { + "title": "Workspace Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RLRunAuditResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Audit Run", + "tags": [ + "rl" + ] + } + }, + "/v1/rl/runs/{run_id}/cancel": { + "post": { + "operationId": "cancel_run_v1_rl_runs__run_id__cancel_post", + "parameters": [ + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RLRunCancelRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RLRunStatusResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Cancel Run", + "tags": [ + "rl" + ] + } + }, + "/v1/rl/runs/{run_id}/events": { + "get": { + "operationId": "get_events_v1_rl_runs__run_id__events_get", + "parameters": [ + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "query", + "name": "from_sequence", + "required": false, + "schema": { + "default": 0, + "title": "From Sequence", + "type": "integer" + } + }, + { + "in": "query", + "name": "tenant_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Tenant Id" + } + }, + { + "in": "query", + "name": "workspace_id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Workspace Id" + } + }, + { + "in": "header", + "name": "x-tenant-id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Tenant-Id" + } + }, + { + "in": "header", + "name": "x-workspace-id", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "X-Workspace-Id" + } + } + ], + "responses": { + "200": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Get Events", + "tags": [ + "rl" + ] + } + }, + "/v1/rl/runs/{run_id}/replay/{artifact_id}": { + "get": { + "operationId": "replay_artifact_v1_rl_runs__run_id__replay__artifact_id__get", + "parameters": [ + { + "in": "path", + "name": "run_id", + "required": true, + "schema": { + "title": "Run Id", + "type": "string" + } + }, + { + "in": "path", + "name": "artifact_id", + "required": true, + "schema": { + "title": "Artifact Id", + "type": "string" + } + }, + { + "in": "query", + "name": "tenant_id", + "required": true, + "schema": { + "title": "Tenant Id", + "type": "string" + } + }, + { + "in": "query", + "name": "workspace_id", + "required": true, + "schema": { + "title": "Workspace Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RLRunReplayResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Replay Artifact", + "tags": [ + "rl" + ] } } } diff --git a/docs/contracts/kernel/KERNEL_EVENT_FAMILY_REGISTRY_V1.md b/docs/contracts/kernel/KERNEL_EVENT_FAMILY_REGISTRY_V1.md index a2490f53..42e4c5de 100644 --- a/docs/contracts/kernel/KERNEL_EVENT_FAMILY_REGISTRY_V1.md +++ b/docs/contracts/kernel/KERNEL_EVENT_FAMILY_REGISTRY_V1.md @@ -29,6 +29,9 @@ These exist because current clients need direct convenience payloads. They are n | --- | --- | --- | --- | | `todo_event` | `projection.todo_snapshot` | `service` | `host` | | `ctree_snapshot` | `projection.ctree_snapshot` | `service` | `host` | +| `rl.run.event` | `projection.rl_run_event` | `service` | `host` | + +`rl.run.event` covers the `/rl/runs/{run_id}/events` stream exposed by the Phase 3 RL API. It is a host-facing run-status projection, not a kernel event envelope and not a replay/conformance truth surface. The stream may carry convenience lifecycle strings such as `run.submitted`, `run.start`, `run.end`, and `cancel.requested`; those strings must not be promoted to canonical kernel event families without a separate schema, semantic dossier, and fixture update. ## CLI bridge stream-only and host-only events diff --git a/docs/contracts/policies/acr/ACR-20260708-rl-phase3-promotion-and-guardrails.md b/docs/contracts/policies/acr/ACR-20260708-rl-phase3-promotion-and-guardrails.md new file mode 100644 index 00000000..3d29984d --- /dev/null +++ b/docs/contracts/policies/acr/ACR-20260708-rl-phase3-promotion-and-guardrails.md @@ -0,0 +1,92 @@ +# ACR-20260708-rl-phase3-promotion-and-guardrails + +- `acr_id`: `ACR-20260708-rl-phase3-promotion-and-guardrails` +- `title`: RL Phase 3 promotion, evidence gates, and danger-zone guard enforcement +- `author`: codex +- `date`: 2026-07-08 +- `status`: implemented + +## 1) Problem Statement + +PR #27 adds RL Phase 3 runtime, target-evidence, API, promotion-audit, and CLI bridge surfaces. Those changes touch kernel danger-zone paths and promotion-critical evidence validators. + +The current branch needs explicit architecture review coverage because a green CI run is not enough for these surfaces: + +- final-report promotion must be tied to content-addressed evidence, not path existence; +- P3-M11 observability/object-store/scheduler claims must require real non-local tokened proof; +- public RL run lifecycle APIs must reject invalid resources and invalid terminal-state transitions; +- danger-zone CI guards must fail closed instead of skipping when guard scripts are missing. + +If we do nothing, the PR can claim `1000/1000` readiness while validating self-consistency rather than artifact truth, and future danger-zone PRs can pass without the documented ACR gate. + +## 2) Scope and Surfaces + +- Kernel modules touched: `breadboard/rl/phase3/**`, `breadboard/rl/phase2/service.py` contract consumers, `agentic_coder_prototype/api/cli_bridge/app.py` route exposure. +- Extension modules touched: none directly. +- Contract surfaces touched: `event`, `artifact`, `provider-request`, `promotion-audit`, `operator-script`, `danger-zone-governance`. +- Is this a **kernel danger-zone** change? `yes` +- Danger-zone: yes. + +## 3) Coupling and Generalization Impact + +- Core -> extension dependency introduced: no. +- Does this add any core -> extension dependency? `no` +- Does this narrow cross-harness parity behavior? `no` +- Does this alter default endpoint semantics? `yes`; it exposes `/rl/*` route projections and tightens accepted live-run lifecycle semantics. +- Coupling risk score: `medium`. + +Rationale: the branch adds a new RL subsystem and public API projection, but the patch keeps the kernel boundary explicit by validating evidence, preserving scorecard-update controls, and failing danger-zone governance closed. + +## 4) Change Classification + +- Classification: `additive` +- Compatibility window: current PR only; no stable external RL API version has shipped from this branch. +- Required schema/version bumps: none for existing kernel contract pack v1; RL Phase 3 report schemas remain v1 and become stricter about existing fields. + +## 5) Evidence and Validation Plan + +- Required contract lane tests: focused RL API/router, live service store, evidence gates, final report builder, and governance script checks. +- Required replay/parity checks: existing PR CI replay/conformance jobs remain required; no replay fixture format is changed by this ACR. +- Required conformance/ablation checks: kernel contract pack hash check and danger-zone ACR check must run instead of skipping. +- Required evidence bundles to refresh: Phase 3 final report artifacts should be regenerated only through existing `scripts/rl_phase3/*` commands when canonical evidence changes; this patch does not hand-edit `docs_tmp` evidence. +- Acceptance criteria: + - stale artifact input hashes fail validation; + - P3-M11 missing tokened object-store round-trip proof fails validation; + - active/core readiness point totals derive from milestone points; + - invalid RL run resources and terminal-state transitions are rejected; + - danger-zone guard scripts exist, hash listed contract-pack files, and require a changed ACR artifact for protected path diffs. + +## 6) Rollout Plan + +- Rollout phases: + 1. Land stricter validators and regression tests on PR #27. + 2. Keep existing `scorecard_update_allowed` false inside generated promotion-review artifacts unless the scorecard promotion path explicitly updates the scorecard artifact. + 3. Let CI enforce the restored danger-zone scripts on subsequent protected-path changes. +- Flags/toggles: none. +- Blast radius constraints: local to PR #27 branch; no production service migration or target credential rotation. +- Monitoring hooks: GitHub checks for Python, conformance, replay determinism, contract governance, and danger-zone guard. + +## 7) Rollback Plan + +- Trigger conditions: + - contract gate failure; + - replay determinism regression; + - boundary/coupling violation; + - sustained operational instability from stricter RL API validation. +- Exact rollback commands: + - `git revert ` for the PR #27 cleanup commit if the stricter validators or CI guards block unexpectedly. + - If only the governance checker misclassifies a safe PR, revert the checker change and keep the runtime validation patch. +- Artifact/state restoration steps: + - Do not mutate canonical `docs_tmp` evidence during rollback. + - Re-run the existing Phase 3 report builders before any future evidence promotion. +- Post-rollback verification: + - run focused RL tests touched by this ACR; + - run `python3 scripts/check_kernel_contract_pack_v1.py --manifest docs/contracts/policies/kernel_contract_pack_v1_manifest.json --repo-root .`; + - run `python3 scripts/check_danger_zone_acr.py --manifest docs/contracts/policies/kernel_danger_zone_manifest_v1.json --changed-files-file `. + +## 8) Approvals + +- Kernel reviewer: PR #27 reviewer gate. +- Contracts reviewer: PR #27 reviewer gate. +- Ops reviewer: PR #27 reviewer gate. +- Final decision: implemented for PR #27 cleanup commit; merge remains subject to normal PR review and CI. diff --git a/docs/reference/SCRIPTS_INDEX.md b/docs/reference/SCRIPTS_INDEX.md index 9f7a7efa..50e49aae 100644 --- a/docs/reference/SCRIPTS_INDEX.md +++ b/docs/reference/SCRIPTS_INDEX.md @@ -19,7 +19,7 @@ Summary counts live at: Current Python scripts inventoried: `260` -Primary category counts: +Primary category counts from the 2026-04-01 inventory remain: - `research`: `229` - `migration`: `10` @@ -28,6 +28,11 @@ Primary category counts: - `dev`: `5` - `archive`: `3` +Additional canonical research campaign families added after that inventory: + +- `rl_phase1`: Phase 1 RL evidence, transfer, and target-validation commands. +- `rl_phase3`: Phase 3/4 RL evidence, promotion, target-runner, and bounded target-validation commands. + ## Category meanings - `dev`: local setup, local helper, and devx scripts @@ -48,6 +53,8 @@ the live repo: - `scripts/release/` - `scripts/archive/` - `scripts/research/parity/` +- `scripts/rl_phase1/` +- `scripts/rl_phase3/` ## Stable vs internal expectation @@ -62,6 +69,8 @@ Treat these as primarily internal: - most `research` - most `migration` - `archive` +- `rl_phase1` +- `rl_phase3` ## Canonical taxonomy @@ -76,6 +85,8 @@ scripts/ │ ├── provider/ │ ├── rendering/ │ └── repo_hygiene/ +├── rl_phase1/ +├── rl_phase3/ ├── release/ ├── ops/ ├── migration/ @@ -100,8 +111,8 @@ examples: - `scripts/research/parity/audit_e4_target_drift.py` - `scripts/research/parity/check_e4_snapshot_coverage.py` -The old top-level script paths still exist only as compatibility wrappers while -the migration window is open. +Legacy top-level script paths still exist only as compatibility wrappers while +the migration window is open. `scripts/rl_phase1/` and `scripts/rl_phase3/` are the exception: they are canonical, phase-scoped research/evidence command families until their long-run evidence campaigns are retired or folded into a broader `scripts/research/rl/` taxonomy. ## Operator quick commands diff --git a/docs/rl_phase1/README.md b/docs/rl_phase1/README.md new file mode 100644 index 00000000..e338544c --- /dev/null +++ b/docs/rl_phase1/README.md @@ -0,0 +1,34 @@ +# BreadBoard RL Phase 1 + +BreadBoard RL Phase 1 is a local, probe-backed rollout substrate for Zyphra RL use cases. It preserves BreadBoard graph/replay/runtime truth and emits trainer-shaped projections without making any trainer or external framework canonical. + +Current verified boundary: M0-M11 are validated locally. M12 8xMI300X validation remains blocked until target hardware execution. + +## What Exists + +| Area | Path | +| --- | --- | +| EnvPackage IR | `breadboard/rl/env_package/` | +| Renderer/token records | `breadboard/rl/renderer/`, `breadboard/rl/export/token_record.py` | +| Session/runtime lifecycle | `breadboard/rl/session/`, `breadboard/rl/runtime/` | +| State/trace/replay | `breadboard/rl/state/`, `breadboard/rl/trace/`, `breadboard/rl/replay/` | +| Hardening/probes | `breadboard/rl/security/` | +| Controlled SWE run | `breadboard/rl/e2e/swe_probe.py` | +| VeRL-shaped JSONL/Parquet probe | `breadboard/rl/export/verl.py` | +| Adapter probe reports | `breadboard/rl/adapters/` | + +## Main Commands + +```bash +python scripts/rl_phase1/run_swe_probe.py +python scripts/rl_phase1/export_verl_probe.py +python scripts/rl_phase1/run_ray_warm_pool_probe.py +python scripts/rl_phase1/build_adapter_probe_reports.py +python -m pytest tests/rl -q +``` + +## Claim Boundary + +Allowed: local controlled SWE toy slice, VeRL-shaped JSONL/Parquet probe, local Ray worker prototype, and fixture/jsonl adapter probe reports. + +Forbidden: production RL rollouts, external benchmark support, production VeRL/BenchFlow/ORS/Prime integrations, trainer execution, and MI300X scale validation. diff --git a/docs/rl_phase1/decision_ledger.md b/docs/rl_phase1/decision_ledger.md new file mode 100644 index 00000000..7a703353 --- /dev/null +++ b/docs/rl_phase1/decision_ledger.md @@ -0,0 +1,10 @@ +# RL Phase 1 Decision Ledger + +| Decision | Status | Rationale | +| --- | --- | --- | +| Keep BreadBoard graph/replay/runtime canonical. | Locked | Trainer rows are projections and cannot replace evidence truth. | +| Use `breadboard/rl/` for new substrate. | Locked | Preserves existing `agentic_coder_prototype/rl` overlay. | +| Use controlled SWE toy before external SWE source. | Current M6 source | Avoids external setup blocking substrate proof. | +| Use JSONL VeRL-shaped probe before DataProto. | Current M7 scope | Proves row contract and smoke consumer first. | +| Treat adapter reports as probes. | Current M9 scope | Preserved/lost-field reports are not production integrations. | +| Backload MI300X validation. | Locked | M12 requires target hardware preflight and run evidence. | diff --git a/docs/rl_phase1/demo_script.md b/docs/rl_phase1/demo_script.md new file mode 100644 index 00000000..a4550d37 --- /dev/null +++ b/docs/rl_phase1/demo_script.md @@ -0,0 +1,39 @@ +# Demo Script + +1. Show `docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml`. +2. Load `examples/rl_env_packages/swe_toy_patch/env_package.yaml` and point out provenance, hardening, renderer, replay, and export eligibility. +3. Run: + +```bash +python scripts/rl_phase1/run_swe_probe.py +``` + +4. Inspect: + +```text +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/run_summary.json +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/qc_report.json +``` + +5. Run: + +```bash +python scripts/rl_phase1/export_verl_probe.py +``` + +6. Inspect: + +```text +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/smoke_consumer_report.json +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/verl_probe_rows.jsonl +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/verl_probe_rows.parquet +``` + +7. Run: + +```bash +python scripts/rl_phase1/run_ray_warm_pool_probe.py +python scripts/rl_phase1/build_adapter_probe_reports.py +``` + +8. Close with the claim boundary: local controlled SWE toy, JSONL/Parquet probe export, local Ray prototype, fixture/jsonl adapter probes; no production or external benchmark support yet. diff --git a/docs/rl_phase1/env_package_ir.md b/docs/rl_phase1/env_package_ir.md new file mode 100644 index 00000000..a43822f7 --- /dev/null +++ b/docs/rl_phase1/env_package_ir.md @@ -0,0 +1,18 @@ +# EnvPackage IR v1alpha + +EnvPackage v1alpha defines a runnable environment package with provenance, tasksets, splits, harness contract, runtime envelope, verifier, reward, renderer, hardening policy, replay requirements, and export eligibility. + +Golden packages: + +| Package | Purpose | +| --- | --- | +| `examples/rl_env_packages/python_console_toy/env_package.yaml` | Trusted local process toy lifecycle proof. | +| `examples/rl_env_packages/swe_toy_patch/env_package.yaml` | Controlled SWE-shaped package with 10 visible task ids and protected split guard. | + +Validation command: + +```bash +python -m pytest tests/rl/env_package -q +``` + +Core rule: runnable, exportable, and trainable are separate states. A package can be runnable but not trainable because of license, contamination, replay, hardening, or token-fidelity gates. diff --git a/docs/rl_phase1/m12_transfer_pack.md b/docs/rl_phase1/m12_transfer_pack.md new file mode 100644 index 00000000..81e96b20 --- /dev/null +++ b/docs/rl_phase1/m12_transfer_pack.md @@ -0,0 +1,147 @@ +# M12 Transfer And Target Preflight + +M12 is the only remaining scored milestone after local M0-M11 validation. It requires target-environment evidence on an 8xMI300X node with ROCm, Ray, VeRL, an inference engine, container runtime, filesystem/CAS smoke, SWE run, VeRL export smoke, warm-pool comparison, and final report. + +Local preparation commands: + +```bash +python scripts/rl_phase1/build_m12_transfer_pack.py +python scripts/rl_phase1/build_m12_transfer_archive.py +python scripts/rl_phase1/verify_m12_transfer_archive.py \ + --manifest ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep/m12_transfer_archive_manifest.json \ + --output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep/m12_archive_verify_report.json +python scripts/rl_phase1/apply_m12_transfer_overlay.py \ + --manifest ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep/m12_transfer_archive_manifest.json \ + --workspace-root .. \ + --output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_overlay_apply_probe/m12_overlay_apply_report.json +python scripts/rl_phase1/run_m12_bootstrap_dry_run.py \ + --repo-root . \ + --workspace-root .. \ + --transfer-prep-dir ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep \ + --output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_bootstrap_dry_run/m12_bootstrap_dry_run_report.json \ + --require-pass +python scripts/rl_phase1/run_m12_preflight.py +python scripts/rl_phase1/build_m12_final_report.py \ + --output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_local_final_report_probe/m12_final_report.json \ + --preflight-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_target_preflight/m12_preflight_report.json \ + --swe-run-summary ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/run_summary.json \ + --verl-smoke-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/smoke_consumer_report.json \ + --ray-probe-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m8_ray_warm_pool_probe/ray_probe_report.json \ + --warm-vs-cold-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m8_ray_warm_pool_probe/warm_vs_cold_report.json +python scripts/rl_phase1/summarize_m12_final_report_remediations.py \ + --final-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_local_final_report_probe/m12_final_report.json +python scripts/rl_phase1/audit_m12_score_promotion.py \ + --output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_local_promotion_audit/m12_promotion_audit.json \ + --final-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_local_final_report_probe/m12_final_report.json \ + --scorecard ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml \ + --claim-ledger ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md \ + --command-log-manifest ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json +python scripts/rl_phase1/check_m12_evidence_consistency.py \ + --phase-dir ../docs_tmp/ZYPHRA/RL_PHASE_1 \ + --output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_evidence_consistency/m12_evidence_consistency.json \ + --require-consistent +``` + +The local evidence-consistency report is itself validated before use: `consistent` and `errors` must match the embedded check table, unknown error rows are rejected, and the report remains non-scoring. + +The generated `m12_test_commands.sh` first verifies the transfer archive through `run_m12_logged_command.py`, writes a structured non-scoring verifier report to `m12_archive_verify/m12_archive_verify_report.json`, then runs target preflight through the same logger. The transfer-pack builder validates that the rendered target script has the same ordered command rows as `m12_transfer_manifest.json`; drift between the manifest command list and generated shell script fails local pack generation. The script creates one `M12_TARGET_RUN_ID` and passes it to every logged command; the logger also exports that value to child commands. Before launching target commands, the script enforces a target-run command-log reuse guard: an existing command-log manifest with a different target run ID is rejected, while intentional same-run retries remain possible by setting `M12_TARGET_RUN_ID` to the existing run ID. It also enforces a target close-out artifact reuse guard: an existing `m12_final_report/m12_final_report.json`, `m12_final_report/m12_remediation_summary.json`, or `m12_promotion_audit/m12_promotion_audit.json` is rejected before target commands start, requiring the operator to archive/remove stale close-out outputs before a fresh target sequence. Final-report eligibility requires all required command logs to share that single target run ID, every target component artifact to record the same ID through `target_run_identity`, the `target_run_identity` summary to be self-consistent with the embedded artifact sections, the command_logs embedded command-summary fields to be self-consistent with embedded command rows, and every required command log to match the expected command text for its required command ID, preventing stale, mixed, copied, or wrong-command evidence from being promoted. The logger streams output to the terminal, writes raw logs under `m12_command_logs/`, writes `argv_json` in each raw log header, and updates `m12_command_logs/command_log_manifest.json` with exit codes, timestamps, log paths, structured argv, sha256 hashes, target run IDs, and the canonical M12 required-command-ID list. If the wrapped process emits output without a trailing newline, the logger inserts one before `# completed_at`, keeping completion metadata as a separate trailer instead of fusing it to command output. Direct logger API callers and the CLI both reject unsafe command IDs before manifest writes, the CLI rejects unsafe target run IDs before launching wrapped commands, the CLI rejects log dirs that would produce log paths outside the manifest directory before launching wrapped commands, and direct logger API callers reject log paths outside the manifest directory. If a wrapped command cannot be spawned at all, the logger writes a failed raw log and manifest row with `exit_code=127` and `spawn_error` notes instead of leaving an untracked partial attempt. Command IDs are restricted to safe filename characters, and reruns preserve prior raw logs by writing `COMMAND_ID.attempt-002.log`, `COMMAND_ID.attempt-003.log`, and so on instead of overwriting `COMMAND_ID.log`; the manifest's top-level command fields point at the latest attempt while its `attempts` list preserves earlier attempts and structured argv. Manifest validation rejects claim-boundary drift, scorecard-update or M12-point claims, required-flag drift, completed command rows without attempt history, invalid status/exit-code congruence, invalid attempt argv, command/argv incongruence, raw-log header/manifest inconsistency, raw-log preamble/trailer layout mismatch, duplicate reserved raw-log header keys, unsafe attempt log paths or target run IDs, top-level/latest-attempt drift, duplicate command rows, unsafe or unknown manifest command IDs, narrowed or missing `required_command_ids`, invalid target run IDs, absolute log paths, parent-directory traversal in log paths, stale `target_run_ids` / `latest_target_run_id` summaries, and stale `all_required_logs_archived` / `all_required_commands_passed` summaries before target evidence can become promotion-ready. If archive verification or preflight fails, stop and preserve the verifier/preflight output, `m12_archive_verify/m12_archive_verify_report.json` if present, `m12_target_preflight/m12_preflight_report.json` if present, and the command-log manifest; do not continue into SWE/Ray/export target runs or award M12 points. If preflight passes, the generated script runs the validation suite, SWE probe, VeRL export, distributed Ray warm-pool probe, concrete M12 load ladder, concrete M12 soak, final-report command, and promotion-audit command in order through the same logger. The final command is logged as an optional `final_report` audit row and builds `m12_final_report/m12_final_report.json`. The promotion audit is logged as an optional `promotion_audit` row and builds `m12_promotion_audit/m12_promotion_audit.json`. These reports are still validation/review candidates, not scorecard edits; scorecard promotion requires a separate reviewed update. + +Implementation detail: the generated target script invokes `run_m12_preflight.py --require-pass` through the logger, so `set -e` stops the script on any blocked preflight after recording the failed command log. It also invokes `build_m12_final_report.py` through the logger with explicit target artifact arguments for archive verification, preflight, SWE, VeRL export, Ray, warm-vs-cold, load ladder, soak, command-log manifest, output path, and `--require-eligible`, so the command log records the exact target inputs consumed and the script exits nonzero unless every final-report gate is satisfied, including canonical required-command IDs, full command-log manifest validation, raw command-log archival, and hash verification. If the generated script exits after `m12_final_report/m12_final_report.json` exists, its ERR trap attempts to write `m12_final_report/m12_remediation_summary.json` through `summarize_m12_final_report_remediations.py` and then preserves the original failing exit code. The `final_report` log row is not itself part of the required eligibility set because the report necessarily reads the manifest before that command's log is appended. After that succeeds, `audit_m12_score_promotion.py --require-ready` independently rechecks required command text from the raw command-log manifest, confirms the final report's embedded required command rows equal the raw manifest rows, checks command-log manifest hashes, checks the optional `final_report` command's exact command text, checks the target final report, rejects promotion-audit control inputs outside the canonical target final-report, command-log manifest, scorecard, and claim-ledger paths, rejects promotion-audit outputs outside the canonical target `m12_promotion_audit/m12_promotion_audit.json` path, checks explicit scorecard and claim-ledger inputs, checks scorecard/claim-ledger pre-promotion state, and enforces promotion-audit missing-requirement and readiness self-consistency from embedded checks. The transfer builder, transfer summaries, generated script validator, and archived overlay verifier also enforce target-script promotion-audit explicit target-path gating for the promotion-audit output, final-report input, scorecard input, claim-ledger input, and command-log manifest input. The audit still emits `scorecard_update_allowed=false`; it only determines whether the evidence is ready for separate human scorecard review. Local preparation can still run these scripts without the require flags to persist blocked reports for audit. + +The remediation-summary helper validates its emitted summary before writing it: counts must match grouped action gates, gates must be known final-report gates, action IDs must be non-empty, artifact paths must be M12 target artifact paths, and eligible summaries cannot retain residual actions. + +`build_m12_transfer_archive.py` writes a companion `m12_transfer_evidence_pack.tar.gz`, `m12_transfer_evidence_pack.tar.gz.sha256`, and `m12_transfer_archive_manifest.json` under the transfer-prep directory. This archive is for transfer audit convenience only: it packages the curated RL Phase 1 source/test/doc overlay, evidence/control files, and generated transfer-prep files with portable colocated archive filenames plus portable workspace/archive-relative entry paths and hashes. It is not a replacement for checking out the exact repo SHA recorded in `m12_transfer_manifest.json`; operators must checkout that repo SHA, then overlay the archived RL Phase 1 source/control files before running target commands. The archive and transfer manifests explicitly set `scorecard_update_allowed=false`, `m12_points_awarded=false`, `archive_is_repo_replacement=false`, `archive_contains_source_overlay=true`, `archive_excludes_pycache=true`, `repo_root_path_portable=true`, `archive_paths_portable=true`, and `source_paths_portable=true`. + +The target preflight validator enforces preflight self-consistency and runtime-fingerprint/top-level evidence matching. A target preflight report cannot claim `preflight_passed` unless its ROCm/GPU/Ray/VeRL/inference/container/CAS evidence is consistent with a passing target run, and its runtime fingerprint must match the top-level preflight evidence used by the final report. + +`verify_m12_transfer_archive.py` validates the archive manifest, tarball, sha256 sidecar, archive member list, per-file member hashes, portable top-level archive path metadata, portable `source_path` metadata with no local absolute path leakage, generated transfer-prep file coverage, non-scoring boundary flags, archived `m12_transfer_manifest.json` identity/boundary/portable-root/coverage/expected-output fields, archived `m12_test_commands.sh` / `m12_transfer_manifest.json` semantic consistency including promotion-audit explicit target-path arguments, and archived readiness/transfer summary identity, boundary, non-scoring flags, and consistency against the archived transfer manifest. When invoked with `--output`, it writes a structured non-scoring pass/fail report with manifest identity, archive hash, included-entry count, transfer-coverage flags, and verifier errors. Run it after copying the transfer-prep directory to the target node and before executing `m12_test_commands.sh`. A verifier failure means transfer drift, script/manifest drift, summary/manifest drift, or archive corruption; preserve the failed verifier output and report and do not start target validation until repaired. + +`m12_apply_overlay.py` is generated into the transfer-prep directory as a standalone stdlib-only bootstrap helper. It verifies archive hashes, unique manifest and tar member paths, portable top-level archive path metadata, portable source metadata, deterministic archive metadata, gzip header mtime, sorted tar member order, tar member metadata, sha256 sidecar consistency, archive size, generated transfer-file coverage, safe destination paths, destination shape, archive readability, archived `m12_transfer_manifest.json` identity/boundary/portable-root/coverage/expected-output fields, archived `m12_test_commands.sh` / `m12_transfer_manifest.json` semantic consistency, and archived readiness/transfer summary identity, boundary, non-scoring flags, and consistency before extracting. The archive writer normalizes gzip mtime, tar member mtime, owner/group metadata, member order, file modes, top-level archive path fields, and serialized source metadata, so repeated builds from identical inputs produce the same archive hash without embedding local absolute source paths. The helper validates the full archive and destination map first and only writes files after all checks pass, so archive drift, corrupt archive bytes, target path collisions, duplicate archive paths, leaked local source metadata, leaked local archive paths, script/manifest drift, inner transfer-manifest drift, or stale archived readiness/transfer summaries cannot partially overlay the target workspace. Overlay reports are validated for status/error-list consistency, write-count bounds, dry-run zero-write behavior, successful-apply complete-write behavior, and `existing_destination_count` consistency against entry-level `exists` fields, so stale or hand-edited overlay summaries cannot pass local evidence checks. Run it in dry-run mode first, then run it with `--apply --allow-overwrite` only after confirming the workspace root is the target workspace containing the exact repo checkout. `m12_target_bootstrap.sh` sequences the target-side path: verify the exact checkout SHA from `m12_transfer_manifest.json`, reject a dirty checkout before overlay by default, run the overlay dry-run, apply the overlay, require the overlaid workspace copy of `m12_test_commands.sh` to exist and be readable, `cd` into `REPO_ROOT`, then hand off to that overlaid command script instead of the mutable staging copy. Set `BOOTSTRAP_DRY_RUN_ONLY=1` to test only the SHA check, dirty-checkout guard, and overlay dry-run without launching target validation. The bootstrap dry-run report records sha256 hashes for the bootstrap script, transfer manifest, archive manifest, and overlay dry-run report so the exact local rehearsal inputs are auditable; its validator reloads the referenced overlay dry-run report, applies the full overlay-report validator, and rejects embedded overlay-summary drift against the referenced file. `ALLOW_M12_DIRTY_CHECKOUT=1` exists only for local rehearsal/debugging against an already-overlaid development checkout; do not use it for target validation promotion. The repo-local `scripts/rl_phase1/apply_m12_transfer_overlay.py` exposes the same overlay behavior for local preparation and report generation. Overlay application and bootstrap dry-run reports are still preparation only: they do not validate M12, update the scorecard, or replace the requirement to run the target commands. + +If a late apply-time filesystem race still occurs after validation, the overlay helper emits a failed report with the actual `written_count`; operators must preserve that report and treat the workspace as needing rollback/recheckout before another promotion-eligible attempt. + +The target load ladder and soak commands are concrete scripts with target-oriented defaults: + +```bash +python scripts/rl_phase1/run_m12_logged_command.py \ + --manifest ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json \ + --log-dir ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs \ + --command-id target_load_ladder \ + -- python scripts/rl_phase1/run_m12_load_ladder.py \ + --package examples/rl_env_packages/python_console_toy/env_package.yaml \ + --output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_load_ladder/load_ladder_report.json + +python scripts/rl_phase1/run_m12_logged_command.py \ + --manifest ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json \ + --log-dir ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs \ + --command-id target_soak \ + -- python scripts/rl_phase1/run_m12_soak.py \ + --package examples/rl_env_packages/python_console_toy/env_package.yaml \ + --output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_soak/soak_report.json +``` + +Operators may override script parameters for a target maintenance window, but promotion still requires the logged command manifest and final-report gates. Local smoke tests may use `--local-mode`, smaller levels, or shorter soak durations; target validation should not use those local smoke reductions. + +Both `run_m12_load_ladder.py` and `run_m12_soak.py` validate their emitted reports before exiting. By default they fail closed for target validation: non-distributed Ray, missing required load levels, failed load levels, missing 100-level pass-or-skip reason, insufficient soak duration, soak runtime failures, or empty soak rows all produce a nonzero exit. Local-only rehearsals must pass `--smoke-mode` explicitly. + +The transfer manifest records `repo.root` as the portable checkout directory name, not the local build-machine absolute path; the exact source identity is the recorded git `head`, branch, and dirty status. The manifest records explicit coverage for the M12 transfer table: + +| Requirement | Coverage | +| --- | --- | +| Repo snapshot or commit SHA | Portable `repo.root`, `repo.root_path_portable=true`, `repo.head`, branch, and dirty status in `m12_transfer_manifest.json`. | +| Python environment lock | `requirements.txt`. | +| RL Phase 1 source/test/doc overlay | `breadboard/rl`, `scripts/rl_phase1`, `tests/rl`, `tests/test_rl_phase1_scorecard_schema.py`, `tests/test_rl_phase1_claim_ledger.py`, `docs/rl_phase1`, and `examples/rl_env_packages`. | +| ROCm/PyTorch/VeRL/Ray versions | Target `m12_preflight_report.json`. | +| EnvPackage set | Python console, SWE toy patch, and math console EnvPackages. | +| Run manifests | Preflight-first `test_commands` plus this runbook. | +| Test command list | `m12_test_commands.sh` and manifest `test_commands`. | +| Expected outputs | Manifest `expected_outputs`. | +| Rollback plan | `m12_rollback_plan.md`. | +| Overlay/bootstrap guard | Generated `m12_apply_overlay.py` with pre-write archive sidecar/hash/size/readability/generated-file/script/manifest/destination-shape validation, overlay report self-consistency validation, generated `m12_target_bootstrap.sh`, target dirty-checkout rejection by default, overlaid command-script handoff from `REPO_ROOT`, `scripts/rl_phase1/apply_m12_transfer_overlay.py`, and local dry-run `m12_overlay_apply_report.json`. | +| Final target report contract | `breadboard/rl/m12/final_report.py`, `scripts/rl_phase1/build_m12_final_report.py`, `scripts/rl_phase1/summarize_m12_final_report_remediations.py`, and target `m12_final_report.json`. | +| Score promotion audit | `breadboard/rl/m12/promotion_audit.py`, `scripts/rl_phase1/audit_m12_score_promotion.py`, and target `m12_promotion_audit.json`. | +| Load ladder and soak evidence | `breadboard/rl/m12/load_soak.py`, `scripts/rl_phase1/run_m12_load_ladder.py`, `scripts/rl_phase1/run_m12_soak.py`, `m12_node_load_ladder/load_ladder_report.json`, `m12_node_soak/soak_report.json`, fail-closed script validation, and final-report validation gates. | +| Raw command log archive | `breadboard/rl/m12/command_logs.py`, `scripts/rl_phase1/run_m12_logged_command.py`, `m12_command_logs/command_log_manifest.json`, `m12_command_log_manifest_template.json`, and final-report validation gates. | + +The transfer pack also writes operator-facing preparation files: + +| File | Purpose | +| --- | --- | +| `m12_readiness_summary.json` | Compact, non-scoring summary of artifact counts, command counts, expected outputs, exact fail-closed script flag keys/values, target-only outputs, and score-promotion rule; validated against `m12_transfer_manifest.json` before transfer pack generation succeeds. | +| `m12_apply_overlay.py` | Standalone target-side overlay applier for the source/control archive; validates hashes, sha sidecar, archive size, archive readability, generated-file coverage, safe destinations, destination shape, inner transfer-manifest identity/boundary, script/manifest consistency, and archived readiness/transfer summary identity, boundary, non-scoring flags, and consistency before any write; reports late write failures with accurate partial-write counts; defaults to dry-run and requires explicit `--apply --allow-overwrite` for writes. | +| `m12_target_bootstrap.sh` | Standalone target-side bootstrap wrapper that checks repo SHA, dry-runs overlay, applies overlay, verifies the overlaid command script, changes to `REPO_ROOT`, and then calls the overlaid `m12_test_commands.sh`. | +| `m12_transfer_summary.json` | Compact transfer-summary view validated against `m12_transfer_manifest.json`; stale counts, fail-closed flags, boundary drift, and readiness-summary references fail local consistency. | +| `m12_load_ladder_report_template.json` | Target report template for load levels 5, 20, 50, and optional/resource-skipped 100. | +| `m12_soak_report_template.json` | Target report template for the minimum 7200-second soak gate and runtime-failure accounting. | +| `m12_command_log_manifest_template.json` | Target manifest template for raw command logs, non-scoring boundary flags, required/optional command flags, command statuses, log paths, target run IDs, and sha256 hashes. The generated script populates deterministic command rows; load/soak rows are populated by operator-supplied logged commands. | +| `m12_promotion_audit/m12_promotion_audit.json` | Target output produced after an eligible final report; proves whether the evidence is ready for separate scorecard review while still setting `scorecard_update_allowed=false`. | +| `m12_transfer_evidence_pack.tar.gz` | Companion source-overlay/evidence archive for transfer audit; not a repo replacement and not validation evidence by itself. | +| `m12_transfer_archive_manifest.json` | Archive hash, portable archive filename fields, included entries, portable source metadata, generated file list, source-overlay flags, Python-cache exclusion flag, archived inner transfer-manifest/script plus readiness/transfer summary semantic validation, and explicit non-scoring/non-repo-replacement boundary. | +| `m12_transfer_evidence_pack.tar.gz.sha256` | One-line sha256 sidecar for transfer integrity checks. | +| `verify_m12_transfer_archive.py` | Target-side verifier for archive manifest, tarball contents, hashes, archived inner transfer-manifest identity/boundary, script/manifest semantic consistency, archived readiness/transfer summary semantic consistency, non-scoring boundary flags, and optional structured verifier-report emission. | + +The preflight report self-validates before the preflight CLI can exit successfully. It includes GPU visibility, ROCm tool discovery, PyTorch/Ray/VeRL/vLLM/SGLang imports, Ray status output where available, inference-engine feasibility, container runtime availability, a filesystem/CAS read/write/hash smoke, and an exact-schema tamper-evident sanitized runtime fingerprint with allowlisted environment keys only and path-like values redacted before hashing. + +The final report builder reads the target archive-verifier report, target preflight report, target SWE probe summary, target VeRL smoke report, target Ray probe report, target warm-vs-cold report, target load-ladder report, target soak report, and target command-log manifest. Malformed required or optional target JSON is recorded as a `read_error` and keeps the final report non-eligible instead of crashing or silently accepting the artifact; the promotion audit applies the same fail-closed behavior to malformed target final-report or command-log JSON. The final report also embeds component `validation_errors` from the archive verifier, SWE summary, VeRL smoke, Ray probe, warm-vs-cold, preflight, load-ladder, and soak validators; score eligibility requires those validator error lists to be empty when the corresponding target artifact is present. It also records `artifact_path_policy` and `target_run_identity`; score eligibility requires the generated target-default `m12_archive_verify`, `m12_node_*`, and command-log artifact paths rather than local M6/M7/M8 preparation paths, every target component artifact with a target-run identity to carry the same `target_run_id` as the required command logs, the `target_run_identity` summary to match the embedded artifact sections, the `artifact_path_policy` summary to match the embedded artifact paths, and every top-level `artifact_paths` entry to match the embedded section `path`. Non-eligible reports include `missing_gate_remediations`, a machine-readable map from every missing gate to the target action and artifact path operators must repair; the final-report validator rejects unknown missing gates, missing/stale remediation rows, stale `target_run_identity` summaries, stale `artifact_path_policy` summaries, stale `artifact_paths` versus embedded section paths, and stale `missing_gates` / `m12_score_eligible` summaries that no longer match the embedded evidence sections. `summarize_m12_final_report_remediations.py` validates a final report and prints or writes a non-scoring grouped remediation summary by target action for target-run recovery after a blocked final report; the generated target script invokes this helper automatically on failures after a final-report artifact exists. The builder sets `m12_score_eligible=false` unless every hard gate is satisfied: target artifact paths match the default target command script, archive verification report is present/readable/valid/passed with non-scoring boundary flags, target artifact target_run_id binding to command-log target_run_id holds, target_run_identity summary self-consistency holds, final-report missing-gate and score-eligibility self-consistency holds, artifact_path_policy self-consistency holds, artifact_paths embedded section-path self-consistency holds, preflight passed, SWE/export/Ray/preflight/load/soak component validators passed, 8 MI300X devices are visible, VeRL and Ray are importable, vLLM or SGLang is available, at least one container runtime is available, filesystem/CAS smoke passed, the sanitized preflight runtime fingerprint is present, exact-schema, and self-hash verified with only allowlisted environment keys, at least 10 SWE rows ran with no unknown statuses and at least one accepted row, JSONL and Parquet exports are tensorizable with matching row counts, Ray ran at least 10 rows over at least two workers in distributed mode rather than `local_mode`, load levels 5/20/50 passed in distributed mode, level 100 either passed in distributed mode or was resource-skipped with a reason, policy-version and queue-backpressure integrity held under load, soak duration is at least 7200 seconds, soak status is passed, soak runtime failures are zero, soak ran in distributed mode, the command-log manifest preserves the canonical M12 required-command-ID list, the full command-log manifest validator returns no errors, required raw command logs are archived with sha256 hashes that match the referenced log files, the manifest summary flags agree that required logs are archived and required commands passed, all required command logs share one non-empty target run ID, all required command logs match the expected command text, and `missing_gate_remediations=[]`. + +The evidence-consistency report independently reads `m12_archive_verify_report.json` and compares it against `m12_transfer_archive_manifest.json`, so a stale copied local verifier report cannot pass the local blocked-state audit. It also reads `m12_remediation_summary.json` and compares it against `m12_final_report.json`, so stale copied remediation guidance cannot pass either. + +The promotion audit builder reads the target final report, scorecard, claim ledger, and command-log manifest. Malformed or missing final-report, command-log, scorecard, or claim-ledger control inputs are recorded as `read_error` fields and keep `promotion_review_ready=false` instead of crashing or silently accepting the promotion. It sets `promotion_review_ready=false` unless the final report, command-log manifest, scorecard, claim-ledger input paths, and promotion-audit output path are the canonical target/control paths, the target final report is score-eligible with no missing gates, required command logs are still hash-verified, every required command log still matches the expected command text in the raw manifest, the final report's embedded required command rows equal the raw command-log manifest rows, the optional `final_report` command log is present, hash-verified, tied to the same target run ID as the required command logs, and matches the expected final-report builder command text, M12 is still unawarded in the scorecard, the scorecard and claim ledger are explicit readable inputs, and the claim ledger still requires separate review. It never edits the scorecard. + +The evidence-consistency checker is a local control-plane guard, not a target-node validation command. It cross-checks the scorecard, claim ledger, blocked-outcome report, handoff, transfer manifest, archive manifest, local preflight, local final-report probe, and local promotion-audit probe. It intentionally stays out of `m12_test_commands.sh` and out of the transfer archive to avoid a self-referential hash loop: the consistency report hashes the transfer manifest and archive manifest, while those manifests hash packaged artifacts. A passing consistency report means the local blocked-state evidence is internally aligned; it still does not award M12 points or prove target validation. + +Target-only load/soak/command-log artifacts are not fabricated by local preparation. If they are absent, the final report remains non-eligible and `--require-eligible` exits nonzero. Preserve that report as a blocked target outcome. + +Local artifacts: + +```text +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep/ +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_overlay_apply_probe/ +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_bootstrap_dry_run/ +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_target_preflight/ +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_local_final_report_probe/ +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_local_promotion_audit/ +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_evidence_consistency/ +``` + +Claim boundary: transfer/preflight preparation only. Do not award M12 points and do not claim MI300X/Ray/VeRL scale validation until the scripts run on the target node and the M12 exit gate passes. diff --git a/docs/rl_phase1/manual_qc_guide.md b/docs/rl_phase1/manual_qc_guide.md new file mode 100644 index 00000000..0c4b687c --- /dev/null +++ b/docs/rl_phase1/manual_qc_guide.md @@ -0,0 +1,21 @@ +# Manual QC Guide + +Review accepted, rejected, and quarantined rows before promoting claims. + +Minimum review dimensions: + +| Dimension | Check | +| --- | --- | +| Trace completeness | Reset, step, evaluate, evidence, projection are present. | +| Verifier evidence | Evidence hash exists and rerun agreement is recorded where available. | +| Hardening status | Quarantine findings are not silently accepted. | +| Replay status | Replay mismatch blocks export. | +| Token/mask validity | M7 rows pass length and logprob gates. | +| Projection manifest | Preserved and lost fields are explicit. | +| Claim wording | Source and support level are named exactly. | + +M6 QC artifact: + +```text +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/qc_report.json +``` diff --git a/docs/rl_phase1/replay_admission.md b/docs/rl_phase1/replay_admission.md new file mode 100644 index 00000000..db5933ba --- /dev/null +++ b/docs/rl_phase1/replay_admission.md @@ -0,0 +1,14 @@ +# Replay Admission + +M4 validates deterministic replay/projection primitives on toy evidence. + +Key rules: + +| Rule | Effect | +| --- | --- | +| Replay mismatch | Blocks export admission. | +| Quarantine status not clear | Blocks export admission. | +| Token records invalid | Blocks export admission. | +| M4 trainability | Always false; later gates are required. | + +Projection manifests record preserved and lost fields while keeping BreadBoard graph/replay/runtime as canonical truth. diff --git a/docs/rl_phase1/runtime_pool_runbook.md b/docs/rl_phase1/runtime_pool_runbook.md new file mode 100644 index 00000000..ec1db151 --- /dev/null +++ b/docs/rl_phase1/runtime_pool_runbook.md @@ -0,0 +1,18 @@ +# Runtime Pool Runbook + +M8 validates local runtime signatures, exact pool routing, worker quarantine, local Ray toy execution, telemetry, and warm-vs-cold reporting. + +Main command: + +```bash +python scripts/rl_phase1/run_ray_warm_pool_probe.py +``` + +Artifacts: + +```text +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m8_ray_warm_pool_probe/ray_probe_report.json +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m8_ray_warm_pool_probe/warm_vs_cold_report.json +``` + +Claim boundary: local Ray local-mode prototype only. Not production distributed rollout, not MI300X validation, not trainer scale. diff --git a/docs/rl_phase1/support_ladder.md b/docs/rl_phase1/support_ladder.md new file mode 100644 index 00000000..145dd9c4 --- /dev/null +++ b/docs/rl_phase1/support_ladder.md @@ -0,0 +1,13 @@ +# Support Ladder + +| Level | Meaning | +| --- | --- | +| Local schema | EnvPackage and record schemas validate. | +| Local lifecycle | Toy reset/step/evaluate/snapshot/restore/terminate works. | +| Local replay/export | Deterministic replay, projection, and JSONL/Parquet probe artifacts work. | +| Controlled SWE toy | 10-task controlled SWE-shaped run with hardening and QC. | +| Fixture adapter probe | Preserved/lost-field report exists with source artifacts, field mappings, fidelity notes, promotion requirements, and no production support. | +| Production support | Requires live external adapter execution, preserved/lost-field closure, docs, and regression tests. Not reached in Phase 1 so far. | +| MI300X validation | Requires M12 target-node run. Still blocked. | + +Current highest support level: local controlled SWE toy + fixture/jsonl adapter probes, with JSONL/Parquet export probe artifacts. diff --git a/docs/rl_phase1/swe_hardening.md b/docs/rl_phase1/swe_hardening.md new file mode 100644 index 00000000..a0a6aa38 --- /dev/null +++ b/docs/rl_phase1/swe_hardening.md @@ -0,0 +1,22 @@ +# SWE Hardening + +M5 validates local hardening primitives and reward-hack probe fixtures. M6 validates a 10-task controlled SWE toy run. + +Controls covered: + +| Control | Evidence | +| --- | --- | +| Python import-hook detection | `tests/rl/security/test_python_import_hook_cleanup.py` | +| Symlink escape detection | `tests/rl/security/test_symlink_escape.py` | +| Process cleanup before verify | `tests/rl/security/test_process_cleanup_contract.py` | +| Verifier evidence hashes | `tests/rl/security/test_verifier_evidence_hashes.py` | +| Quarantine-first behavior | `tests/rl/security/test_quarantine_rules.py` | +| Adversarial fixtures | `tests/rl/security/test_reward_hack_probe_suite.py` | + +M6 run artifacts: + +```text +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/ +``` + +Claim boundary: controlled SWE toy slice only. This is not SWE-Gym, SWE-rebench-V2, SETA, or Toolathlon support. diff --git a/docs/rl_phase1/verl_export_contract.md b/docs/rl_phase1/verl_export_contract.md new file mode 100644 index 00000000..a261ce74 --- /dev/null +++ b/docs/rl_phase1/verl_export_contract.md @@ -0,0 +1,31 @@ +# VeRL-Shaped Export Contract + +M7 emits `bb.verl_probe_row.v1alpha` JSONL and Parquet rows from the M6 controlled SWE run. + +Artifact: + +```text +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/verl_probe_rows.jsonl +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/verl_probe_rows.parquet +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/projection_manifest.json +``` + +Smoke report: + +```text +docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/smoke_consumer_report.json +``` + +Required row groups: + +| Group | Fields | +| --- | --- | +| Identity | rollout, trajectory, episode, task, split, package, group ids. | +| Policy | policy id/version, checkpoint ref, model requested/served, provider, engine, sampling config, policy staleness. | +| Tokens | prompt ids, completion ids, input ids, masks, completion logprobs when trainable candidate, explicit logprob availability/unavailability status. | +| Renderer | renderer id/version/config hash, tokenizer hash, chat template hash, stop ids, fidelity class. | +| Reward/runtime | reward scalar/vector, verifier id/version/hash, evidence refs, runtime backend/signature, image digest, state refs, artifact refs, package hash, metrics. | +| Admission | row status, hardening status, replay status, quarantine status, trainable flag, eligible exports, blocked reasons. | +| Projection | row projection ids plus export-level manifest preserving BreadBoard graph/replay/runtime as canonical truth. | + +Claim boundary: JSONL/Parquet probe only. This is not DataProto, not trainer execution, and not GRPO/PPO readiness. diff --git a/examples/rl_env_packages/math_console_toy/env_package.yaml b/examples/rl_env_packages/math_console_toy/env_package.yaml new file mode 100644 index 00000000..e844c0fb --- /dev/null +++ b/examples/rl_env_packages/math_console_toy/env_package.yaml @@ -0,0 +1,96 @@ +schema_version: bb.env_package.v1alpha +package_id: bb.math_console_toy.v1alpha +version: 0.1.0 +package_hash: sha256:dae346419318191efbcdaff56222b2ea7d1dde7a83191abef28df76f0d5c8aca +provenance: + created_at: "2026-06-17" + license: internal-dev-only + source_usage_policy: runnable-debug-only + contamination_scope: train_visible + source_refs: [] + source_hashes: + fixture: sha256:math-console-toy +tasksets: + - taskset_id: math_console_toy + source_kind: local_math_fixture + source_hash: sha256:math-console-toy-taskset + task_id_field: task_id + prompt_fields: + - prompt + allowed_splits: + - train_probe +splits: + train_probe: + split_id: train_probe + split_type: train + taskset_id: math_console_toy + selector: + task_ids: + - math_toy_001 + split_hash: sha256:math-console-toy-train-probe + immutable: true + optimizer_visible: false + trainer_visible: false + protected: false +harness: + harness_id: bb.math_console_toy + interaction_mode: multi_turn + observation_schema: + kind: messages + action_schema: + kind: python_snippet + termination: + on_tool: submit_answer + tools: + - python + - submit_answer + max_turns: 4 + hidden_state_policy: never_visible +runtime: + backend: local_process + isolation_level: trusted_dev + agent_user: sandbox + network: none + secrets_policy: ambient_forbidden + pool_key_fields: + - backend + - package_id +verifier: + verifier_id: math_console_toy_exact + kind: exact_match + code_hash: sha256:math-console-toy-verifier + input_contract: + requires: + - answer + output_contract: + reward: float + isolated_from_agent: true + rerun_policy: + required_for_promotion: false +reward: + reward_id: math_console_toy_reward + kind: binary + terminal_reward: true +renderer: + renderer_id: identity_renderer + tokenizer_hash: sha256:identity-tokenizer + chat_template_hash: sha256:identity-template + bridge_to_next_turn_required: true + mask_contract: + loss_mask_required: true +hardening: null +replay: + replay_required: true + live_replay_parity_required: true +exports: + allowed_formats: + - bb_graph + - jsonl_transition + trainable: false + requires_token_records: true + requires_replay_pass: true + support_level: experimental + trainability_status: debug_only + trainability_blockers: + - toy_fixture_not_training_data + support_evidence_refs: [] diff --git a/examples/rl_env_packages/python_console_toy/env_package.yaml b/examples/rl_env_packages/python_console_toy/env_package.yaml new file mode 100644 index 00000000..858d6d4f --- /dev/null +++ b/examples/rl_env_packages/python_console_toy/env_package.yaml @@ -0,0 +1,96 @@ +schema_version: bb.env_package.v1alpha +package_id: bb.python_console_toy.v1alpha +version: 0.1.0 +package_hash: sha256:579ba761d64475240a3c23a81d31aa8ddc361223763f2fa6534e608d70e1ef04 +provenance: + created_at: "2026-06-17" + license: internal-dev-only + source_usage_policy: runnable-debug-only + contamination_scope: train_visible + source_refs: [] + source_hashes: + fixture: sha256:python-console-toy +tasksets: + - taskset_id: python_console_toy + source_kind: local_fixture + source_hash: sha256:python-console-toy-taskset + task_id_field: task_id + prompt_fields: + - prompt + allowed_splits: + - train_probe +splits: + train_probe: + split_id: train_probe + split_type: train + taskset_id: python_console_toy + selector: + task_ids: + - py_toy_001 + split_hash: sha256:python-console-toy-train-probe + immutable: true + optimizer_visible: false + trainer_visible: false + protected: false +harness: + harness_id: bb.python_console_toy + interaction_mode: multi_turn + observation_schema: + kind: messages + action_schema: + kind: python_snippet + termination: + on_tool: submit_answer + tools: + - python + - submit_answer + max_turns: 4 + hidden_state_policy: never_visible +runtime: + backend: local_process + isolation_level: trusted_dev + agent_user: sandbox + network: none + secrets_policy: ambient_forbidden + pool_key_fields: + - backend + - package_id +verifier: + verifier_id: python_console_toy_exact + kind: exact_match + code_hash: sha256:python-console-toy-verifier + input_contract: + requires: + - answer + output_contract: + reward: float + isolated_from_agent: true + rerun_policy: + required_for_promotion: false +reward: + reward_id: python_console_toy_reward + kind: binary + terminal_reward: true +renderer: + renderer_id: identity_renderer + tokenizer_hash: sha256:identity-tokenizer + chat_template_hash: sha256:identity-template + bridge_to_next_turn_required: true + mask_contract: + loss_mask_required: true +hardening: null +replay: + replay_required: true + live_replay_parity_required: true +exports: + allowed_formats: + - bb_graph + - jsonl_transition + trainable: false + requires_token_records: true + requires_replay_pass: true + support_level: experimental + trainability_status: debug_only + trainability_blockers: + - toy_fixture_not_training_data + support_evidence_refs: [] diff --git a/examples/rl_env_packages/swe_toy_patch/env_package.yaml b/examples/rl_env_packages/swe_toy_patch/env_package.yaml new file mode 100644 index 00000000..ab1fddb1 --- /dev/null +++ b/examples/rl_env_packages/swe_toy_patch/env_package.yaml @@ -0,0 +1,138 @@ +schema_version: bb.env_package.v1alpha +package_id: bb.swe_toy_patch.v1alpha +version: 0.1.0 +package_hash: sha256:f8c19ad6821d49345652e21b81e9bf85fb32a2f108b1cd61e659881022a80435 +provenance: + created_at: "2026-06-17" + license: internal-dev-only + source_usage_policy: runnable-debug-only + contamination_scope: train_visible + source_refs: [] + source_hashes: + fixture: sha256:swe-toy-patch +tasksets: + - taskset_id: swe_toy_patch + source_kind: swe_rebench_v2 + source_hash: sha256:swe-toy-patch-taskset + task_id_field: task_id + prompt_fields: + - repo_snapshot + - issue + allowed_splits: + - train_probe + - protected_probe +splits: + train_probe: + split_id: train_probe + split_type: train + taskset_id: swe_toy_patch + selector: + task_ids: + - swe_toy_001 + - swe_toy_002 + - swe_toy_003 + - swe_toy_004 + - swe_toy_005 + - swe_toy_006 + - swe_toy_007 + - swe_toy_008 + - swe_toy_009 + - swe_toy_010 + split_hash: sha256:swe-toy-patch-train-probe + immutable: true + optimizer_visible: false + trainer_visible: false + protected: false + protected_probe: + split_id: protected_probe + split_type: protected + taskset_id: swe_toy_patch + selector: + task_ids: + - swe_toy_hidden_001 + split_hash: sha256:swe-toy-patch-protected-probe + immutable: true + optimizer_visible: false + trainer_visible: false + protected: true +harness: + harness_id: bb.swe_toy_patch + interaction_mode: patch_submit + observation_schema: + kind: repo_issue + action_schema: + kind: unified_diff + termination: + on_tool: submit_patch + tools: + - read_file + - write_file + - run_tests + - submit_patch + max_turns: 12 + hidden_state_policy: never_visible +runtime: + backend: docker + isolation_level: single_tenant_untrusted + image_digest: sha256:swe-toy-patch-image + agent_user: sandbox + network: none + secrets_policy: ambient_forbidden + pool_key_fields: + - backend + - package_id + - image_digest +verifier: + verifier_id: swe_toy_patch_pytest + kind: pytest + code_hash: sha256:swe-toy-patch-verifier + input_contract: + requires: + - workspace_ref + - patch_ref + output_contract: + reward: float + passed: bool + isolated_from_agent: true + rerun_policy: + required_for_promotion: true +reward: + reward_id: swe_toy_patch_reward + kind: binary + terminal_reward: true +renderer: + renderer_id: zaya_style_tool_renderer_probe + tokenizer_hash: sha256:zaya-style-tokenizer-probe + chat_template_hash: sha256:zaya-style-template-probe + bridge_to_next_turn_required: true + mask_contract: + loss_mask_required: true + tool_action_mask_required: true +hardening: + policy_id: swe_toy_patch_hardening + threat_level: untrusted_agent_code + agent_non_root_required: true + verifier_isolated_required: true + quarantine_on_findings: + - conftest_outside_tests + - sitecustomize_shadow + - pth_injection + - symlink_escape + - verifier_output_tamper +replay: + replay_required: true + live_replay_parity_required: true +exports: + allowed_formats: + - bb_graph + - jsonl_transition + - verl_jsonl_probe + trainable: false + requires_token_records: true + requires_replay_pass: true + support_level: experimental + trainability_status: debug_only + trainability_blockers: + - toy_fixture_not_training_data + - no_real_swe_source_license + support_evidence_refs: [] diff --git a/examples/rl_quickstart/README.md b/examples/rl_quickstart/README.md new file mode 100644 index 00000000..2f0e1ae3 --- /dev/null +++ b/examples/rl_quickstart/README.md @@ -0,0 +1,127 @@ +# RL quickstart: BreadBoard EnvPackage + NeMo Gym AgentLoop + veRL/vLLM + +These scripts show the current Zyphra stack at the function-calling seam: + +- BreadBoard loads an `EnvPackage` from `examples/rl_env_packages/`. +- `zyphra_verl.nemo_gym_loop.NeMoGymToolUseLoop` renders the row with the model chat template, calls veRL's rollout server client, parses the result with veRL `ToolParser`, and scores with NeMo Gym's `ToolCallComparator`. +- vLLM serves the model through its OpenAI-compatible endpoint. The quickstart actor adapts that endpoint to veRL's `LLMServerClient`/`GlobalRequestLoadBalancer` path. + +The example row is a single function-calling task: call `get_weather` with `city="Paris"`. + +## Files + +- `single_turn_vllm_rollout.py` runs one row through a live vLLM OpenAI server and the Zyphra NeMo Gym AgentLoop. +- `reward_check.py` runs the no-GPU gold/negative verifier preflight against the same AgentLoop `_score` path: gold returns `1.0`, negative returns `0.0`. + +## Prerequisites + +Run from `breadboard_repo_integration_main_20260326` unless you adjust paths. + +You need: + +- a GPU node with ROCm-visible devices for the vLLM rollout script; +- the `vllm/vllm-openai-rocm:nightly` image; +- the Zyphra `verl_wrapper` directory from the Phase 4 payload; +- the veRL and NeMo Gym checkouts pinned by that wrapper (`verl_wrapper/third_party/verl` and `verl_wrapper/third_party/nemo-gym`); +- Python deps installed inside the container: BreadBoard import path, `zyphra_verl`, wrapper-pinned veRL, wrapper-pinned NeMo Gym, `ray`, `omegaconf`, `transformers`, and `requests`; +- a model served by vLLM. The small smoke-test default is `Qwen/Qwen2.5-0.5B-Instruct`. + +Example paths used below: + +```bash +BB_REPO=/workspace/breadboard_repo_integration_main_20260326 +PAYLOAD=/workspace/real_rollout_agentloop_attempt_20260706T213000Z +VERL_WRAPPER=$PAYLOAD/verl_wrapper +NEMO_GYM_DIR=$VERL_WRAPPER/third_party/nemo-gym +VERL_DIR=$VERL_WRAPPER/third_party/verl +MODEL=Qwen/Qwen2.5-0.5B-Instruct +PORT=8000 +``` + +## Gold/negative reward check + +This check does not start vLLM and does not need a GPU, but it still imports the real wrapper and NeMo Gym verifier. + +```bash +docker run --rm --ipc=host \ + -v "$PWD":/workspace/breadboard_repo_integration_main_20260326 \ + -v "$PAYLOAD":/workspace/real_rollout_agentloop_attempt_20260706T213000Z \ + --entrypoint bash \ + vllm/vllm-openai-rocm:nightly \ + -lc 'set -euo pipefail + export PYTHONPATH=/workspace/breadboard_repo_integration_main_20260326:/workspace/real_rollout_agentloop_attempt_20260706T213000Z/verl_wrapper/src:/workspace/real_rollout_agentloop_attempt_20260706T213000Z/verl_wrapper/third_party/verl:/workspace/real_rollout_agentloop_attempt_20260706T213000Z/verl_wrapper/third_party/nemo-gym:${PYTHONPATH:-} + cd /workspace/breadboard_repo_integration_main_20260326 + python examples/rl_quickstart/reward_check.py \ + --verl-wrapper /workspace/real_rollout_agentloop_attempt_20260706T213000Z/verl_wrapper \ + --nemo-gym-dir /workspace/real_rollout_agentloop_attempt_20260706T213000Z/verl_wrapper/third_party/nemo-gym' +``` + +Expected output is one JSON line with: + +```json +{"checks":{"gold_reward":1.0,"negative_reward":0.0}} +``` + +The actual line includes package IDs, hashes, and the verifier module path. + +## Single-turn vLLM rollout + +Start vLLM inside the ROCm container, then run the script against that server in the same container. This command assumes the Phase 4 `verl_wrapper` is mounted beside the BreadBoard repo. + +```bash +docker run --rm --ipc=host \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + --device=/dev/kfd --device=/dev/dri --group-add video \ + -e HIP_VISIBLE_DEVICES="${HIP_VISIBLE_DEVICES:-0}" \ + -e HF_HOME=/workspace/hf_home \ + -v "$PWD":/workspace/breadboard_repo_integration_main_20260326 \ + -v "$PAYLOAD":/workspace/real_rollout_agentloop_attempt_20260706T213000Z \ + --entrypoint bash \ + vllm/vllm-openai-rocm:nightly \ + -lc 'set -euo pipefail + export BB_REPO=/workspace/breadboard_repo_integration_main_20260326 + export VERL_WRAPPER=/workspace/real_rollout_agentloop_attempt_20260706T213000Z/verl_wrapper + export NEMO_GYM_DIR=$VERL_WRAPPER/third_party/nemo-gym + export VERL_DIR=$VERL_WRAPPER/third_party/verl + export MODEL=Qwen/Qwen2.5-0.5B-Instruct + export PORT=8000 + export PYTHONPATH=$BB_REPO:$VERL_WRAPPER/src:$VERL_DIR:$NEMO_GYM_DIR:${PYTHONPATH:-} + python -m vllm.entrypoints.openai.api_server \ + --model "$MODEL" \ + --host 127.0.0.1 \ + --port "$PORT" \ + --tensor-parallel-size 1 \ + --gpu-memory-utilization 0.35 \ + --max-model-len 2048 \ + --no-enable-log-requests \ + --trust-remote-code & + server_pid=$! + trap "kill $server_pid 2>/dev/null || true" EXIT + python - < dict[str, str]: + verl_src = verl_wrapper / "src" + verl_dir = verl_wrapper / "third_party" / "verl" + nemo_dir = nemo_gym_dir or (verl_wrapper / "third_party" / "nemo-gym") + missing = [str(path) for path in (verl_src, verl_dir, nemo_dir) if not path.exists()] + if missing: + raise FileNotFoundError(f"exact wrapper paths missing: {', '.join(missing)}") + for path in reversed((verl_src, verl_dir, nemo_dir)): + sys.path.insert(0, str(path)) + os.environ.setdefault("ZYPHRA_NEMO_GYM_DIR", str(nemo_dir)) + return {"wrapper_src": str(verl_src), "verl": str(verl_dir), "nemo_gym": str(nemo_dir)} + + +def _json_line(payload: dict[str, Any]) -> str: + return json.dumps(payload, sort_keys=True, separators=(",", ":")) + + + +def run_reward_check(args: argparse.Namespace) -> dict[str, Any]: + from breadboard.rl.env_package import load_env_package + + env_package = load_env_package(args.env_package) + wrapper_paths = _add_runtime_paths(args.verl_wrapper, args.nemo_gym_dir) + + from zyphra_verl.nemo_gym_loop import NeMoGymToolUseLoop, _load_canonical_verifier + + verifier, fn_call_cls = _load_canonical_verifier() + loop = NeMoGymToolUseLoop.__new__(NeMoGymToolUseLoop) + loop._FnCall = fn_call_cls + loop._ExpectedFunctionCall = verifier.ExpectedFunctionCall + loop._comparator = verifier.ToolCallComparator( + config=verifier.ToolCallComparatorConfig(word_count_similarity_threshold=args.word_similarity_threshold) + ) + + expected = {"type": "function_call", "name": "get_weather", "arguments": {"city": "Paris"}} + gold_tool_call = SimpleNamespace(name="get_weather", arguments=json.dumps({"city": "Paris"}, sort_keys=True)) + negative_tool_call = SimpleNamespace(name="get_weather", arguments=json.dumps({"city": "Lyon"}, sort_keys=True)) + + gold_reward = float(loop._score(expected, [gold_tool_call])) + negative_reward = float(loop._score(expected, [negative_tool_call])) + + assert gold_reward == 1.0, f"gold reward should be 1.0, got {gold_reward}" + assert negative_reward == 0.0, f"negative reward should be 0.0, got {negative_reward}" + + return { + "schema_version": "bb.rl_quickstart.nemo_gym_reward_check.v1", + "env_package_id": env_package.package_id, + "env_package_hash": env_package.package_hash, + "agent_loop": "zyphra_verl.nemo_gym_loop.NeMoGymToolUseLoop", + "verifier_module": str(getattr(verifier, "__file__", "")), + "wrapper_paths": wrapper_paths, + "checks": { + "gold_reward": gold_reward, + "negative_reward": negative_reward, + "gold_assertion": "reward == 1.0", + "negative_assertion": "reward == 0.0", + }, + "passed": True, + } + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Check Zyphra's NeMo Gym AgentLoop reward path on one gold and one negative tool call." + ) + parser.add_argument("--env-package", type=Path, default=DEFAULT_ENV_PACKAGE, help="BreadBoard EnvPackage YAML to load.") + parser.add_argument("--verl-wrapper", type=Path, required=True, help="Path to verl_wrapper containing src/zyphra_verl.") + parser.add_argument("--nemo-gym-dir", type=Path, default=None, help="Path to the NeMo Gym checkout; also sets ZYPHRA_NEMO_GYM_DIR.") + parser.add_argument("--word-similarity-threshold", type=float, default=0.1) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_arg_parser().parse_args(argv) + print(_json_line(run_reward_check(args))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/rl_quickstart/single_turn_vllm_rollout.py b/examples/rl_quickstart/single_turn_vllm_rollout.py new file mode 100644 index 00000000..7aa4d1cc --- /dev/null +++ b/examples/rl_quickstart/single_turn_vllm_rollout.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from pathlib import Path +from typing import Any +from uuid import uuid4 + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) + + + +DEFAULT_ENV_PACKAGE = Path(__file__).resolve().parents[1] / "rl_env_packages" / "math_console_toy" / "env_package.yaml" +DEFAULT_MODEL = "Qwen/Qwen2.5-0.5B-Instruct" +DEFAULT_BASE_URL = "http://127.0.0.1:8000" + + +def _json_line(payload: dict[str, Any]) -> str: + return json.dumps(payload, sort_keys=True, separators=(",", ":")) + + +def _add_runtime_paths(verl_wrapper: Path, nemo_gym_dir: Path | None) -> dict[str, str]: + verl_src = verl_wrapper / "src" + verl_dir = verl_wrapper / "third_party" / "verl" + nemo_dir = nemo_gym_dir or (verl_wrapper / "third_party" / "nemo-gym") + missing = [str(path) for path in (verl_src, verl_dir, nemo_dir) if not path.exists()] + if missing: + raise FileNotFoundError(f"exact wrapper paths missing: {', '.join(missing)}") + for path in reversed((verl_src, verl_dir, nemo_dir)): + sys.path.insert(0, str(path)) + os.environ.setdefault("ZYPHRA_NEMO_GYM_DIR", str(nemo_dir)) + return {"wrapper_src": str(verl_src), "verl": str(verl_dir), "nemo_gym": str(nemo_dir)} + + +def _tool_use_row() -> dict[str, Any]: + return { + "messages": [ + { + "role": "user", + "content": "Use the available tool once. Call get_weather with city Paris.", + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Return weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ], + "expected_action": {"type": "function_call", "name": "get_weather", "arguments": {"city": "Paris"}}, + } + + + +async def run_single_turn_rollout(args: argparse.Namespace) -> dict[str, Any]: + from breadboard.rl.env_package import load_env_package + + env_package = load_env_package(args.env_package) + wrapper_paths = _add_runtime_paths(args.verl_wrapper, args.nemo_gym_dir) + + import ray + import requests + from omegaconf import OmegaConf + from transformers import AutoTokenizer + from verl.workers.rollout.llm_server import GlobalRequestLoadBalancer, LLMServerClient + from verl.workers.rollout.replica import TokenOutput + from zyphra_verl.nemo_gym_loop import NeMoGymToolUseLoop, ToolParser, _load_canonical_verifier + + tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True) + if tokenizer.pad_token_id is None: + tokenizer.pad_token = tokenizer.eos_token + + @ray.remote(num_cpus=0) + class VLLMOpenAICompletionActor: + def __init__(self, model: str, base_url: str): + self.model = model + self.base_url = base_url.rstrip("/") + self.tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True) + if self.tokenizer.pad_token_id is None: + self.tokenizer.pad_token = self.tokenizer.eos_token + self.generate_calls = 0 + self.last_text = "" + + def generate(self, request_id, prompt_ids, sampling_params, **kwargs): + self.generate_calls += 1 + prompt = self.tokenizer.decode(prompt_ids, skip_special_tokens=False) + response = requests.post( + f"{self.base_url}/v1/completions", + json={ + "model": self.model, + "prompt": prompt, + "max_tokens": int(sampling_params.get("max_tokens", 128)), + "temperature": float(sampling_params.get("temperature", 0.0)), + "logprobs": 1, + }, + timeout=float(os.environ.get("BB_RL_QUICKSTART_VLLM_TIMEOUT", "180")), + ) + response.raise_for_status() + data = response.json() + choice = data["choices"][0] + text = choice.get("text") or "" + self.last_text = text + token_ids = self.tokenizer.encode(text, add_special_tokens=False) + return TokenOutput( + token_ids=token_ids, + log_probs=[0.0] * len(token_ids), + num_preempted=0, + extra_fields={ + "real_vllm_http_server": True, + "server_base_url": self.base_url, + "openai_completion_id": data.get("id", ""), + "request_id": request_id, + "raw_generation_text": text, + }, + ) + + def status(self): + return {"generate_calls": self.generate_calls, "last_text": self.last_text} + + if not ray.is_initialized(): + ray.init(num_cpus=2, include_dashboard=False, ignore_reinit_error=True, logging_level="ERROR") + + actor = VLLMOpenAICompletionActor.remote(args.model, args.base_url) + load_balancer = GlobalRequestLoadBalancer.remote(servers={args.base_url.rstrip("/"): actor}) + server_manager = LLMServerClient(config=OmegaConf.create({}), load_balancer_handle=load_balancer) + + verifier, fn_call_cls = _load_canonical_verifier() + loop = NeMoGymToolUseLoop.__new__(NeMoGymToolUseLoop) + loop.response_length = args.response_length + loop.rollout_config = type( + "RolloutConfig", + (), + { + "response_length": args.response_length, + "prompt_length": args.prompt_length, + "multi_turn": type("MultiTurn", (), {"format": args.tool_format})(), + }, + )() + loop.tokenizer = tokenizer + loop.processor = None + loop.server_manager = server_manager + loop.loop = asyncio.get_running_loop() + loop.apply_chat_template_kwargs = {} + loop.system_prompt = [] + loop.tool_parser = ToolParser.get_tool_parser(args.tool_format, tokenizer) + loop._FnCall = fn_call_cls + loop._ExpectedFunctionCall = verifier.ExpectedFunctionCall + loop._comparator = verifier.ToolCallComparator( + config=verifier.ToolCallComparatorConfig(word_count_similarity_threshold=args.word_similarity_threshold) + ) + + row = _tool_use_row() + output = await loop.run( + {"max_tokens": args.max_tokens, "temperature": args.temperature}, + raw_prompt=row["messages"], + extra_info={"tools": row["tools"], "expected_action": row["expected_action"]}, + ) + actor_status = ray.get(actor.status.remote()) + return { + "schema_version": "bb.rl_quickstart.single_turn_vllm_rollout.v1", + "env_package_id": env_package.package_id, + "env_package_hash": env_package.package_hash, + "agent_loop": "zyphra_verl.nemo_gym_loop.NeMoGymToolUseLoop", + "agent_loop_registry_name": "nemo_gym_tool_use", + "server_manager_class": f"{server_manager.__class__.__module__}.{server_manager.__class__.__name__}", + "base_url": args.base_url.rstrip("/"), + "model": args.model, + "wrapper_paths": wrapper_paths, + "request_id": uuid4().hex, + "reward_score": float(output.reward_score), + "metrics": dict(output.metrics or {}), + "response_token_count": len(output.response_ids), + "real_vllm_http_server": bool(output.extra_fields.get("real_vllm_http_server")), + "actor_status": actor_status, + } + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run one NeMo Gym function-calling row through Zyphra's veRL AgentLoop using a vLLM OpenAI server." + ) + parser.add_argument("--env-package", type=Path, default=DEFAULT_ENV_PACKAGE, help="BreadBoard EnvPackage YAML to load.") + parser.add_argument("--verl-wrapper", type=Path, required=True, help="Path to verl_wrapper containing src/zyphra_verl.") + parser.add_argument("--nemo-gym-dir", type=Path, default=None, help="Path to the NeMo Gym checkout; also sets ZYPHRA_NEMO_GYM_DIR.") + parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="Existing vLLM OpenAI-compatible server URL.") + parser.add_argument("--model", default=DEFAULT_MODEL, help="Model name served by vLLM.") + parser.add_argument("--tool-format", default="hermes", help="veRL ToolParser format.") + parser.add_argument("--prompt-length", type=int, default=1024) + parser.add_argument("--response-length", type=int, default=128) + parser.add_argument("--max-tokens", type=int, default=128) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--word-similarity-threshold", type=float, default=0.1) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_arg_parser().parse_args(argv) + print(_json_line(asyncio.run(run_single_turn_rollout(args)))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/requirements.txt b/requirements.txt index 679c9036..85d01bb3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # Core dependencies openai>=1.0.0 anthropic>=0.31.0 +pyarrow>=15.0.0 # Required for RL VeRL Parquet probe export/smoke tests ray>=2.0.0 pyyaml>=6.0.0 python-lsp-server>=1.7.0 diff --git a/scripts/_inventory/python_scripts_inventory_summary_20260401.md b/scripts/_inventory/python_scripts_inventory_summary_20260401.md index 504898ca..6605dfed 100644 --- a/scripts/_inventory/python_scripts_inventory_summary_20260401.md +++ b/scripts/_inventory/python_scripts_inventory_summary_20260401.md @@ -13,6 +13,12 @@ Total inventoried Python scripts: `260` - `release`: `8` - `research`: `229` + +## Post-inventory canonical campaign families + +- `scripts/rl_phase1/`: canonical Phase 1 RL evidence, transfer, and target-validation command family added after the 2026-04-01 inventory snapshot. +- `scripts/rl_phase3/`: canonical Phase 3/4 RL evidence, promotion, target-runner, and bounded target-validation command family added after the 2026-04-01 inventory snapshot. + ## Canonicalized moved script paths - `guardrail_metrics.py` -> `ops/guardrail_metrics.py` diff --git a/scripts/check_danger_zone_acr.py b/scripts/check_danger_zone_acr.py new file mode 100755 index 00000000..27fec161 --- /dev/null +++ b/scripts/check_danger_zone_acr.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Enforce Kernel Danger-Zone ACR prerequisites for PR changed files. + +Exit codes: +- 0: pass +- 2: danger-zone policy failure or malformed manifest +- 3: invalid input/runtime error +""" + +from __future__ import annotations + +import argparse +import fnmatch +import json +import re +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_MANIFEST = ROOT / "docs" / "contracts" / "policies" / "kernel_danger_zone_manifest_v1.json" +EXPECTED_SCHEMA = "breadboard.kernel_danger_zone_manifest.v1" +ACR_PATTERN = "docs/contracts/policies/acr/ACR-*.md" + + +def _load_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"manifest must be a JSON object: {path}") + return payload + + +def _normalize_path(raw_path: str) -> str: + path = raw_path.strip().replace("\\", "/") + while path.startswith("./"): + path = path[2:] + return path + + +def _read_changed_files(path: Path) -> list[str]: + files: list[str] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + normalized = _normalize_path(line) + if normalized: + files.append(normalized) + return files + + +def _matches_glob(path: str, pattern: str) -> bool: + normalized_pattern = _normalize_path(pattern) + if normalized_pattern.endswith("/**"): + prefix = normalized_pattern[:-3] + return path == prefix or path.startswith(prefix + "/") + return fnmatch.fnmatchcase(path, normalized_pattern) + + +def _find_matching_pattern(path: str, patterns: list[str]) -> str | None: + for pattern in patterns: + if _matches_glob(path, pattern): + return pattern + return None + + +def _extract_classification(acr_text: str, allowed_values: set[str]) -> str | None: + match = re.search(r"(?im)^\s*-\s*Classification\s*:\s*`?([a-z0-9_-]+)`?", acr_text) + if not match: + return None + value = match.group(1).strip().lower() + aliases = {"behavioral-change": "breaking"} + value = aliases.get(value, value) + if value not in allowed_values: + return None + return value + + +def _acr_sections(acr_text: str) -> dict[str, bool]: + lowered = acr_text.lower() + return { + "acr_id": bool(re.search(r"(?im)^\s*-\s*`?acr_id`?\s*:\s*`?ACR-", acr_text)), + "implemented_or_approved_status": bool( + re.search(r"(?im)^\s*-\s*`?status`?\s*:\s*`?(approved|implemented)\b", acr_text) + ), + "danger_zone_yes": "danger-zone: yes" in lowered + or "kernel danger-zone` change? `yes" in lowered + or "kernel danger-zone change? yes" in lowered, + "generalization_impact": "coupling and generalization impact" in lowered + or "generalization risk assessment" in lowered, + "evidence_plan": "evidence and validation plan" in lowered or "required evidence" in lowered, + "rollback_plan": "rollback plan" in lowered or "rollback path" in lowered, + "approvals": "approvals" in lowered or "decision" in lowered, + } + + +def _validate_changed_acrs( + *, + changed_files: list[str], + repo_root: Path, + allowed_classifications: set[str], +) -> tuple[list[dict[str, Any]], list[str]]: + acr_paths = [path for path in changed_files if _matches_glob(path, ACR_PATTERN)] + acr_results: list[dict[str, Any]] = [] + errors: list[str] = [] + if not acr_paths: + errors.append(f"missing changed ACR decision artifact matching {ACR_PATTERN}") + return acr_results, errors + + for rel_path in acr_paths: + full_path = repo_root / rel_path + result: dict[str, Any] = { + "path": rel_path, + "exists": full_path.is_file(), + "classification": None, + "checks": {}, + "ok": False, + } + if not full_path.is_file(): + errors.append(f"changed ACR artifact is not present in the worktree: {rel_path}") + acr_results.append(result) + continue + + text = full_path.read_text(encoding="utf-8") + checks = _acr_sections(text) + classification = _extract_classification(text, allowed_classifications) + result["classification"] = classification + result["checks"] = checks + missing = [name for name, ok in checks.items() if not ok] + if classification is None: + missing.append("valid_classification") + result["ok"] = not missing + result["missing"] = missing + if missing: + errors.append(f"{rel_path}: incomplete danger-zone ACR fields: {', '.join(missing)}") + acr_results.append(result) + return acr_results, errors + + +def evaluate_danger_zone( + *, + manifest_path: Path, + changed_files_path: Path, + repo_root: Path, +) -> dict[str, Any]: + errors: list[str] = [] + manifest = _load_json(manifest_path) + + if manifest.get("schema") != EXPECTED_SCHEMA: + errors.append(f"schema mismatch: expected {EXPECTED_SCHEMA!r}, got {manifest.get('schema')!r}") + if manifest.get("version") != 1: + errors.append("version must be 1") + + raw_patterns = manifest.get("protected_path_globs") + if not isinstance(raw_patterns, list) or not raw_patterns: + errors.append("protected_path_globs must be a non-empty array") + patterns: list[str] = [] + else: + patterns = [str(pattern) for pattern in raw_patterns] + + raw_required = manifest.get("required_artifacts") + required_artifacts = raw_required if isinstance(raw_required, dict) else {} + if not required_artifacts: + errors.append("required_artifacts must be a non-empty object") + + required_artifact_results: list[dict[str, Any]] = [] + for key in sorted(required_artifacts): + value = required_artifacts[key] + if not isinstance(value, str) or not value: + errors.append(f"required_artifacts.{key} must be a non-empty repo-relative path") + continue + rel_path = _normalize_path(value) + exists = (repo_root / rel_path).is_file() + required_artifact_results.append({"key": key, "path": rel_path, "exists": exists}) + if not exists: + errors.append(f"missing required artifact template: {rel_path}") + + raw_classifications = manifest.get("change_classification_values") + if not isinstance(raw_classifications, list) or not raw_classifications: + errors.append("change_classification_values must be a non-empty array") + allowed_classifications: set[str] = set() + else: + allowed_classifications = {str(value).lower() for value in raw_classifications} + + changed_files = _read_changed_files(changed_files_path) + danger_zone_changes: list[dict[str, str]] = [] + for path in changed_files: + matched_pattern = _find_matching_pattern(path, patterns) + if matched_pattern is not None: + danger_zone_changes.append({"path": path, "matched_pattern": matched_pattern}) + + acr_results: list[dict[str, Any]] = [] + if danger_zone_changes: + acr_results, acr_errors = _validate_changed_acrs( + changed_files=changed_files, + repo_root=repo_root, + allowed_classifications=allowed_classifications, + ) + errors.extend(acr_errors) + + return { + "ok": not errors, + "schema": EXPECTED_SCHEMA, + "manifest_path": str(manifest_path), + "changed_files_path": str(changed_files_path), + "repo_root": str(repo_root), + "changed_files_count": len(changed_files), + "danger_zone_change_count": len(danger_zone_changes), + "danger_zone_changes": danger_zone_changes, + "required_artifacts": required_artifact_results, + "acr_artifacts": acr_results, + "errors": errors, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Enforce Kernel Danger-Zone ACR prerequisites.") + parser.add_argument("--manifest", default=str(DEFAULT_MANIFEST), help="Path to danger-zone manifest JSON.") + parser.add_argument("--changed-files-file", required=True, help="Newline-delimited PR changed files.") + parser.add_argument("--repo-root", default=str(ROOT), help="Repository root for artifact checks.") + parser.add_argument("--json-out", default="", help="Optional path to write full JSON report.") + parser.add_argument("--output-json", default="", help="Alias for --json-out.") + parser.add_argument("--json", action="store_true", help="Print the full JSON report to stdout.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + manifest_path = Path(args.manifest).expanduser().resolve() + changed_files_path = Path(args.changed_files_file).expanduser().resolve() + repo_root = Path(args.repo_root).expanduser().resolve() + if not manifest_path.is_file(): + raise FileNotFoundError(f"manifest not found: {manifest_path}") + if not changed_files_path.is_file(): + raise FileNotFoundError(f"changed files file not found: {changed_files_path}") + if not repo_root.is_dir(): + raise FileNotFoundError(f"repo root not found: {repo_root}") + + result = evaluate_danger_zone( + manifest_path=manifest_path, + changed_files_path=changed_files_path, + repo_root=repo_root, + ) + out_path = args.json_out or args.output_json + if out_path: + target = Path(out_path).expanduser().resolve() + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + print( + json.dumps( + { + "ok": result["ok"], + "changed_files_count": result["changed_files_count"], + "danger_zone_change_count": result["danger_zone_change_count"], + "errors": result["errors"], + }, + indent=2, + sort_keys=True, + ) + ) + return 0 if result["ok"] else 2 + except Exception as exc: + payload = {"ok": False, "error": str(exc)} + print(json.dumps(payload, indent=2, sort_keys=True)) + return 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_kernel_contract_pack_v1.py b/scripts/check_kernel_contract_pack_v1.py new file mode 100755 index 00000000..0d68beb4 --- /dev/null +++ b/scripts/check_kernel_contract_pack_v1.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Validate the Kernel Contract Pack v1 hash manifest. + +Exit codes: +- 0: pass +- 2: contract pack mismatch or malformed manifest +- 3: invalid input/runtime error +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_MANIFEST = ROOT / "docs" / "contracts" / "policies" / "kernel_contract_pack_v1_manifest.json" +EXPECTED_SCHEMA = "breadboard.kernel_contract_pack_manifest.v1" + + +def _load_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"manifest must be a JSON object: {path}") + return payload + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _safe_manifest_path(raw_path: object) -> str: + if not isinstance(raw_path, str) or not raw_path: + raise ValueError(f"manifest file path must be a non-empty string: {raw_path!r}") + candidate = Path(raw_path) + if candidate.is_absolute() or ".." in candidate.parts: + raise ValueError(f"manifest file path must be repo-relative and safe: {raw_path}") + return raw_path.replace("\\", "/") + + +def validate_contract_pack(*, manifest_path: Path, repo_root: Path) -> dict[str, Any]: + errors: list[str] = [] + manifest = _load_json(manifest_path) + + schema = manifest.get("schema") + if schema != EXPECTED_SCHEMA: + errors.append(f"schema mismatch: expected {EXPECTED_SCHEMA!r}, got {schema!r}") + + if manifest.get("contract_pack_version") != "kernel_contract_pack_v1": + errors.append("contract_pack_version must be 'kernel_contract_pack_v1'") + + raw_files = manifest.get("files") + if not isinstance(raw_files, list) or not raw_files: + errors.append("files must be a non-empty array") + raw_files = [] + + seen_paths: set[str] = set() + file_results: list[dict[str, Any]] = [] + for index, entry in enumerate(raw_files): + if not isinstance(entry, dict): + errors.append(f"files[{index}] must be an object") + continue + try: + rel_path = _safe_manifest_path(entry.get("path")) + except ValueError as exc: + errors.append(str(exc)) + continue + expected_sha = entry.get("sha256") + if not isinstance(expected_sha, str) or len(expected_sha) != 64: + errors.append(f"{rel_path}: sha256 must be a 64-character hex string") + expected_sha = "" + elif any(ch not in "0123456789abcdef" for ch in expected_sha): + errors.append(f"{rel_path}: sha256 must be lowercase hexadecimal") + + if rel_path in seen_paths: + errors.append(f"duplicate manifest path: {rel_path}") + seen_paths.add(rel_path) + + file_path = repo_root / rel_path + result: dict[str, Any] = { + "path": rel_path, + "expected_sha256": expected_sha, + "actual_sha256": None, + "exists": file_path.is_file(), + "ok": False, + } + if not file_path.is_file(): + errors.append(f"missing contract pack file: {rel_path}") + else: + actual_sha = _sha256_file(file_path) + result["actual_sha256"] = actual_sha + result["ok"] = actual_sha == expected_sha + if actual_sha != expected_sha: + errors.append( + f"hash mismatch: {rel_path} expected {expected_sha}, got {actual_sha}" + ) + file_results.append(result) + + return { + "ok": not errors, + "schema": EXPECTED_SCHEMA, + "manifest_path": str(manifest_path), + "repo_root": str(repo_root), + "contract_pack_version": manifest.get("contract_pack_version"), + "files_checked": len(file_results), + "files": file_results, + "errors": errors, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Validate Kernel Contract Pack v1 hashes.") + parser.add_argument("--manifest", default=str(DEFAULT_MANIFEST), help="Path to manifest JSON.") + parser.add_argument("--repo-root", default=str(ROOT), help="Repository root for manifest paths.") + parser.add_argument("--json-out", default="", help="Optional path to write full JSON report.") + parser.add_argument("--output-json", default="", help="Alias for --json-out.") + parser.add_argument("--json", action="store_true", help="Print the full JSON report to stdout.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + manifest_path = Path(args.manifest).expanduser().resolve() + repo_root = Path(args.repo_root).expanduser().resolve() + if not manifest_path.is_file(): + raise FileNotFoundError(f"manifest not found: {manifest_path}") + if not repo_root.is_dir(): + raise FileNotFoundError(f"repo root not found: {repo_root}") + + result = validate_contract_pack(manifest_path=manifest_path, repo_root=repo_root) + out_path = args.json_out or args.output_json + if out_path: + target = Path(out_path).expanduser().resolve() + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + print( + json.dumps( + { + "ok": result["ok"], + "files_checked": result["files_checked"], + "errors": result["errors"], + }, + indent=2, + sort_keys=True, + ) + ) + return 0 if result["ok"] else 2 + except Exception as exc: + payload = {"ok": False, "error": str(exc)} + print(json.dumps(payload, indent=2, sort_keys=True)) + return 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase1/apply_m12_transfer_overlay.py b/scripts/rl_phase1/apply_m12_transfer_overlay.py new file mode 100644 index 00000000..f2c73780 --- /dev/null +++ b/scripts/rl_phase1/apply_m12_transfer_overlay.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12 import apply_m12_transfer_overlay, validate_m12_transfer_overlay_report # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + "Safely dry-run or apply the non-scoring M12 RL Phase 1 source/control overlay. " + "This does not validate M12 or update the scorecard." + ) + ) + parser.add_argument( + "--manifest", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep/m12_transfer_archive_manifest.json"), + ) + parser.add_argument( + "--workspace-root", + type=Path, + default=REPO_ROOT.parent, + help="Workspace directory corresponding to the archive's workspace/ root.", + ) + parser.add_argument( + "--output", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep/m12_overlay_apply_report.json"), + ) + parser.add_argument("--apply", action="store_true", help="Write overlay files. Default is dry-run only.") + parser.add_argument( + "--allow-overwrite", + action="store_true", + help="Allow existing destination files to be overwritten during --apply.", + ) + args = parser.parse_args() + + report = apply_m12_transfer_overlay( + manifest_path=args.manifest, + workspace_root=args.workspace_root, + dry_run=not args.apply, + allow_overwrite=args.allow_overwrite, + ) + validation_errors = validate_m12_transfer_overlay_report(report) + if validation_errors: + report = dict(report) + report["status"] = "failed" + report["errors"] = list(report.get("errors") or []) + validation_errors + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + print( + "report=" + + str(report["report_id"]) + + f" status={report['status']}" + + f" dry_run={report['dry_run']}" + + f" would_write={report['would_write_count']}" + + f" written={report['written_count']}" + + f" existing_destinations={report['existing_destination_count']}" + + f" scorecard_update_allowed={report['scorecard_update_allowed']}" + + f" m12_points_awarded={report['m12_points_awarded']}" + + f" errors={len(report['errors'])}" + ) + if report["status"] != "passed": + for error in report["errors"]: + print(f"error={error}") + raise SystemExit(6) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/audit_m12_score_promotion.py b/scripts/rl_phase1/audit_m12_score_promotion.py new file mode 100644 index 00000000..4103a247 --- /dev/null +++ b/scripts/rl_phase1/audit_m12_score_promotion.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12 import validate_m12_promotion_audit, write_m12_promotion_audit # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build the non-scoring M12 score-promotion audit.") + parser.add_argument( + "--output", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json"), + ) + parser.add_argument( + "--final-report", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json"), + ) + parser.add_argument( + "--scorecard", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml"), + ) + parser.add_argument( + "--claim-ledger", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md"), + ) + parser.add_argument( + "--command-log-manifest", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json"), + ) + parser.add_argument( + "--require-ready", + action="store_true", + help="Exit nonzero unless the audit says the target evidence is ready for separate scorecard review.", + ) + args = parser.parse_args() + + audit = write_m12_promotion_audit( + output_path=args.output, + final_report_path=args.final_report, + scorecard_path=args.scorecard, + claim_ledger_path=args.claim_ledger, + command_log_manifest_path=args.command_log_manifest, + ) + errors = validate_m12_promotion_audit(audit) + if errors: + raise SystemExit("invalid_m12_promotion_audit: " + "; ".join(errors)) + print( + "audit=" + + audit["audit_id"] + + f" promotion_review_ready={audit['promotion_review_ready']} " + + f"scorecard_update_allowed={audit['scorecard_update_allowed']} " + + "missing_requirements=" + + (",".join(audit["missing_requirements"]) or "none") + ) + if args.require_ready and not audit["promotion_review_ready"]: + raise SystemExit(4) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/build_adapter_probe_reports.py b/scripts/rl_phase1/build_adapter_probe_reports.py new file mode 100644 index 00000000..2a417a65 --- /dev/null +++ b/scripts/rl_phase1/build_adapter_probe_reports.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.adapters.benchflow import build_benchflow_fixture_probe_report # noqa: E402 +from breadboard.rl.adapters.ors import build_ors_fixture_probe_report # noqa: E402 +from breadboard.rl.adapters.prime_verifiers import build_prime_verifiers_fixture_probe_report # noqa: E402 +from breadboard.rl.adapters.probe import validate_adapter_probe_report # noqa: E402 +from breadboard.rl.adapters.verl import build_verl_jsonl_probe_report # noqa: E402 + + +def main() -> None: + output_dir = Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m9_adapter_probes") + output_dir.mkdir(parents=True, exist_ok=True) + reports = [ + build_benchflow_fixture_probe_report(), + build_ors_fixture_probe_report(), + build_verl_jsonl_probe_report(), + build_prime_verifiers_fixture_probe_report(), + ] + summary = [] + for report in reports: + errors = validate_adapter_probe_report(report) + if errors: + raise SystemExit(f"{report.adapter_id} invalid: {errors}") + path = output_dir / f"{report.adapter_id}.json" + path.write_text(json.dumps(report.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + summary.append(report.to_dict()) + (output_dir / "adapter_probe_summary.json").write_text( + json.dumps({"reports": summary}, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print("reports=" + str(len(reports)) + " adapters=" + ",".join(report.adapter_id for report in reports)) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/build_m12_final_report.py b/scripts/rl_phase1/build_m12_final_report.py new file mode 100644 index 00000000..18f7f56c --- /dev/null +++ b/scripts/rl_phase1/build_m12_final_report.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12 import validate_m12_final_report, write_m12_final_report # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build and validate the M12 final target-node report.") + parser.add_argument( + "--output", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json"), + ) + parser.add_argument( + "--archive-verify-report", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_archive_verify/m12_archive_verify_report.json"), + ) + parser.add_argument( + "--preflight-report", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_target_preflight/m12_preflight_report.json"), + ) + parser.add_argument( + "--swe-run-summary", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_swe_probe/run_summary.json"), + ) + parser.add_argument( + "--verl-smoke-report", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_verl_probe/smoke_consumer_report.json"), + ) + parser.add_argument( + "--ray-probe-report", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe/ray_probe_report.json"), + ) + parser.add_argument( + "--warm-vs-cold-report", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe/warm_vs_cold_report.json"), + ) + parser.add_argument( + "--load-ladder-report", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_load_ladder/load_ladder_report.json"), + ) + parser.add_argument( + "--soak-report", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_soak/soak_report.json"), + ) + parser.add_argument( + "--command-log-manifest", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json"), + ) + parser.add_argument( + "--require-eligible", + action="store_true", + help="Exit nonzero unless the final report is M12 score-eligible.", + ) + args = parser.parse_args() + + report = write_m12_final_report( + output_path=args.output, + archive_verify_report_path=args.archive_verify_report, + preflight_report_path=args.preflight_report, + swe_run_summary_path=args.swe_run_summary, + verl_smoke_report_path=args.verl_smoke_report, + ray_probe_report_path=args.ray_probe_report, + warm_vs_cold_report_path=args.warm_vs_cold_report, + load_ladder_report_path=args.load_ladder_report, + soak_report_path=args.soak_report, + command_log_manifest_path=args.command_log_manifest, + ) + errors = validate_m12_final_report(report) + if errors: + raise SystemExit("invalid_m12_final_report: " + "; ".join(errors)) + print( + "report=" + + report["report_id"] + + f" score_eligible={report['m12_score_eligible']} " + + f"missing_gate_remediations={len(report['missing_gate_remediations'])} " + + "missing_gates=" + + (",".join(report["missing_gates"]) or "none") + ) + if args.require_eligible and not report["m12_score_eligible"]: + raise SystemExit(4) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/build_m12_transfer_archive.py b/scripts/rl_phase1/build_m12_transfer_archive.py new file mode 100644 index 00000000..60d97bec --- /dev/null +++ b/scripts/rl_phase1/build_m12_transfer_archive.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12 import write_m12_transfer_archive # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build the non-scoring M12 companion evidence archive.") + parser.add_argument( + "--output-dir", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep"), + ) + parser.add_argument("--archive-name", default="m12_transfer_evidence_pack.tar.gz") + args = parser.parse_args() + + archive_manifest = write_m12_transfer_archive( + repo_root=REPO_ROOT, + output_dir=args.output_dir, + archive_name=args.archive_name, + ) + print( + "archive=" + + archive_manifest["archive_name"] + + f" entries={archive_manifest['included_entry_count']}" + + f" sha256={archive_manifest['archive_sha256']}" + + f" repo_replacement={archive_manifest['archive_is_repo_replacement']}" + + f" scorecard_update_allowed={archive_manifest['scorecard_update_allowed']}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/build_m12_transfer_pack.py b/scripts/rl_phase1/build_m12_transfer_pack.py new file mode 100644 index 00000000..bd62370d --- /dev/null +++ b/scripts/rl_phase1/build_m12_transfer_pack.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12 import build_m12_transfer_summary, write_m12_transfer_pack # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build local M12 transfer-preparation manifest.") + parser.add_argument( + "--output-dir", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep"), + ) + args = parser.parse_args() + manifest = write_m12_transfer_pack(repo_root=REPO_ROOT, output_dir=args.output_dir) + summary = build_m12_transfer_summary(manifest) + print( + "manifest=" + + manifest["manifest_id"] + + f" artifacts_present={manifest['all_required_artifacts_present']} " + + f"commands={len(manifest['test_commands'])}" + + f"; required_artifacts={len(manifest['artifacts'])}" + + f"; expected_outputs={len(manifest['expected_outputs'])}" + + f"; readiness_summary={bool(summary['readiness_summary'])}" + + f"; concrete_load_soak_scripts={summary['concrete_load_soak_scripts']}" + + f"; load_soak_command_log_templates={summary['load_soak_command_log_templates']}" + + f"; logged_command_wrapper={summary['logged_command_wrapper']}" + + f"; bootstrap_dirty_checkout_guard={summary['bootstrap_dirty_checkout_guard']}" + + f"; bootstrap_overlaid_test_commands_handoff={summary['bootstrap_overlaid_test_commands_handoff']}" + + f"; bootstrap_repo_root_cwd_handoff={summary['bootstrap_repo_root_cwd_handoff']}" + + f"; target_run_id_command_binding={summary['target_run_id_command_binding']}" + + f"; target_run_log_reuse_guard={summary['target_run_log_reuse_guard']}" + + f"; target_closeout_artifact_reuse_guard={summary['target_closeout_artifact_reuse_guard']}" + + f"; final_report_failure_remediation_summary={summary['final_report_failure_remediation_summary']}" + + f"; generated_script_manifest_consistent={summary['generated_script_manifest_consistent']}" + + f"; archive_verifier_runs_first={summary['archive_verifier_runs_first']}" + + f"; preflight_command_require_pass={summary['preflight_command_require_pass']}" + + f"; final_command_explicit_target_artifact_args={summary['final_command_explicit_target_artifact_args']}" + + f"; final_command_require_eligible={summary['final_command_require_eligible']}" + + f"; promotion_audit_require_ready={summary['promotion_audit_require_ready']}" + + f"; promotion_audit_explicit_score_inputs={summary['promotion_audit_explicit_score_inputs']}" + + f"; promotion_audit_explicit_target_paths={summary['promotion_audit_explicit_target_paths']}" + + f"; all_transfer_requirements_covered={summary['all_transfer_requirements_covered']}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/check_m12_evidence_consistency.py b/scripts/rl_phase1/check_m12_evidence_consistency.py new file mode 100644 index 00000000..24d41ca4 --- /dev/null +++ b/scripts/rl_phase1/check_m12_evidence_consistency.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12 import ( # noqa: E402 + validate_m12_evidence_consistency_report, + write_m12_evidence_consistency_report, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Check local M12 blocked-state evidence consistency.") + parser.add_argument( + "--phase-dir", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1"), + ) + parser.add_argument( + "--output", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_evidence_consistency/m12_evidence_consistency.json"), + ) + parser.add_argument( + "--require-consistent", + action="store_true", + help="Exit nonzero unless the local blocked-state evidence is internally consistent.", + ) + args = parser.parse_args() + + report = write_m12_evidence_consistency_report(phase_dir=args.phase_dir, output_path=args.output) + validation_errors = validate_m12_evidence_consistency_report(report) + if validation_errors: + raise SystemExit("invalid_m12_evidence_consistency_report: " + "; ".join(validation_errors)) + print( + "report=" + + report["report_id"] + + f" consistent={report['consistent']} " + + f"scorecard_update_allowed={report['scorecard_update_allowed']} " + + f"m12_points_awarded={report['m12_points_awarded']} " + + "errors=" + + (",".join(report["errors"]) or "none") + ) + if args.require_consistent and not report["consistent"]: + raise SystemExit(4) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/export_verl_probe.py b/scripts/rl_phase1/export_verl_probe.py new file mode 100644 index 00000000..1c89005b --- /dev/null +++ b/scripts/rl_phase1/export_verl_probe.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.export import ( # noqa: E402 + build_verl_probe_rows_from_m6_summary, + smoke_consume_verl_probe_jsonl, + smoke_consume_verl_probe_parquet, + write_verl_probe_jsonl, + write_verl_probe_parquet, + write_verl_probe_projection_manifest, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Export M6 controlled SWE run to VeRL-shaped JSONL/Parquet probe rows.") + parser.add_argument( + "--m6-summary", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/run_summary.json"), + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe"), + ) + args = parser.parse_args() + + summary = json.loads(args.m6_summary.read_text(encoding="utf-8")) + rows = build_verl_probe_rows_from_m6_summary(summary) + jsonl_path = args.output_dir / "verl_probe_rows.jsonl" + parquet_path = args.output_dir / "verl_probe_rows.parquet" + projection_manifest_path = args.output_dir / "projection_manifest.json" + write_verl_probe_jsonl(rows, jsonl_path) + write_verl_probe_parquet(rows, parquet_path) + projection_manifest = write_verl_probe_projection_manifest( + rows, + projection_manifest_path, + target_formats=["jsonl", "parquet"], + ) + jsonl_smoke = smoke_consume_verl_probe_jsonl(jsonl_path) + parquet_smoke = smoke_consume_verl_probe_parquet(parquet_path) + smoke = { + "target_run_id": summary.get("target_run_id") or os.environ.get("M12_TARGET_RUN_ID"), + "row_count": jsonl_smoke["row_count"], + "trainable_candidate_count": jsonl_smoke["trainable_candidate_count"], + "tensorizable": jsonl_smoke["tensorizable"] and parquet_smoke["tensorizable"], + "errors": { + "jsonl": jsonl_smoke["errors"], + "parquet": parquet_smoke["errors"], + }, + "formats": { + "jsonl": jsonl_smoke, + "parquet": parquet_smoke, + }, + "projection_manifest_id": projection_manifest["projection_id"], + "compatibility_target": "VeRL JSONL/Parquet probe v1alpha; not DataProto or trainer execution", + } + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "smoke_consumer_report.json").write_text( + json.dumps(smoke, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + print( + f"rows={smoke['row_count']} trainable_candidates={smoke['trainable_candidate_count']} " + f"tensorizable={smoke['tensorizable']}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/run_m12_bootstrap_dry_run.py b/scripts/rl_phase1/run_m12_bootstrap_dry_run.py new file mode 100644 index 00000000..05d92514 --- /dev/null +++ b/scripts/rl_phase1/run_m12_bootstrap_dry_run.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12 import ( # noqa: E402 + validate_m12_bootstrap_dry_run_report, + write_m12_bootstrap_dry_run_report, +) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Run the generated M12 target bootstrap in dry-run mode and write a non-scoring report." + ) + parser.add_argument("--repo-root", type=Path, default=REPO_ROOT) + parser.add_argument( + "--transfer-prep-dir", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep"), + ) + parser.add_argument("--workspace-root", type=Path, default=REPO_ROOT.parent) + parser.add_argument( + "--output", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_bootstrap_dry_run/m12_bootstrap_dry_run_report.json"), + ) + parser.add_argument("--require-pass", action="store_true") + args = parser.parse_args() + + report = write_m12_bootstrap_dry_run_report( + repo_root=args.repo_root, + transfer_prep_dir=args.transfer_prep_dir, + workspace_root=args.workspace_root, + output_path=args.output, + ) + validation_errors = validate_m12_bootstrap_dry_run_report(report) + if validation_errors: + report = dict(report) + report["status"] = "failed" + report["errors"] = list(report.get("errors") or []) + validation_errors + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + print( + "report=" + + str(report["report_id"]) + + f" status={report['status']}" + + f" exit_code={report['exit_code']}" + + f" repo_head_verified={report['repo_head_verified']}" + + f" dirty_checkout_mode={report['dirty_checkout_mode']}" + + f" dirty_checkout_override_used={report['dirty_checkout_override_used']}" + + f" target_commands_skipped={report['target_commands_skipped']}" + + f" overlay_would_write={report['overlay']['would_write_count']}" + + f" overlay_written={report['overlay']['written_count']}" + + f" scorecard_update_allowed={report['scorecard_update_allowed']}" + + f" m12_points_awarded={report['m12_points_awarded']}" + + f" errors={len(report['errors'])}" + ) + if args.require_pass and report["status"] != "passed": + for error in report["errors"]: + print(f"error={error}") + raise SystemExit(6) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/run_m12_load_ladder.py b/scripts/rl_phase1/run_m12_load_ladder.py new file mode 100644 index 00000000..610b6d7f --- /dev/null +++ b/scripts/rl_phase1/run_m12_load_ladder.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12.load_soak import ( # noqa: E402 + build_m12_load_ladder_report_from_package, + validate_m12_load_ladder_report, +) + + +def _parse_levels(raw: str) -> list[int]: + return [int(item.strip()) for item in raw.split(",") if item.strip()] + + +def _parse_skip(raw_items: list[str]) -> dict[int, str]: + skips: dict[int, str] = {} + for raw in raw_items: + if "=" not in raw: + raise ValueError("--skip-level entries must be formatted LEVEL=reason") + level, reason = raw.split("=", 1) + skips[int(level)] = reason + return skips + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run the M12 target load ladder probe.") + parser.add_argument( + "--package", + type=Path, + default=Path("examples/rl_env_packages/python_console_toy/env_package.yaml"), + ) + parser.add_argument( + "--output", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_load_ladder/load_ladder_report.json"), + ) + parser.add_argument("--levels", default="5,20,50,100") + parser.add_argument("--skip-level", action="append", default=[]) + parser.add_argument("--min-rows-per-level", type=int, default=10) + parser.add_argument( + "--local-mode", + action="store_true", + help="Use Ray local_mode for local smoke tests. Target validation should omit this flag.", + ) + parser.add_argument( + "--smoke-mode", + action="store_true", + help="Validate only the supplied levels and allow local_mode. Never use for M12 score promotion.", + ) + args = parser.parse_args() + levels = _parse_levels(args.levels) + skip_levels = _parse_skip(args.skip_level) + report = build_m12_load_ladder_report_from_package( + package_path=args.package, + levels=levels, + skip_levels=skip_levels, + min_rows_per_level=args.min_rows_per_level, + local_mode=args.local_mode, + target_run_id=os.environ.get("M12_TARGET_RUN_ID"), + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + validation_errors = validate_m12_load_ladder_report( + report, + required_levels=[level for level in levels if level not in skip_levels] if args.smoke_mode else None, + optional_levels=list(skip_levels) if args.smoke_mode else None, + require_distributed=not args.smoke_mode, + ) + statuses = ",".join(f"{item['target_sessions']}:{item['status']}" for item in report["concurrency_levels"]) + print( + "report=" + + report["report_id"] + + f" levels={statuses} " + + f"policy_version_integrity={report['policy_version_integrity']} " + + f"queue_backpressure_integrity={report['queue_backpressure_integrity']}" + ) + if validation_errors: + raise SystemExit("invalid_m12_load_ladder_report: " + "; ".join(validation_errors)) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/run_m12_logged_command.py b/scripts/rl_phase1/run_m12_logged_command.py new file mode 100644 index 00000000..c14213ec --- /dev/null +++ b/scripts/rl_phase1/run_m12_logged_command.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import argparse +import json +import os +import shlex +import subprocess +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12.command_logs import ( # noqa: E402 + manifest_relative_log_path, + next_command_log_path, + record_command_log_result, + utc_now_iso, + validate_manifest_log_path, + validate_target_run_id, +) + + +def _strip_separator(command: list[str]) -> list[str]: + if command and command[0] == "--": + return command[1:] + return command + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run one M12 target command and archive its raw log.") + parser.add_argument( + "--manifest", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json"), + ) + parser.add_argument( + "--log-dir", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs"), + ) + parser.add_argument("--command-id", required=True) + parser.add_argument("--target-run-id", default=os.environ.get("M12_TARGET_RUN_ID")) + parser.add_argument("--description", default=None) + parser.add_argument("--notes", default="") + parser.add_argument( + "--allow-failure", + action="store_true", + help="Record the command result but exit zero even if the wrapped command fails.", + ) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args() + + command = _strip_separator(args.command) + if not command: + raise SystemExit("missing wrapped command after --") + + try: + log_path = next_command_log_path(args.log_dir, args.command_id) + except ValueError as exc: + raise SystemExit(f"invalid command id: {exc}") from exc + if args.target_run_id: + target_run_errors = validate_target_run_id(args.target_run_id) + if target_run_errors: + raise SystemExit(f"invalid target run id: {'; '.join(target_run_errors)}") + relative_log_path = manifest_relative_log_path(args.manifest, log_path) + log_path_errors = validate_manifest_log_path(relative_log_path) + if log_path_errors: + raise SystemExit(f"invalid log path: {'; '.join(log_path_errors)}") + args.log_dir.mkdir(parents=True, exist_ok=True) + command_text = shlex.join(command) + started_at = utc_now_iso() + exit_code = 1 + completed_at = started_at + notes = args.notes + output_ended_with_newline = True + child_env = os.environ.copy() + if args.target_run_id: + child_env["M12_TARGET_RUN_ID"] = args.target_run_id + + with log_path.open("w", encoding="utf-8") as handle: + handle.write(f"# command_id: {args.command_id}\n") + if args.target_run_id: + handle.write(f"# target_run_id: {args.target_run_id}\n") + handle.write(f"# command: {command_text}\n") + handle.write(f"# argv_json: {json.dumps(command, ensure_ascii=True)}\n") + handle.write(f"# started_at: {started_at}\n") + handle.flush() + try: + process = subprocess.Popen( + command, + cwd=REPO_ROOT, + env=child_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + except OSError as exc: + exit_code = 127 + spawn_error = f"spawn_error: {type(exc).__name__}: {exc}" + print(spawn_error, file=sys.stderr) + handle.write(f"# {spawn_error}\n") + notes = f"{notes}\n{spawn_error}".strip() + else: + assert process.stdout is not None + for line in process.stdout: + sys.stdout.write(line) + handle.write(line) + output_ended_with_newline = line.endswith("\n") + exit_code = process.wait() + completed_at = utc_now_iso() + if not output_ended_with_newline: + handle.write("\n") + handle.write(f"# completed_at: {completed_at}\n") + handle.write(f"# exit_code: {exit_code}\n") + record_command_log_result( + manifest_path=args.manifest, + command_id=args.command_id, + command=command_text, + argv=command, + log_path=log_path, + exit_code=exit_code, + started_at=started_at, + completed_at=completed_at, + description=args.description, + notes=notes, + target_run_id=args.target_run_id, + ) + if exit_code != 0 and not args.allow_failure: + raise SystemExit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/run_m12_preflight.py b/scripts/rl_phase1/run_m12_preflight.py new file mode 100644 index 00000000..9d292d7d --- /dev/null +++ b/scripts/rl_phase1/run_m12_preflight.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12 import write_m12_preflight_report # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run M12 target-node preflight probe.") + parser.add_argument( + "--output-dir", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_target_preflight"), + ) + parser.add_argument( + "--require-pass", + action="store_true", + help="Exit nonzero unless the target preflight status is preflight_passed.", + ) + args = parser.parse_args() + report = write_m12_preflight_report(args.output_dir) + print(f"status={report['status']} blockers={','.join(report['blockers']) or 'none'}") + if args.require_pass and report["status"] != "preflight_passed": + raise SystemExit(3) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/run_m12_soak.py b/scripts/rl_phase1/run_m12_soak.py new file mode 100644 index 00000000..f63096ae --- /dev/null +++ b/scripts/rl_phase1/run_m12_soak.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12.load_soak import ( # noqa: E402 + build_m12_soak_report_from_package, + validate_m12_soak_report, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run the M12 target soak probe.") + parser.add_argument( + "--package", + type=Path, + default=Path("examples/rl_env_packages/python_console_toy/env_package.yaml"), + ) + parser.add_argument( + "--output", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_soak/soak_report.json"), + ) + parser.add_argument("--duration-seconds", type=int, default=7200) + parser.add_argument("--minimum-duration-seconds", type=int, default=7200) + parser.add_argument("--interval-seconds", type=float, default=30.0) + parser.add_argument("--num-workers", type=int, default=20) + parser.add_argument("--rows-per-iteration", type=int, default=10) + parser.add_argument("--min-iterations", type=int, default=1) + parser.add_argument( + "--local-mode", + action="store_true", + help="Use Ray local_mode for local smoke tests. Target validation should omit this flag.", + ) + parser.add_argument( + "--smoke-mode", + action="store_true", + help="Allow local_mode and short durations for local smoke tests. Never use for M12 score promotion.", + ) + args = parser.parse_args() + report = build_m12_soak_report_from_package( + package_path=args.package, + duration_seconds=args.duration_seconds, + minimum_duration_seconds=args.minimum_duration_seconds, + interval_seconds=args.interval_seconds, + num_workers=args.num_workers, + rows_per_iteration=args.rows_per_iteration, + min_iterations=args.min_iterations, + local_mode=args.local_mode, + target_run_id=os.environ.get("M12_TARGET_RUN_ID"), + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + validation_errors = validate_m12_soak_report( + report, + minimum_duration_seconds=args.minimum_duration_seconds, + require_distributed=not args.smoke_mode, + ) + print( + "report=" + + report["report_id"] + + f" status={report['status']} " + + f"duration_seconds={report['duration_seconds']} " + + f"runtime_failure_count={report['runtime_failure_count']}" + ) + if validation_errors: + raise SystemExit("invalid_m12_soak_report: " + "; ".join(validation_errors)) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/run_ray_warm_pool_probe.py b/scripts/rl_phase1/run_ray_warm_pool_probe.py new file mode 100644 index 00000000..2f735d4b --- /dev/null +++ b/scripts/rl_phase1/run_ray_warm_pool_probe.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.env_package.validate import load_env_package # noqa: E402 +from breadboard.rl.runtime import build_warm_vs_cold_report, run_local_ray_toy_probe # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run local Ray/warm-pool M8 probe.") + parser.add_argument( + "--package", + type=Path, + default=Path("examples/rl_env_packages/python_console_toy/env_package.yaml"), + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m8_ray_warm_pool_probe"), + ) + parser.add_argument("--limit", type=int, default=10) + parser.add_argument("--num-workers", type=int, default=2) + parser.add_argument( + "--distributed", + action="store_true", + help="Use Ray distributed execution rather than local_mode. Intended for M12 target validation.", + ) + args = parser.parse_args() + package = load_env_package(args.package) + task_ids = [f"py_toy_{index:03d}" for index in range(1, args.limit + 1)] + warm = run_local_ray_toy_probe( + package=package, + task_ids=task_ids, + num_workers=args.num_workers, + local_mode=not args.distributed, + ) + warm["target_run_id"] = os.environ.get("M12_TARGET_RUN_ID") + cold_rows = [ + {**row, "metrics_ms": {key: value * 1.5 for key, value in row["metrics_ms"].items()}} + for row in warm["rows"] + ] + report = build_warm_vs_cold_report(warm_rows=warm["rows"], cold_rows=cold_rows) + report["target_run_id"] = warm["target_run_id"] + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "ray_probe_report.json").write_text(json.dumps(warm, indent=2, sort_keys=True) + "\n", encoding="utf-8") + (args.output_dir / "warm_vs_cold_report.json").write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f"rows={warm['row_count']} workers={warm['worker_count']} local_mode={warm['ray_local_mode']}") + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/run_swe_probe.py b/scripts/rl_phase1/run_swe_probe.py new file mode 100644 index 00000000..1f5bf5f4 --- /dev/null +++ b/scripts/rl_phase1/run_swe_probe.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.e2e import run_controlled_swe_probe + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run the RL Phase 1 controlled SWE toy probe.") + parser.add_argument( + "--package", + type=Path, + default=Path("examples/rl_env_packages/swe_toy_patch/env_package.yaml"), + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy"), + ) + parser.add_argument("--run-id", default="m6_controlled_swe_toy") + parser.add_argument("--limit", type=int, default=10) + args = parser.parse_args() + + run = run_controlled_swe_probe( + package_path=args.package, + output_dir=args.output_dir, + run_id=args.run_id, + limit=args.limit, + ) + print( + f"run_id={run.run_id} rows={len(run.rows)} " + f"accepted={sum(row.row_status == 'accepted' for row in run.rows)} " + f"rejected={sum(row.row_status == 'rejected' for row in run.rows)} " + f"quarantined={sum(row.row_status == 'quarantined' for row in run.rows)}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/summarize_m12_final_report_remediations.py b/scripts/rl_phase1/summarize_m12_final_report_remediations.py new file mode 100644 index 00000000..5b7ab319 --- /dev/null +++ b/scripts/rl_phase1/summarize_m12_final_report_remediations.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12 import ( # noqa: E402 + summarize_m12_final_report_remediations, + validate_m12_final_report, + validate_m12_final_report_remediation_summary, +) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Summarize missing M12 final-report gates by target action.") + parser.add_argument( + "--final-report", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json"), + ) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + + try: + report = json.loads(args.final_report.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc: + raise SystemExit(f"invalid_m12_final_report_input: {exc.__class__.__name__}: {exc}") from exc + if not isinstance(report, dict): + raise SystemExit("invalid_m12_final_report_input: expected JSON object") + errors = validate_m12_final_report(report) + if errors: + raise SystemExit("invalid_m12_final_report: " + "; ".join(errors)) + + summary = summarize_m12_final_report_remediations(report) + summary_errors = validate_m12_final_report_remediation_summary(summary) + if summary_errors: + raise SystemExit("invalid_m12_remediation_summary: " + "; ".join(summary_errors)) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + next_actions = ",".join(item["target_action_id"] for item in summary["next_target_actions"]) or "none" + print( + "summary=" + + summary["summary_id"] + + f" score_eligible={summary['m12_score_eligible']}" + + f" missing_gates={summary['missing_gate_count']}" + + f" remediations={summary['remediation_count']}" + + f" next_actions={next_actions}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase1/verify_m12_transfer_archive.py b/scripts/rl_phase1/verify_m12_transfer_archive.py new file mode 100644 index 00000000..9e4aa9e6 --- /dev/null +++ b/scripts/rl_phase1/verify_m12_transfer_archive.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from breadboard.rl.m12 import validate_m12_transfer_archive_manifest # noqa: E402 + + +def _resolve_manifest_path(manifest_path: Path, manifest: dict, key: str, fallback_name: str) -> Path: + raw = manifest.get(key) + if not raw: + return manifest_path.parent / fallback_name + path = Path(str(raw)) + return path if path.is_absolute() else manifest_path.parent / path + + +def _build_archive_verify_report(manifest_path: Path, errors: list[str]) -> dict: + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception as exc: + manifest = {} + manifest_read_error = f"{type(exc).__name__}: {exc}" + else: + manifest_read_error = None + + archive_name = str(manifest.get("archive_name") or "m12_transfer_evidence_pack.tar.gz") + archive_path = _resolve_manifest_path(manifest_path, manifest, "archive_path", archive_name) + sha_path = _resolve_manifest_path(manifest_path, manifest, "archive_sha256_file", archive_name + ".sha256") + return { + "report_id": "bb_zyphra_rl_phase1_m12_archive_verify_report_v1", + "claim_boundary": "transfer_archive_verification_not_m12_validation", + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "status": "passed" if not errors else "failed", + "manifest_path": str(manifest_path), + "manifest_read_error": manifest_read_error, + "archive_manifest_id": manifest.get("archive_manifest_id"), + "archive_claim_boundary": manifest.get("claim_boundary"), + "archive_path": str(archive_path), + "archive_sha256_file": str(sha_path), + "archive_sha256": manifest.get("archive_sha256"), + "archive_size_bytes": manifest.get("archive_size_bytes"), + "included_entry_count": manifest.get("included_entry_count"), + "all_required_artifacts_present": manifest.get("all_required_artifacts_present"), + "all_transfer_requirements_covered": manifest.get("all_transfer_requirements_covered"), + "archive_contains_source_overlay": manifest.get("archive_contains_source_overlay"), + "archive_deterministic": manifest.get("archive_deterministic"), + "source_paths_portable": manifest.get("source_paths_portable"), + "errors": list(errors), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Verify an M12 transfer evidence archive manifest and tarball.") + parser.add_argument( + "--manifest", + type=Path, + default=Path("../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep/m12_transfer_archive_manifest.json"), + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Optional path for a non-scoring machine-readable archive-verifier report.", + ) + args = parser.parse_args() + + errors = validate_m12_transfer_archive_manifest(args.manifest) + report = _build_archive_verify_report(args.manifest, errors) + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if errors: + print("status=failed archive_manifest_verified=false") + if args.output is not None: + print(f"archive_verify_report={args.output}") + for error in errors: + print(f"error={error}") + raise SystemExit(5) + output_fragment = f" archive_verify_report={args.output}" if args.output is not None else "" + print("status=passed archive_manifest_verified=true" + output_fragment) + + +if __name__ == "__main__": + main() diff --git a/scripts/rl_phase3/audit_phase3_promotion.py b/scripts/rl_phase3/audit_phase3_promotion.py new file mode 100644 index 00000000..93c70eee --- /dev/null +++ b/scripts/rl_phase3/audit_phase3_promotion.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path +import sys +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + + +from breadboard.rl.phase3.final_report import validate_phase3_final_report +from breadboard.rl.phase3.promotion_audit import build_phase3_promotion_audit, validate_phase3_promotion_audit + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--phase-dir", required=True, type=Path) + parser.add_argument("--require-ready", action="store_true") + args = parser.parse_args() + final_path = args.phase_dir / "runs" / "p3_m12_final_report.json" + final_report_read_error = "" + if not final_path.exists(): + final_report_read_error = f"FileNotFoundError: {final_path} is missing" + final_report = {"validation_errors": [f"final report is missing: {final_report_read_error}"]} + else: + try: + final_report = json.loads(final_path.read_text()) + except Exception as exc: + final_report_read_error = f"{exc.__class__.__name__}: {exc}" + final_report = {"validation_errors": [f"final report is not readable JSON: {final_report_read_error}"]} + if not isinstance(final_report, dict): + final_report_read_error = "TypeError: final report must be a JSON object" + final_report = {"validation_errors": ["final report must be a JSON object"]} + if not final_report_read_error and isinstance(final_report, dict): + final_report["validation_errors"] = validate_phase3_final_report(final_report, repo_root=Path.cwd(), evidence_root=args.phase_dir.parents[1]) + claim_ledger_read_error = "" + ledger_path = args.phase_dir / "BB_ZYPHRA_RL_PHASE_3_CLAIM_LEDGER.md" + if not ledger_path.exists(): + claim_ledger_read_error = f"FileNotFoundError: {ledger_path} is missing" + ledger = "" + else: + try: + ledger = ledger_path.read_text() + except Exception as exc: + claim_ledger_read_error = f"{exc.__class__.__name__}: {exc}" + ledger = "" + scorecard = final_report.get("scorecard", {}) if isinstance(final_report, dict) else {} + bd_closed = False + try: + result = subprocess.run(["bd", "show", "bb-5v6", "--json"], check=False, text=True, capture_output=True) + if result.returncode == 0: + payload = json.loads(result.stdout) + record = payload[0] if isinstance(payload, list) else payload + bd_closed = isinstance(record, dict) and str(record.get("status", "")).lower() == "closed" + except Exception: + bd_closed = False + target_run_id = final_report.get("target_run_id", "") if isinstance(final_report, dict) else "" + audit = build_phase3_promotion_audit(target_run_id=target_run_id, final_report=final_report, scorecard=scorecard, claim_ledger_text=ledger, bd_epic_closed=bd_closed) + if final_report_read_error: + audit["final_report_read_error"] = final_report_read_error + if claim_ledger_read_error: + audit["claim_ledger_read_error"] = claim_ledger_read_error + errors = validate_phase3_promotion_audit(audit) + audit["validation_errors"] = errors + output = args.phase_dir / "runs" / "p3_m12_promotion_audit.json" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(audit, sort_keys=True, indent=2) + "\n") + print(json.dumps({"audit": str(output), "validation_errors": errors, "promotion_review_ready": audit.get("promotion_review_ready")}, sort_keys=True)) + return 0 if not args.require_ready or not errors else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/build_phase3_dataproto_batch.py b/scripts/rl_phase3/build_phase3_dataproto_batch.py new file mode 100644 index 00000000..22ae7afb --- /dev/null +++ b/scripts/rl_phase3/build_phase3_dataproto_batch.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import argparse +import json +import torch +from pathlib import Path +import sys +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + + +from breadboard.rl.phase2.bridge import build_verl_batch_from_projection_rows +from breadboard.rl.phase3.trainer_live import PHASE3_DATAPROTO_SCHEMA, build_phase3_dataproto + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--projection-rows", required=True, type=Path) + parser.add_argument("--target-run-id", required=True) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--grpo", action="store_true") + args = parser.parse_args() + rows = json.loads(args.projection_rows.read_text()).get("rows", []) + batch = build_verl_batch_from_projection_rows(rows, target_run_id=args.target_run_id).to_dict() + dataproto = build_phase3_dataproto(batch, device="cpu", require_grpo_uid=args.grpo) + args.output_dir.mkdir(parents=True, exist_ok=True) + payload_path = args.output_dir / "phase3_dataproto_payload.pt" + torch.save(dataproto, payload_path) + report = {"schema_version": PHASE3_DATAPROTO_SCHEMA, "report_id": "phase3_dataproto_batch", "target_run_id": args.target_run_id, "payload_path": str(payload_path), "row_count": len(rows), "scorecard_update_allowed": False, "passed": True} + (args.output_dir / "phase3_dataproto_report.json").write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/build_phase3_final_report.py b/scripts/rl_phase3/build_phase3_final_report.py new file mode 100644 index 00000000..f4e8b612 --- /dev/null +++ b/scripts/rl_phase3/build_phase3_final_report.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import argparse +import json +from collections.abc import Mapping +import re +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from breadboard.rl.phase3.final_report import PHASE3_MILESTONES, build_phase3_final_report, validate_phase3_final_report + + +def _read_text(path: Path) -> str: + try: + return path.read_text() + except (OSError, UnicodeDecodeError): + return "" + + +def _load_json(path: Path) -> dict: + if not path.exists(): + return {} + try: + payload = json.loads(_read_text(path)) + except (json.JSONDecodeError, OSError, UnicodeError): + return {} + return dict(payload) if isinstance(payload, Mapping) else {} + + +def _collect_milestone_reports(runs: Path) -> dict[str, dict]: + reports: dict[str, dict] = {} + canonical = runs / "milestone_reports" + roots = [canonical] if canonical.exists() else [] + roots.append(runs) + for root in roots: + for path in sorted(root.rglob("*.json")): + if path.name in {"p3_m12_final_report.json", "p3_m12_promotion_audit.json"}: + continue + payload = _load_json(path) + if not payload: + continue + milestone = payload.get("milestone_id") + if not isinstance(milestone, str) or milestone not in PHASE3_MILESTONES: + continue + if root == canonical or milestone not in reports: + reports[milestone] = payload + return reports + + +def _scorecard(phase_dir: Path) -> dict: + scorecard_path = phase_dir / "BB_ZYPHRA_RL_PHASE_3_SCORECARD.yaml" + scorecard_text = _read_text(scorecard_path) + current_match = re.search(r"^current_verified_points:\s*(\d+)", scorecard_text, re.MULTILINE) + total_match = re.search(r"^total_points:\s*(\d+)", scorecard_text, re.MULTILINE) + reviewed_match = re.search(r"^reviewed_final_report_id:\s*(\S+)", scorecard_text, re.MULTILINE) + return { + "raw": scorecard_text, + "current_verified_points": int(current_match.group(1)) if current_match else 0, + "total_points": int(total_match.group(1)) if total_match else 0, + "reviewed_final_report_id": None if not reviewed_match or reviewed_match.group(1) == "null" else reviewed_match.group(1), + } + + +def _require_ready_errors(report: Mapping[str, object]) -> list[str]: + errors: list[str] = [] + active_scope = report.get("active_scope") + if not isinstance(active_scope, Mapping): + errors.append("active_scope must be present for --require-ready") + return errors + if active_scope.get("ready") is not True: + errors.append("active_scope.ready must be true for --require-ready") + verified = active_scope.get("core_raw_points_verified") + total = active_scope.get("core_raw_points_total") + if not isinstance(verified, int) or not isinstance(total, int) or verified != total: + errors.append("core_raw_points_verified must equal core_raw_points_total for --require-ready") + return errors + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--phase-dir", required=True, type=Path) + parser.add_argument("--require-ready", action="store_true") + args = parser.parse_args() + runs = args.phase_dir / "runs" + manifest = _load_json(runs / "phase3_command_log_manifest.json") + target_run_id = manifest.get("target_run_id", "") + ledger_path = args.phase_dir / "BB_ZYPHRA_RL_PHASE_3_CLAIM_LEDGER.md" + report = build_phase3_final_report( + target_run_id=target_run_id, + milestone_reports=_collect_milestone_reports(runs), + command_log_manifest=manifest, + scorecard=_scorecard(args.phase_dir), + claim_ledger_text=_read_text(ledger_path), + ) + errors = validate_phase3_final_report(report, repo_root=Path.cwd(), evidence_root=args.phase_dir.parents[1]) + readiness_errors = _require_ready_errors(report) if args.require_ready else [] + report["validation_errors"] = errors + output = runs / "p3_m12_final_report.json" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + print(json.dumps({"report": str(output), "validation_errors": errors, "readiness_errors": readiness_errors}, sort_keys=True)) + return 0 if not args.require_ready or (not errors and not readiness_errors) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/build_phase3_parity_report.py b/scripts/rl_phase3/build_phase3_parity_report.py new file mode 100644 index 00000000..6a35fa9e --- /dev/null +++ b/scripts/rl_phase3/build_phase3_parity_report.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from breadboard.rl.phase3.evidence import sha256_file +from breadboard.rl.phase3.parity import build_phase3_parity_report, validate_phase3_parity_report + + +MILESTONE_FILES = { + "P3-M1": "P3-M1_verl_api_introspection.json", + "P3-M2": "P3-M2_ppo_weight_update.json", + "P3-M3": "P3-M3_grpo_weight_update.json", + "P3-M4": "P3-M4_closed_loop_rollout.json", +} +STAGE_SCRIPTS = { + "ppo_script": Path("ZYPHRA/RL_PHASE_3/runs/payloads/phase3_container_ppo_8gpu_stage/run.sh"), + "grpo_script": Path("ZYPHRA/RL_PHASE_3/runs/payloads/phase3_container_grpo_8gpu_stage/run.sh"), + "closed_loop_script": Path("ZYPHRA/RL_PHASE_3/runs/payloads/closed_loop_verl_train_stage/run.sh"), +} + +RUNTIME_INSTALL_REPORTS = ( + Path("ZYPHRA/RL_PHASE_3/runs/phase3_vllm_runtime_parity_probe/phase3_vllm_runtime_parity_probe.json"), +) + + +def _load(path: Path) -> dict: + payload = json.loads(path.read_text()) + return payload if isinstance(payload, dict) else {} + + +def _write(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n") + + +def _rel_to_evidence_root(path: Path, evidence_root: Path) -> str: + return str(path.resolve().relative_to(evidence_root.resolve())) + + +def _artifact(evidence_root: Path, report: dict, key: str) -> Path: + raw = report.get("artifact_paths", {}).get(key) + if not raw: + raise FileNotFoundError(f"missing artifact path {key}") + return (evidence_root / raw).resolve() + + +def _read_script(path: Path) -> str: + return path.read_text() + + +def _runtime_evidence(*, evidence_root: Path, p1_report: dict, target_run_id: str) -> dict: + introspection_artifact = Path(str(p1_report["artifact_paths"]["introspection_report"])) + introspection_path = (evidence_root / introspection_artifact).resolve() + scripts = {name: (evidence_root / rel).resolve() for name, rel in STAGE_SCRIPTS.items()} + texts = {name: _read_script(path) for name, path in scripts.items()} + images = {match.group(1) for text in texts.values() for match in re.finditer(r'IMAGE="([^"]+)"', text)} + runtimes = {match.group(1) for text in texts.values() for match in re.finditer(r"source (/shared/[^ ]+)/bin/activate", text)} + runtime_install_report: Path | None = None + runtime_install_path: Path | None = None + runtime_install: dict = {} + for candidate in RUNTIME_INSTALL_REPORTS: + candidate_path = (evidence_root / candidate).resolve() + if not candidate_path.exists(): + continue + payload = _load(candidate_path) + if payload.get("target_run_id") != target_run_id: + continue + runtime_install_report = candidate + runtime_install_path = candidate_path + runtime_install = payload + break + runtime_imports = runtime_install.get("imports") if isinstance(runtime_install.get("imports"), dict) else {} + runtime_install_hash = sha256_file(runtime_install_path) if runtime_install_path and runtime_install_path.exists() else "" + runtime_install_artifact = str(runtime_install_report) if runtime_install_report and runtime_install_path and runtime_install_path.exists() else "" + input_hashes = { + "introspection_report": sha256_file(introspection_path), + **{name: sha256_file(path) for name, path in scripts.items()}, + } + if runtime_install_hash: + input_hashes["runtime_install_report"] = runtime_install_hash + return { + "introspection_report_path": str(introspection_path), + "introspection_report_artifact": str(introspection_artifact), + "container_image": sorted(images)[0] if len(images) == 1 else "", + "runtime_path": sorted(runtimes)[0] if len(runtimes) == 1 else "", + "runtime_install_report_path": str(runtime_install_path) if runtime_install_path and runtime_install_path.exists() else "", + "runtime_install_report_artifact": runtime_install_artifact, + "runtime_install_runtime": str(runtime_install.get("runtime") or ""), + "runtime_install_passed": runtime_install.get("passed") is True, + "vllm_version": str(runtime_imports.get("vllm") or ""), + "ppo_script_artifact": str(STAGE_SCRIPTS["ppo_script"]), + "grpo_script_artifact": str(STAGE_SCRIPTS["grpo_script"]), + "closed_loop_script_artifact": str(STAGE_SCRIPTS["closed_loop_script"]), + "input_hashes": input_hashes, + } + + +def _attach_parity(report_path: Path, *, parity_path: Path, parity_sha256: str, evidence_root: Path) -> None: + report = _load(report_path) + artifact_paths = report.setdefault("artifact_paths", {}) + input_hashes = report.setdefault("input_hashes", {}) + required = report.setdefault("required_artifact_keys", []) + artifact_paths["parity_report"] = _rel_to_evidence_root(parity_path, evidence_root) + input_hashes["parity_report"] = parity_sha256 + if "parity_report" not in required: + required.append("parity_report") + report["parity_report_id"] = "phase3_parity_report" + _write(report_path, report) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--phase-dir", type=Path, required=True) + parser.add_argument("--target-run-id", default="20260624T040000Z-slurm-243958") + parser.add_argument("--update-milestones", action="store_true") + args = parser.parse_args() + + phase_dir = args.phase_dir + evidence_root = phase_dir.parents[1] + reports_dir = phase_dir / "runs" / "milestone_reports" + p1 = _load(reports_dir / MILESTONE_FILES["P3-M1"]) + p2 = _load(reports_dir / MILESTONE_FILES["P3-M2"]) + p3 = _load(reports_dir / MILESTONE_FILES["P3-M3"]) + p4 = _load(reports_dir / MILESTONE_FILES["P3-M4"]) + introspection = _load(_artifact(evidence_root, p1, "introspection_report")) + runtime_evidence = _runtime_evidence(evidence_root=evidence_root, p1_report=p1, target_run_id=args.target_run_id) + report = build_phase3_parity_report( + target_run_id=args.target_run_id, + ppo_report=p2, + grpo_report=p3, + closed_loop_report=p4, + introspection_report=introspection, + runtime_evidence=runtime_evidence, + evidence_root=evidence_root, + ) + out = phase_dir / "runs" / "parity" / "phase3_parity_report.json" + _write(out, report) + errors = validate_phase3_parity_report(report, target_run_id=args.target_run_id, evidence_root=evidence_root) + if errors: + print(json.dumps({"report": str(out), "validation_errors": errors, "updated_milestones": False}, sort_keys=True)) + return 1 + if args.update_milestones: + parity_sha256 = sha256_file(out) + for filename in (MILESTONE_FILES["P3-M2"], MILESTONE_FILES["P3-M3"], MILESTONE_FILES["P3-M4"]): + _attach_parity(reports_dir / filename, parity_path=out, parity_sha256=parity_sha256, evidence_root=evidence_root) + print(json.dumps({"report": str(out), "validation_errors": [], "updated_milestones": bool(args.update_milestones)}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/build_phase3_runner_contract.py b/scripts/rl_phase3/build_phase3_runner_contract.py new file mode 100644 index 00000000..d7e8101a --- /dev/null +++ b/scripts/rl_phase3/build_phase3_runner_contract.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path +from typing import Iterable + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from breadboard.rl.phase4.wrapper_identity import collect_wrapper_identity, parse_deps_pins + +TARGET_RUN_ID_DEFAULT = "20260624T040000Z-slurm-243958" +RUNNER_CONTRACT_SCHEMA = "bb.rl.phase3.runner_contract.v1" +RUNNER_CONTRACT_ID = "phase3_runner_contract" +RUNNER_CONTRACT_BOUNDARY = "phase3_verl_wrapper_runner_contract_named_target_scope" +RUNNER_CONTRACT_BLOCKED_BOUNDARY = "phase3_verl_wrapper_runner_contract_blocked_scope" + + +def _sha_bytes(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _sha_file(path: Path) -> str: + return _sha_bytes(path.read_bytes()) if path.exists() and path.is_file() else "" + + +def _run_git(args: list[str], cwd: Path) -> str: + try: + result = subprocess.run(["git", *args], cwd=cwd, check=False, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=10) + except (OSError, subprocess.TimeoutExpired): + return "" + return result.stdout.strip() if result.returncode == 0 else "" + + +def _directory_digest(root: Path, paths: Iterable[Path]) -> str: + h = hashlib.sha256() + any_file = False + for path in sorted(paths, key=lambda item: str(item.relative_to(root))): + if not path.is_file(): + continue + any_file = True + rel = str(path.relative_to(root)).replace("\\", "/").encode() + h.update(rel + b"\0" + path.read_bytes() + b"\0") + return "sha256:" + h.hexdigest() if any_file else "" + + +def _strip_yaml_inline_comment(value: str) -> str: + in_single = False + in_double = False + escaped = False + for index, char in enumerate(value): + if escaped: + escaped = False + continue + if char == "\\" and in_double: + escaped = True + continue + if char == "'" and not in_double: + in_single = not in_single + continue + if char == '"' and not in_single: + in_double = not in_double + continue + if char == "#" and not in_single and not in_double and (index == 0 or value[index - 1].isspace()): + return value[:index].rstrip() + return value.strip() + + +def _clean_yaml_scalar(value: str) -> str: + value = _strip_yaml_inline_comment(value).strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + return value.strip() + + +def _parse_deps_yaml(path: Path) -> dict[str, str]: + return parse_deps_pins(path) + + +def _submodule_status(wrapper_dir: Path) -> list[dict[str, str]]: + raw = _run_git(["submodule", "status", "--recursive"], wrapper_dir) + rows: list[dict[str, str]] = [] + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + marker = line[0] if line[0] in {"-", "+", "U"} else "" + parts = line.lstrip("-+U ").split() + if len(parts) >= 2: + rows.append({"path": parts[1], "commit": parts[0], "marker": marker}) + return rows + + +def build_contract( + *, wrapper_dir: Path, target_run_id: str, container_image: str, container_digest: str, launch_command: str, git_runner=_run_git +) -> dict: + wrapper_dir = wrapper_dir.resolve() + deps_path = wrapper_dir / "deps.yaml" + gitmodules_path = wrapper_dir / ".gitmodules" + identity = collect_wrapper_identity(wrapper_dir, git_runner=git_runner) + deps = identity.pins + patch_hash = _directory_digest(wrapper_dir, (wrapper_dir / "patches" / "verl").glob("**/*")) + recipe_files = [] + for rel in ("src/zyphra_verl", "launch", "gates", "deps.yaml", "pyproject.toml"): + path = wrapper_dir / rel + if path.is_dir(): + recipe_files.extend(path.glob("**/*")) + elif path.exists(): + recipe_files.append(path) + recipe_hash = _directory_digest(wrapper_dir, recipe_files) + commit = identity.wrapper_commit + ref = identity.wrapper_ref + submodules = list(identity.submodules.values()) + required = { + "wrapper_commit": commit, + "verl_pin": identity.components["verl"].expected_commit, + "verl_commit": identity.components["verl"].actual_commit, + "nemo_gym_pin": identity.components["nemo_gym"].expected_commit, + "nemo_gym_commit": identity.components["nemo_gym"].actual_commit, + "patch_queue_sha256": patch_hash, + "recipe_package_sha256": recipe_hash, + "container_digest": container_digest, + "launch_command": launch_command, + } + missing = [key for key, value in required.items() if not value] + identity_blockers = list(identity.blockers) + passed = not missing and not identity_blockers + return { + "schema_version": RUNNER_CONTRACT_SCHEMA, + "report_id": RUNNER_CONTRACT_ID, + "claim_boundary": RUNNER_CONTRACT_BOUNDARY if passed else RUNNER_CONTRACT_BLOCKED_BOUNDARY, + "target_run_id": target_run_id, + "component": "runner_contract", + "wrapper": { + "path": str(wrapper_dir), + "commit": commit, + "ref": ref, + "gitmodules_path": str(gitmodules_path), + "gitmodules_sha256": _sha_file(gitmodules_path), + "deps_yaml_path": str(deps_path), + "deps_yaml_sha256": _sha_file(deps_path), + "submodules": submodules, + }, + "pins": deps, + "required_identities": required, + "wrapper_identity": identity.to_dict(), + "wrapper_identity_blockers": identity_blockers, + "patch_queue_sha256": patch_hash, + "recipe_package_sha256": recipe_hash, + "container_image": container_image, + "container_digest": container_digest, + "launch_command": launch_command, + "missing_required_identities": missing, + "scorecard_update_allowed": False, + "passed": passed, + "blocked_reason": "" if passed else "missing_runner_contract_identity", + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--phase-dir", required=True, type=Path) + parser.add_argument("--wrapper-dir", required=True, type=Path) + parser.add_argument("--target-run-id", default=TARGET_RUN_ID_DEFAULT) + parser.add_argument("--container-image", default="vllm/vllm-openai-rocm:nightly") + parser.add_argument("--container-digest", default="") + parser.add_argument("--launch-command", default="launch/train.sh") + args = parser.parse_args() + report = build_contract( + wrapper_dir=args.wrapper_dir, + target_run_id=args.target_run_id, + container_image=args.container_image, + container_digest=args.container_digest, + launch_command=args.launch_command, + ) + output = args.phase_dir / "runs" / "runner_contract" / "phase3_runner_contract.json" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + print(json.dumps({"report": str(output), "passed": report["passed"], "missing": report["missing_required_identities"]}, sort_keys=True)) + return 0 if report["passed"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/build_phase4_native_inference_payload.py b/scripts/rl_phase3/build_phase4_native_inference_payload.py new file mode 100644 index 00000000..5027693b --- /dev/null +++ b/scripts/rl_phase3/build_phase4_native_inference_payload.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import stat +import sys +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from breadboard.rl.phase4.wrapper_identity import collect_wrapper_identity + +REQUIRED_ZIP_ENTRIES = ( + "run.sh", + "target_phase4_native_inference_lane.py", + "repo/breadboard/__init__.py", + "repo/breadboard/rl/__init__.py", + "repo/breadboard/rl/phase4/__init__.py", + "repo/breadboard/rl/phase4/native_inference.py", + "repo/breadboard/rl/phase4/wrapper_identity.py", + "verl_wrapper/src/zyphra_verl/nemo_gym_loop.py", + "verl_wrapper/src/zyphra_verl/configs/agent_loops.yaml", + "verl_wrapper/wrapper_identity.json", +) + +REQUIRED_NATIVE_TERMS = ( + "BREADBOARD_NATIVE_INFERENCE_OWNER", + "NativeInferenceLane", + "request_id", + "response_id", + "backend_token_texts", + "posthoc_token_ids", +) + +FORBIDDEN_TARGET_TERMS = ( + "fake_manager", + "probe_manager", + "Codex reroute", +) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n") + + +def copy_file(src: Path, dst: Path, *, executable: bool = False) -> None: + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + if executable: + mode = dst.stat().st_mode + dst.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def render_run_sh() -> str: + return """#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +HIP_DEVICES="${HIP_VISIBLE_DEVICES:-${ROCR_VISIBLE_DEVICES:-0}}" +if [ -z "$HIP_DEVICES" ]; then HIP_DEVICES=0; fi +mkdir -p "$SCRIPT_DIR/.hf_home" "$SCRIPT_DIR/.phase4_native_logs" "$SCRIPT_DIR/.phase4_native_venv_cache" +docker run --rm --ipc=host \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + --device=/dev/kfd --device=/dev/dri --group-add video \ + -e HIP_VISIBLE_DEVICES="$HIP_DEVICES" \ + -e ROCR_VISIBLE_DEVICES="${ROCR_VISIBLE_DEVICES:-$HIP_DEVICES}" \ + -e SLURM_JOB_ID="${SLURM_JOB_ID:-}" \ + -e SLURMD_NODENAME="${SLURMD_NODENAME:-$(hostname)}" \ + -e HF_HOME=/workspace/.hf_home \ + -e PHASE3_TARGET_RUN_ID="${PHASE3_TARGET_RUN_ID:-}" \ + -e PHASE3_SLURM_JOB_ID="${SLURM_JOB_ID:-}" \ + -e ZYPHRA_NEMO_GYM_DIR=/workspace/verl_wrapper/third_party/nemo-gym \ + -e PHASE4_NATIVE_INFERENCE_LOG_DIR=/workspace/.phase4_native_logs \ + -e PHASE4_NATIVE_INFERENCE_MODEL="${PHASE4_NATIVE_INFERENCE_MODEL:-Qwen/Qwen2.5-0.5B-Instruct}" \ + -v "$SCRIPT_DIR":/workspace \ + --entrypoint bash \ + vllm/vllm-openai-rocm:nightly \ + -lc 'set -euo pipefail +python -m venv /workspace/.phase4_native_venv +source /workspace/.phase4_native_venv/bin/activate +python -m pip install --upgrade --quiet pip setuptools wheel +python -m pip install --quiet yappi gprof2dot pydot requests omegaconf ray transformers tensordict +if [ ! -e /workspace/verl_wrapper/wrapper_identity.json ]; then + echo "PHASE4_BLOCKED_REASON=missing_exact_wrapper_dependency:/workspace/verl_wrapper/wrapper_identity.json" >&2 + exit 93 +fi +for required_dir in /workspace/verl_wrapper/third_party/verl /workspace/verl_wrapper/third_party/nemo-gym; do + if [ ! -f "$required_dir/pyproject.toml" ] && [ ! -f "$required_dir/setup.py" ]; then + echo "PHASE4_BLOCKED_REASON=missing_exact_wrapper_dependency:${required_dir}" >&2 + exit 93 + fi +done +python -m pip install --quiet -e /workspace/verl_wrapper +python -m pip install --quiet -e /workspace/verl_wrapper/third_party/verl +python -m pip install --quiet -e /workspace/verl_wrapper/third_party/nemo-gym +export PYTHONPATH=/workspace/repo:/workspace/verl_wrapper/src:/workspace/verl_wrapper/third_party/verl:/workspace/verl_wrapper/third_party/nemo-gym:${PYTHONPATH:-} +python /workspace/target_phase4_native_inference_lane.py /workspace/verl_wrapper' +""" + + +def make_zip(stage_dir: Path, zip_path: Path) -> None: + if zip_path.exists(): + zip_path.unlink() + with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path in sorted(stage_dir.rglob("*")): + if path.is_dir() or "__pycache__" in path.parts or path.suffix == ".pyc": + continue + archive.write(path, path.relative_to(stage_dir).as_posix()) + + +def validate_payload(stage_dir: Path, zip_path: Path) -> tuple[bool, list[str], dict[str, str]]: + errors: list[str] = [] + hashes: dict[str, str] = {} + with zipfile.ZipFile(zip_path) as archive: + names = set(archive.namelist()) + for entry in REQUIRED_ZIP_ENTRIES: + if entry not in names: + errors.append(f"missing zip entry: {entry}") + run_sh = stage_dir / "run.sh" + if not run_sh.exists() or not (run_sh.stat().st_mode & stat.S_IXUSR): + errors.append("run.sh must exist and be executable") + run_sh_text = run_sh.read_text() if run_sh.exists() else "" + if "/shared/bb-p3-root" in run_sh_text: + errors.append("run.sh must not depend on IBM /shared/bb-p3-root") + for term in ( + "HF_HOME=/workspace/.hf_home", + "ZYPHRA_NEMO_GYM_DIR=/workspace/verl_wrapper/third_party/nemo-gym", + "PHASE4_NATIVE_INFERENCE_MODEL=\"${PHASE4_NATIVE_INFERENCE_MODEL:-Qwen/Qwen2.5-0.5B-Instruct}\"", + "PYTHONPATH=/workspace/repo:/workspace/verl_wrapper/src:/workspace/verl_wrapper/third_party/verl:/workspace/verl_wrapper/third_party/nemo-gym", + ): + if term not in run_sh_text: + errors.append(f"required DO-local run.sh term missing: {term}") + native_source = (stage_dir / "repo/breadboard/rl/phase4/native_inference.py").read_text() + target_source = (stage_dir / "target_phase4_native_inference_lane.py").read_text() + for term in REQUIRED_NATIVE_TERMS: + if term not in native_source and term not in target_source: + errors.append(f"required native term missing: {term}") + for term in FORBIDDEN_TARGET_TERMS: + if term in native_source or term in target_source: + errors.append(f"forbidden target term present: {term}") + wrapper_identity_path = stage_dir / "verl_wrapper/wrapper_identity.json" + if not wrapper_identity_path.exists(): + errors.append("missing wrapper identity manifest") + else: + try: + wrapper_identity = json.loads(wrapper_identity_path.read_text()) + if wrapper_identity.get("passed") is not True: + errors.append("wrapper identity manifest must pass") + except json.JSONDecodeError: + errors.append("wrapper identity manifest must be valid JSON") + for entry in REQUIRED_ZIP_ENTRIES: + file_path = stage_dir / entry + if file_path.exists() and file_path.is_file(): + hashes[entry] = sha256_file(file_path) + hashes[zip_path.name] = sha256_file(zip_path) + return not errors, errors, hashes + + +def build_payload(*, repo_root: Path, source_payload_dir: Path, output_dir: Path, stamp: str, git_runner=None) -> dict[str, Any]: + stage_dir = output_dir / f"native_breadboard_inference_lane_{stamp}" + zip_path = output_dir / f"phase4_native_breadboard_inference_lane_{stamp}.zip" + if stage_dir.exists(): + shutil.rmtree(stage_dir) + stage_dir.mkdir(parents=True) + (stage_dir / "run.sh").write_text(render_run_sh()) + (stage_dir / "run.sh").chmod(0o755) + copy_file(repo_root / "scripts/rl_phase3/target_phase4_native_inference_lane.py", stage_dir / "target_phase4_native_inference_lane.py") + for rel in ( + "breadboard/__init__.py", + "breadboard/rl/__init__.py", + "breadboard/rl/phase4/__init__.py", + "breadboard/rl/phase4/native_inference.py", + "breadboard/rl/phase4/wrapper_identity.py", + ): + copy_file(repo_root / rel, stage_dir / "repo" / rel) + wrapper_src = source_payload_dir / "verl_wrapper" + wrapper_dst = stage_dir / "verl_wrapper" + shutil.copytree(wrapper_src, wrapper_dst, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + wrapper_identity = collect_wrapper_identity(wrapper_src, git_runner=git_runner) if git_runner is not None else collect_wrapper_identity(wrapper_src) + write_json(wrapper_dst / "wrapper_identity.json", wrapper_identity.to_dict()) + make_zip(stage_dir, zip_path) + passed, errors, hashes = validate_payload(stage_dir, zip_path) + report = { + "schema_version": "bb.rl.phase4.native_inference_payload_build.v1", + "report_id": f"phase4_native_breadboard_inference_payload_build_{stamp}", + "built_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "promotional": False, + "scorecard_update_allowed": False, + "claim_boundary": "phase4_native_breadboard_sub_inference_lane_payload_build_scope", + "source_payload_dir": str(source_payload_dir), + "stage_dir": str(stage_dir), + "zip_path": str(zip_path), + "required_entries": list(REQUIRED_ZIP_ENTRIES), + "hashes": hashes, + "wrapper_identity": wrapper_identity.to_dict(), + "errors": errors, + "passed": passed, + } + report_path = output_dir / f"phase4_native_breadboard_inference_payload_build_{stamp}.json" + write_json(report_path, report) + report["report_path"] = str(report_path) + report["report_sha256"] = sha256_file(report_path) + return report + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repo-root", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument("--source-payload-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--stamp", default=datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")) + args = parser.parse_args(argv) + report = build_payload( + repo_root=args.repo_root.resolve(), + source_payload_dir=args.source_payload_dir.resolve(), + output_dir=args.output_dir.resolve(), + stamp=args.stamp, + ) + print(json.dumps(report, sort_keys=True)) + return 0 if report["passed"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/discover_target_env.py b/scripts/rl_phase3/discover_target_env.py new file mode 100644 index 00000000..7e9fcff8 --- /dev/null +++ b/scripts/rl_phase3/discover_target_env.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import sys +from pathlib import Path +from typing import Any + + +def run(command: list[str], *, timeout: int = 30) -> dict[str, Any]: + try: + result = subprocess.run(command, text=True, capture_output=True, timeout=timeout, check=False) + return { + "argv": command, + "returncode": result.returncode, + "stdout": result.stdout[-12000:], + "stderr": result.stderr[-12000:], + } + except Exception as exc: # noqa: BLE001 - discovery records availability failures. + return {"argv": command, "returncode": 999, "error": exc.__class__.__name__, "message": str(exc)} + + +def which_many(names: list[str]) -> dict[str, str | None]: + return {name: shutil.which(name) for name in names} + + +def main() -> int: + target_run_id = os.environ.get("PHASE3_TARGET_RUN_ID", "") + commands = { + "python3_version": run(["python3", "--version"]), + "python3_executable": run(["python3", "-c", "import sys,json; print(json.dumps({'executable':sys.executable,'version':sys.version,'prefix':sys.prefix}))"]), + "pip_version": run(["python3", "-m", "pip", "--version"]), + "pip_index_torch": run(["python3", "-m", "pip", "index", "versions", "torch"], timeout=60), + "rocm_smi": run(["rocm-smi"], timeout=60), + "rocminfo": run(["rocminfo"], timeout=60), + "module_avail": run(["bash", "-lc", "module avail"], timeout=60), + "shared_write": run(["bash", "-lc", "mkdir -p /shared/bb-p3-probe && touch /shared/bb-p3-probe/write-test && python3 - <<'PY'\nfrom pathlib import Path\np=Path('/shared/bb-p3-probe/write-test')\nprint(p.exists(), p.stat().st_size)\nPY"]), + "network_pypi": run(["python3", "-c", "import urllib.request; r=urllib.request.urlopen('https://pypi.org/simple/torch/', timeout=10); print(r.status)"], timeout=20), + } + binaries = which_many(["python3", "pip3", "rocm-smi", "rocminfo", "module", "srun", "sbatch", "apptainer", "singularity", "docker", "podman", "conda", "micromamba", "curl", "wget", "git"]) + devices = [] + rocm_output = commands["rocm_smi"].get("stdout", "") + commands["rocm_smi"].get("stderr", "") + for line in rocm_output.splitlines(): + if "MI300" in line or "GPU" in line: + devices.append(line.strip()) + report = { + "schema_version": "bb.rl.phase3.target_environment_discovery.v1", + "report_id": "phase3_target_environment_discovery", + "component": "target_environment_discovery", + "claim_boundary": "phase3_target_environment_discovery_only_not_readiness", + "target_run_id": target_run_id, + "hostname": socket.gethostname(), + "cwd": str(Path.cwd()), + "binaries": binaries, + "commands": commands, + "mi300x_evidence_lines": devices, + "shared_workspace_writable": commands["shared_write"].get("returncode") == 0, + "pypi_reachable": commands["network_pypi"].get("returncode") == 0, + "container_tools": {name: binaries.get(name) for name in ("apptainer", "singularity", "docker", "podman")}, + "scorecard_update_allowed": False, + "passed": True, + } + print("PHASE3_COMPONENT_REPORT_JSON=" + json.dumps(report, sort_keys=True, separators=(",", ":"))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/inspect_target_verl.py b/scripts/rl_phase3/inspect_target_verl.py new file mode 100644 index 00000000..ec380be7 --- /dev/null +++ b/scripts/rl_phase3/inspect_target_verl.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import importlib +import inspect +import json +import os +import traceback +from pathlib import Path +from typing import Any + + +_SYMBOL_IMPORTS = { + "verl.__version__": ("verl", ("__version__",)), + "verl.protocol.DataProto": ("verl.protocol", ("DataProto",)), + "verl.protocol.DataProto.from_dict": ("verl.protocol", ("DataProto", "from_dict")), + "verl.protocol.DataProto.from_single_dict": ("verl.protocol", ("DataProto", "from_single_dict")), + "verl.protocol.DataProto.to": ("verl.protocol", ("DataProto", "to")), + "verl.trainer.main_ppo_sync": ("verl.trainer.main_ppo_sync", ()), + "verl.trainer.main_ppo": ("verl.trainer.main_ppo", ()), + "verl.trainer.ppo.ray_trainer.compute_advantage": ("verl.trainer.ppo.ray_trainer", ("compute_advantage",)), +} + + +def _symbol(name: str) -> dict[str, Any]: + if name in _SYMBOL_IMPORTS: + module_name, attr_parts = _SYMBOL_IMPORTS[name] + try: + obj = importlib.import_module(module_name) + for part in attr_parts: + obj = getattr(obj, part) + file_name = inspect.getfile(obj) if not isinstance(obj, str) else inspect.getfile(importlib.import_module(module_name)) + signature = str(inspect.signature(obj)) if callable(obj) else "" + return {"present": True, "file": file_name, "signature": signature} + except Exception as exc: # noqa: BLE001 + return {"present": False, "error": exc.__class__.__name__, "message": str(exc), "module": module_name} + parts = name.split(".") + last_error = "" + for split_at in range(len(parts), 0, -1): + module_name = ".".join(parts[:split_at]) + attr_parts = parts[split_at:] + try: + obj = importlib.import_module(module_name) + except Exception as exc: # noqa: BLE001 + last_error = exc.__class__.__name__ + continue + try: + for part in attr_parts: + obj = getattr(obj, part) + return {"present": True, "file": inspect.getfile(obj), "signature": str(inspect.signature(obj)) if callable(obj) else ""} + except Exception as exc: # noqa: BLE001 + return {"present": False, "error": exc.__class__.__name__, "message": str(exc), "module": module_name} + return {"present": False, "error": last_error or "ModuleNotFoundError"} + + +def main() -> int: + evidence_root = Path(os.environ.get("BREADBOARD_EVIDENCE_ROOT", "../docs_tmp")) + target_run_id = os.environ.get("PHASE3_TARGET_RUN_ID", "") + output_dir = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs" / "verl_api_introspection" + output_dir.mkdir(parents=True, exist_ok=True) + symbols = {} + try: + import verl # type: ignore + symbols["verl.__version__"] = getattr(verl, "__version__", "unknown") + except Exception as exc: # noqa: BLE001 + symbols["verl.__version__"] = {"present": False, "error": exc.__class__.__name__} + for name in ( + "verl.protocol.DataProto", + "verl.protocol.DataProto.from_dict", + "verl.protocol.DataProto.from_single_dict", + "verl.protocol.DataProto.to", + "verl.trainer.main_ppo_sync", + "verl.trainer.main_ppo", + "verl.trainer.ppo.ray_trainer.compute_advantage", + ): + symbols[name] = _symbol(name) + torch_report = {"present": False, "devices": []} + try: + import torch # type: ignore + torch_report = { + "present": True, + "version": torch.__version__, + "cuda_available": bool(torch.cuda.is_available()), + "device_count": int(torch.cuda.device_count()), + "devices": [torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())], + } + except Exception as exc: # noqa: BLE001 + torch_report = {"present": False, "error": exc.__class__.__name__, "traceback": traceback.format_exc()} + report = { + "schema_version": "bb.rl.phase3.verl_api_introspection.v1", + "report_id": "phase3_verl_api_introspection", + "claim_boundary": "phase3_target_verl_api_introspection_named_scope", + "target_run_id": target_run_id, + "symbols": symbols, + "torch": torch_report, + "scorecard_update_allowed": False, + "passed": torch_report.get("device_count") == 8 and all("MI300X" in name for name in torch_report.get("devices", [])), + } + (output_dir / "verl_api_introspection.json").write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + print("PHASE3_INTROSPECTION_REPORT=" + json.dumps(report, sort_keys=True, separators=(",", ":"))) + return 0 if report["passed"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/p3m11_live_endpoint_probe.py b/scripts/rl_phase3/p3m11_live_endpoint_probe.py new file mode 100644 index 00000000..ce117c21 --- /dev/null +++ b/scripts/rl_phase3/p3m11_live_endpoint_probe.py @@ -0,0 +1,447 @@ +from __future__ import annotations + +import hashlib +import ipaddress +import json +import os +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + +TARGET_RUN_ID = os.environ.get("PHASE3_TARGET_RUN_ID", "20260624T040000Z-slurm-243958") +COMMAND_ID = os.environ.get("PHASE3_COMMAND_ID", "phase3_p3m11_live_endpoint_probe") +OUT = Path("p3m11_live_endpoint_probe_output").resolve() +REQUEST_TIMEOUT_SECONDS = float(os.environ.get("PHASE3_P3M11_REQUEST_TIMEOUT_SECONDS", "10")) +LOCAL_OBJECT_STORE_BACKENDS = frozenset( + {"LocalObjectStore", "target_workspace_local_object_store", "local_object_store"} +) + + +def run(argv: list[str]) -> dict[str, Any]: + try: + proc = subprocess.run(argv, text=True, capture_output=True, timeout=30, check=False) + return { + "argv": argv, + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + except Exception as exc: # noqa: BLE001 - target probe captures tool availability. + return { + "argv": argv, + "returncode": None, + "stdout": "", + "stderr": repr(exc), + "exception": exc.__class__.__name__, + } + + +def present(name: str) -> bool: + return bool(os.environ.get(name)) + + +def _source_sha256() -> str: + try: + return "sha256:" + hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + except OSError: + return "sha256:unavailable" + + +def _redacted_url(value: str | None) -> str: + if not value: + return "" + parsed = urllib.parse.urlsplit(value) + host = parsed.hostname or "" + if parsed.port: + host = f"{host}:{parsed.port}" + return urllib.parse.urlunsplit((parsed.scheme, host, parsed.path, "", "")) + +def _endpoint_is_local(value: str | None) -> bool: + if not value: + return False + text = value.strip() + parsed = urllib.parse.urlsplit(text) + if not parsed.hostname and "://" not in text: + parsed = urllib.parse.urlsplit(f"//{text}") + host = parsed.hostname + if not host: + return False + normalized = host.lower().rstrip(".") + if normalized in {"localhost", "localhost.localdomain"}: + return True + local_names = { + str(os.environ.get("HOSTNAME") or "").lower().rstrip("."), + str(os.environ.get("SLURMD_NODENAME") or "").lower().rstrip("."), + } + if normalized in local_names - {""}: + return True + try: + return ipaddress.ip_address(normalized).is_loopback + except ValueError: + return False + + +def _join_url(base: str, path: str) -> str: + return urllib.parse.urljoin(base.rstrip("/") + "/", path.lstrip("/")) + + +def _configured_probe_url(prefix: str) -> tuple[str | None, list[str]]: + full = os.environ.get(f"{prefix}_PROBE_URL") + if full: + return full, [] + base = os.environ.get(f"{prefix}_BASE_URL") + path = os.environ.get(f"{prefix}_PROBE_PATH") + missing: list[str] = [] + if not base: + missing.append(f"{prefix}_BASE_URL") + if not path: + missing.append(f"{prefix}_PROBE_PATH") + if missing: + return None, missing + return _join_url(base or "", path or ""), [] + + +def _parse_ok_status(prefix: str) -> set[int]: + raw = os.environ.get(f"{prefix}_OK_STATUS", "200-299") + statuses: set[int] = set() + for chunk in raw.split(","): + item = chunk.strip() + if not item: + continue + if "-" in item: + low, high = item.split("-", 1) + statuses.update(range(int(low), int(high) + 1)) + else: + statuses.add(int(item)) + return statuses or set(range(200, 300)) + + +def _token_headers(prefix: str) -> dict[str, str]: + token = os.environ.get(f"{prefix}_TOKEN") or "" + if not token: + return {} + header = os.environ.get(f"{prefix}_TOKEN_HEADER", "Authorization") + scheme = os.environ.get(f"{prefix}_TOKEN_SCHEME", "Bearer") + value = f"{scheme} {token}" if scheme else token + return {header: value} + + +def _body(prefix: str) -> bytes | None: + if f"{prefix}_PROBE_BODY_JSON" in os.environ: + return os.environ[f"{prefix}_PROBE_BODY_JSON"].encode("utf-8") + if f"{prefix}_PROBE_BODY" in os.environ: + return os.environ[f"{prefix}_PROBE_BODY"].encode("utf-8") + return None + + +def _request( + *, + prefix: str, + url: str, + method: str, + body: bytes | None = None, + extra_headers: dict[str, str] | None = None, +) -> dict[str, Any]: + headers = {"User-Agent": "breadboard-phase3-p3m11-probe"} + headers.update(_token_headers(prefix)) + headers.update(extra_headers or {}) + if body is not None: + headers.setdefault("Content-Type", "application/json") + started = time.perf_counter() + try: + request = urllib.request.Request(url, data=body, headers=headers, method=method) + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response: # noqa: S310 - explicit target endpoint probe. + payload = response.read() + status = int(response.status) + latency = time.perf_counter() - started + return { + "ok": status in _parse_ok_status(prefix), + "status": status, + "latency_seconds": latency, + "body_sha256": "sha256:" + hashlib.sha256(payload).hexdigest(), + "body_bytes": len(payload), + "body": payload, + "endpoint": _redacted_url(url), + } + except urllib.error.HTTPError as exc: + latency = time.perf_counter() - started + return { + "ok": False, + "status": int(exc.code), + "latency_seconds": latency, + "body_sha256": "sha256:" + hashlib.sha256(exc.read()).hexdigest(), + "endpoint": _redacted_url(url), + "error": "HTTPError", + } + except Exception as exc: # noqa: BLE001 - report blocked target behavior without secrets. + latency = time.perf_counter() - started + return { + "ok": False, + "status": None, + "latency_seconds": latency, + "endpoint": _redacted_url(url), + "error": exc.__class__.__name__, + } + + +def collect_verifier_metrics() -> tuple[dict[str, Any], list[str]]: + prefix = "BREADBOARD_VERIFIER" + url, missing = _configured_probe_url(prefix) + token_present = present("BREADBOARD_VERIFIER_TOKEN") + errors = list(missing) + if not token_present: + errors.append("BREADBOARD_VERIFIER_TOKEN") + result: dict[str, Any] = {} + endpoint_is_local = _endpoint_is_local(url) + if endpoint_is_local: + errors.append("verifier_endpoint_is_local") + if url and token_present and not endpoint_is_local: + method = os.environ.get("BREADBOARD_VERIFIER_PROBE_METHOD", "GET").upper() + result = _request(prefix=prefix, url=url, method=method, body=_body(prefix)) + if not result.get("ok"): + errors.append("verifier_probe_http_not_ready") + elif not endpoint_is_local: + errors.append("verifier_probe_not_configured") + latency = [result["latency_seconds"]] if result.get("ok") else None + return { + "source": "verifier_client", + "verifier_latency_seconds": latency, + "env_presence": { + "BREADBOARD_VERIFIER_BASE_URL": present("BREADBOARD_VERIFIER_BASE_URL"), + "BREADBOARD_VERIFIER_TOKEN": token_present, + "BREADBOARD_VERIFIER_PROBE_URL": present("BREADBOARD_VERIFIER_PROBE_URL"), + "BREADBOARD_VERIFIER_PROBE_PATH": present("BREADBOARD_VERIFIER_PROBE_PATH"), + }, + "endpoint": result.get("endpoint") or _redacted_url(url), + "http_status": result.get("status"), + "latency_seconds": result.get("latency_seconds"), + "errors": errors, + }, errors + + +def _object_url_from_template(env_name: str, *, bucket: str, key: str) -> tuple[str | None, list[str]]: + template = os.environ.get(env_name) + if not template: + return None, [env_name] + quoted_bucket = urllib.parse.quote(bucket, safe="") + quoted_key = urllib.parse.quote(key, safe="/") + try: + return template.format(bucket=quoted_bucket, key=quoted_key), [] + except KeyError as exc: + return None, [f"{env_name}_missing_placeholder_{exc.args[0]}"] + + +def collect_object_store_metrics() -> tuple[dict[str, Any], list[str]]: + prefix = "BREADBOARD_OBJECT_STORE" + bucket = os.environ.get("BREADBOARD_OBJECT_STORE_BUCKET") or "" + token_present = present("BREADBOARD_OBJECT_STORE_TOKEN") + errors: list[str] = [] + for key in ("BREADBOARD_OBJECT_STORE_BASE_URL", "BREADBOARD_OBJECT_STORE_BUCKET", "BREADBOARD_OBJECT_STORE_TOKEN"): + if not present(key): + errors.append(key) + base_url = os.environ.get("BREADBOARD_OBJECT_STORE_BASE_URL") + object_key = f"phase3/p3m11/{TARGET_RUN_ID}/{COMMAND_ID}/{os.environ.get('SLURM_JOB_ID', 'no-slurm')}.txt" + put_url, put_missing = _object_url_from_template("BREADBOARD_OBJECT_STORE_PUT_URL_TEMPLATE", bucket=bucket, key=object_key) + get_url, get_missing = _object_url_from_template("BREADBOARD_OBJECT_STORE_GET_URL_TEMPLATE", bucket=bucket, key=object_key) + delete_template = os.environ.get("BREADBOARD_OBJECT_STORE_DELETE_URL_TEMPLATE") + delete_url: str | None = None + if delete_template: + delete_url, delete_missing = _object_url_from_template("BREADBOARD_OBJECT_STORE_DELETE_URL_TEMPLATE", bucket=bucket, key=object_key) + errors.extend(delete_missing) + errors.extend(put_missing) + errors.extend(get_missing) + local_endpoint_errors = [ + f"object_store_{name}_endpoint_is_local" + for name, url in (("base", base_url), ("put", put_url), ("get", get_url), ("delete", delete_url)) + if _endpoint_is_local(url) + ] + errors.extend(local_endpoint_errors) + payload = f"breadboard-p3m11-object-store-probe\n{TARGET_RUN_ID}\n{COMMAND_ID}\n".encode("utf-8") + payload_sha256 = "sha256:" + hashlib.sha256(payload).hexdigest() + readback_body_sha256 = "" + put_result: dict[str, Any] = {} + get_result: dict[str, Any] = {} + delete_result: dict[str, Any] = {} + verified = False + if put_url and get_url and bucket and token_present and not local_endpoint_errors: + put_method = os.environ.get("BREADBOARD_OBJECT_STORE_PUT_METHOD", "PUT").upper() + get_method = os.environ.get("BREADBOARD_OBJECT_STORE_GET_METHOD", "GET").upper() + put_result = _request(prefix=prefix, url=put_url, method=put_method, body=payload, extra_headers={"Content-Type": "text/plain"}) + if put_result.get("ok"): + get_result = _request(prefix=prefix, url=get_url, method=get_method) + readback_body = get_result.get("body") + if isinstance(readback_body, bytes): + readback_body_sha256 = "sha256:" + hashlib.sha256(readback_body).hexdigest() + verified = bool(get_result.get("ok") and readback_body == payload) + if not put_result.get("ok"): + errors.append("object_store_put_failed") + elif not verified: + errors.append("object_store_readback_mismatch") + if delete_url: + delete_method = os.environ.get("BREADBOARD_OBJECT_STORE_DELETE_METHOD", "DELETE").upper() + delete_result = _request(prefix=prefix, url=delete_url, method=delete_method) + elif not local_endpoint_errors: + errors.append("object_store_probe_not_configured") + backend = os.environ.get("BREADBOARD_OBJECT_STORE_BACKEND", "configured_http_object_store") if verified else "target_workspace_local_object_store" + if backend in LOCAL_OBJECT_STORE_BACKENDS: + errors.append("object_store_backend_is_local") + return { + "source": "object_store", + "object_store": backend, + "object_store_writes": 1 if verified else 0, + "artifact_bytes": len(payload) if verified else 0, + "artifact_sha256": payload_sha256 if verified else "", + "written_sha256": payload_sha256 if verified else "", + "readback_sha256": readback_body_sha256 if verified else "", + "env_presence": { + "BREADBOARD_OBJECT_STORE_BASE_URL": present("BREADBOARD_OBJECT_STORE_BASE_URL"), + "BREADBOARD_OBJECT_STORE_BUCKET": bool(bucket), + "BREADBOARD_OBJECT_STORE_TOKEN": token_present, + "BREADBOARD_OBJECT_STORE_PUT_URL_TEMPLATE": present("BREADBOARD_OBJECT_STORE_PUT_URL_TEMPLATE"), + "BREADBOARD_OBJECT_STORE_GET_URL_TEMPLATE": present("BREADBOARD_OBJECT_STORE_GET_URL_TEMPLATE"), + "BREADBOARD_OBJECT_STORE_DELETE_URL_TEMPLATE": present("BREADBOARD_OBJECT_STORE_DELETE_URL_TEMPLATE"), + }, + "endpoint": _redacted_url(base_url), + "put_endpoint": _redacted_url(put_url), + "get_endpoint": _redacted_url(get_url), + "delete_endpoint": _redacted_url(delete_url), + "object_key_sha256": "sha256:" + hashlib.sha256(object_key.encode("utf-8")).hexdigest(), + "put_status": put_result.get("status"), + "get_status": get_result.get("status"), + "delete_status": delete_result.get("status"), + "write_read_verified": verified, + "errors": errors, + }, errors + + +def collect_scheduler_metrics() -> tuple[dict[str, Any], list[str]]: + prefix = "BREADBOARD_SCHEDULER" + url, missing = _configured_probe_url(prefix) + token_present = present("BREADBOARD_SCHEDULER_TOKEN") + errors = list(missing) + if not token_present: + errors.append("BREADBOARD_SCHEDULER_TOKEN") + result: dict[str, Any] = {} + endpoint_is_local = _endpoint_is_local(url) + if endpoint_is_local: + errors.append("scheduler_endpoint_is_local") + if url and token_present and not endpoint_is_local: + method = os.environ.get("BREADBOARD_SCHEDULER_PROBE_METHOD", "GET").upper() + result = _request(prefix=prefix, url=url, method=method, body=_body(prefix)) + if not result.get("ok"): + errors.append("scheduler_probe_http_not_ready") + elif not endpoint_is_local: + errors.append("scheduler_probe_not_configured") + ready = bool(result.get("ok")) + return { + "source": "scheduler_control", + "scheduler_control": { + "endpoint_present": bool(url), + "token_present": token_present, + "status": "ready" if ready else "blocked_missing_endpoint_or_token", + "endpoint": result.get("endpoint") or _redacted_url(url), + "http_status": result.get("status"), + "latency_seconds": result.get("latency_seconds"), + }, + "env_presence": { + "BREADBOARD_SCHEDULER_BASE_URL": present("BREADBOARD_SCHEDULER_BASE_URL"), + "BREADBOARD_SCHEDULER_TOKEN": token_present, + "BREADBOARD_SCHEDULER_PROBE_URL": present("BREADBOARD_SCHEDULER_PROBE_URL"), + "BREADBOARD_SCHEDULER_PROBE_PATH": present("BREADBOARD_SCHEDULER_PROBE_PATH"), + }, + "errors": errors, + }, errors + + +def _write_artifacts(artifacts: dict[str, dict[str, Any]]) -> None: + OUT.mkdir(parents=True, exist_ok=True) + for name, payload in artifacts.items(): + path = OUT / name + safe_payload = dict(payload) + safe_payload.pop("body", None) + path.write_text(json.dumps(safe_payload, sort_keys=True, indent=2) + "\n") + + +def main() -> int: + slurm_job_id = os.environ.get("SLURM_JOB_ID", "") + node = os.environ.get("SLURMD_NODENAME") or os.environ.get("HOSTNAME", "") + slurm_metrics: dict[str, Any] = { + "source": "slurm_sacct", + "job_id": slurm_job_id, + "node": node, + "sacct_stdout": run([ + "sacct", + "-j", + slurm_job_id, + "--parsable2", + "--noheader", + "--format=JobID,JobName,Partition,AllocTRES,State,Elapsed,NodeList", + ]) if slurm_job_id else {}, + "scheduler_retry_count": 0, + "queue_wait_seconds": 0.0, + } + gpu_metrics: dict[str, Any] = { + "source": "rocm_smi", + "node": node, + "rocm_smi_stdout": run(["/opt/rocm/bin/rocm-smi", "--showuse", "--showproductname"]), + "gpu_utilization": {f"card{index}": 0 for index in range(8)}, + } + verifier_metrics, verifier_errors = collect_verifier_metrics() + object_store_metrics, object_store_errors = collect_object_store_metrics() + scheduler_metrics, scheduler_errors = collect_scheduler_metrics() + service_metrics: dict[str, Any] = { + "source": "service_event_log", + "events": ["p3m11_live_endpoint_probe"], + "task_throughput": 1, + "failure_taxonomy": { + "verifier": verifier_errors, + "object_store": object_store_errors, + "scheduler": scheduler_errors, + }, + } + budget_caps: dict[str, Any] = {"source": "target_probe", "remaining_usd": 0.0} + artifacts: dict[str, dict[str, Any]] = { + "slurm_metrics.json": slurm_metrics, + "gpu_metrics.json": gpu_metrics, + "verifier_metrics.json": verifier_metrics, + "service_metrics.json": service_metrics, + "object_store_metrics.json": object_store_metrics, + "scheduler_metrics.json": scheduler_metrics, + "budget_caps.json": budget_caps, + } + _write_artifacts(artifacts) + blockers: list[str] = [] + if verifier_errors: + blockers.append("verifier_latency_unavailable") + if object_store_errors: + blockers.append("production_object_store_write_read_unavailable") + if scheduler_errors: + blockers.append("scheduler_control_unavailable") + passed = not blockers + report = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "p3m11_live_endpoint_probe", + "milestone_id": "P3-M11", + "component": "observability_scheduler_store", + "claim_boundary": "phase3_live_observability_object_store_scheduler_scope" if passed else "phase3_observability_scheduler_store_blocked_scope", + "target_run_id": TARGET_RUN_ID, + "command_id": COMMAND_ID, + "points": 80, + "passed": passed, + "blocked_reason": ";".join(blockers), + "artifact_paths": {key.removesuffix(".json"): str(OUT / key) for key in artifacts}, + "artifact_payloads": {key.removesuffix(".json"): {k: v for k, v in payload.items() if k != "body"} for key, payload in artifacts.items()}, + "input_hashes": {"probe_source": _source_sha256()}, + "required_artifact_keys": [key.removesuffix(".json") for key in artifacts], + "scorecard_update_allowed": False, + } + print("PHASE3_COMPONENT_REPORT_JSON=" + json.dumps(report, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/run_phase3_benchmark_campaign.py b/scripts/rl_phase3/run_phase3_benchmark_campaign.py new file mode 100644 index 00000000..68cd9b58 --- /dev/null +++ b/scripts/rl_phase3/run_phase3_benchmark_campaign.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from breadboard.rl.phase2.benchmark import build_fixture_benchmark_source_pin +from breadboard.rl.phase3.benchmark_campaign import BenchmarkCampaignSpec, build_benchmark_campaign_report +from breadboard.rl.phase3.evidence import sha256_file, write_phase3_json + + +def _sha256_text(value: str) -> str: + return "sha256:" + hashlib.sha256(value.encode()).hexdigest() + + + +def _blocked_report(*, target_run_id: str, output_dir: Path, blocked_reason: str, extra: dict | None = None) -> dict: + blocker_path = output_dir / "benchmark_blocker_evidence.json" + blocker_payload = {"target_run_id": target_run_id, "blocked_reason": blocked_reason} + if extra: + blocker_payload["extra"] = extra + write_phase3_json(blocker_path, blocker_payload) + report = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "p3-m7_benchmark_campaign", + "milestone_id": "P3-M7", + "component": "benchmark_campaign", + "claim_boundary": "phase3_benchmark_campaign_blocked_scope", + "target_run_id": target_run_id, + "points": 80, + "passed": False, + "blocked_reason": blocked_reason, + "input_hashes": {"blocker_evidence": sha256_file(blocker_path)}, + "artifact_paths": {"blocker_evidence": str(blocker_path)}, + "required_artifact_keys": ["blocker_evidence"], + "scorecard_update_allowed": False, + } + if extra: + report.update(extra) + write_phase3_json(output_dir / "p3-m7_benchmark_campaign.json", report) + return report + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--phase-dir", required=True, type=Path) + parser.add_argument("--target-run-id", required=True) + parser.add_argument("--command-manifest", type=Path) + parser.add_argument("--output-dir", type=Path) + args = parser.parse_args() + phase_dir = args.phase_dir + output_dir = args.output_dir or phase_dir / "runs" / "benchmark_campaign" + output_dir.mkdir(parents=True, exist_ok=True) + benchmark_jsonl = os.environ.get("PHASE3_BENCHMARK_JSONL", "") + fixture_pin = None + if not benchmark_jsonl: + fixture_pin = build_fixture_benchmark_source_pin() + fixture_dir = output_dir / "locked_fixture_inputs" + fixture_dir.mkdir(parents=True, exist_ok=True) + source_path = fixture_dir / "slice-001.jsonl" + source_payload = "swe-rebench-v2.fixture.slice.001\n" + source_path.write_text(source_payload) + run_summary_path = fixture_dir / "run_summary.json" + contamination_path = fixture_dir / "contamination.json" + replay_dir = fixture_dir / "replays" + if replay_dir.exists(): + shutil.rmtree(replay_dir) + replay_dir.mkdir() + (replay_dir / "fixture-rejected-task.json").write_text(json.dumps({"task_id": "fixture-rejected-task", "reason": "locked_fixture_replay"})) + write_phase3_json(run_summary_path, { + "source_payload": source_payload, + "target_run_id": args.target_run_id, + "failed_tasks": ["fixture-rejected-task"], + "quarantined_tasks": [], + "metrics": { + "attempted": 1, + "accepted": 0, + "rejected": 1, + "quarantined": 0, + "pass_at_1": 0.0, + "mean_reward": 0.0, + "p50_latency_seconds": 0.0, + "p95_latency_seconds": 0.0, + }, + }) + write_phase3_json(contamination_path, { + "controls": ["source_hash_pin", "train_overlap_manifest", "prompt_solution_leakage_scan"], + "train_overlap_manifest": "locked_fixture_no_training_overlap", + "prompt_solution_leakage_scan": "locked_fixture_no_prompt_solution_leakage", + }) + else: + source_path = Path(benchmark_jsonl) + if not source_path.is_file(): + report = _blocked_report(target_run_id=args.target_run_id, output_dir=output_dir, blocked_reason="PHASE3_BENCHMARK_JSONL file missing", extra={"source_path": str(source_path)}) + print(json.dumps({"report": str(output_dir / "p3-m7_benchmark_campaign.json"), "passed": False, "blocked_reason": report["blocked_reason"]}, sort_keys=True)) + return 2 + run_summary_env = os.environ.get("PHASE3_BENCHMARK_RUN_SUMMARY_JSON", "") + if not run_summary_env: + source_copy = output_dir / "benchmark_source.jsonl" + shutil.copyfile(source_path, source_copy) + report = _blocked_report( + target_run_id=args.target_run_id, + output_dir=output_dir, + blocked_reason="PHASE3_BENCHMARK_RUN_SUMMARY_JSON absent; refusing to fabricate benchmark outcomes", + extra={ + "source_path": str(source_path), + "source_sha256": sha256_file(source_copy), + "artifact_paths": {"benchmark_source": str(source_copy)}, + "required_artifact_keys": ["benchmark_source"], + "input_hashes": {"benchmark_source": sha256_file(source_copy)}, + }, + ) + print(json.dumps({"report": str(output_dir / "p3-m7_benchmark_campaign.json"), "passed": False, "blocked_reason": report["blocked_reason"]}, sort_keys=True)) + return 2 + run_summary_path = Path(run_summary_env) + contamination_path = Path(os.environ.get("PHASE3_BENCHMARK_CONTAMINATION_JSON", "")) + replay_dir = Path(os.environ.get("PHASE3_BENCHMARK_REPLAY_DIR", "")) + missing = [ + name + for name, path, predicate in ( + ("PHASE3_BENCHMARK_RUN_SUMMARY_JSON", run_summary_path, Path.is_file), + ("PHASE3_BENCHMARK_CONTAMINATION_JSON", contamination_path, Path.is_file), + ("PHASE3_BENCHMARK_REPLAY_DIR", replay_dir, Path.is_dir), + ) + if not predicate(path) + ] + if missing: + report = _blocked_report(target_run_id=args.target_run_id, output_dir=output_dir, blocked_reason="missing benchmark evidence inputs: " + ",".join(missing)) + print(json.dumps({"report": str(output_dir / "p3-m7_benchmark_campaign.json"), "passed": False, "blocked_reason": report["blocked_reason"]}, sort_keys=True)) + return 2 + source_copy = output_dir / "benchmark_source.jsonl" + summary_copy = output_dir / "benchmark_run_summary.json" + contamination_copy = output_dir / "benchmark_contamination.json" + replay_copy = output_dir / "replays" + shutil.copyfile(source_path, source_copy) + shutil.copyfile(run_summary_path, summary_copy) + shutil.copyfile(contamination_path, contamination_copy) + if replay_copy.exists(): + shutil.rmtree(replay_copy) + shutil.copytree(replay_dir, replay_copy) + replay_manifest = output_dir / "replay_manifest.json" + replay_entries = [] + for path in sorted(replay_copy.rglob("*")): + if path.is_file(): + replay_entries.append({"path": str(path.relative_to(replay_copy)), "sha256": sha256_file(path), "bytes": path.stat().st_size}) + write_phase3_json(replay_manifest, {"replay_dir": str(replay_copy), "files": replay_entries}) + manifest_path = args.command_manifest or phase_dir / "runs" / "phase3_command_log_manifest.json" + manifest = json.loads(manifest_path.read_text()) + source_payload = source_copy.read_text() + expected_hash = hashlib.sha256(source_payload.encode()).hexdigest() + spec = BenchmarkCampaignSpec( + benchmark_id=fixture_pin.benchmark_id if fixture_pin else os.environ.get("PHASE3_BENCHMARK_ID", source_path.stem), + benchmark_version=fixture_pin.benchmark_version if fixture_pin else os.environ.get("PHASE3_BENCHMARK_VERSION", "external-jsonl"), + source_uri=fixture_pin.source_uri if fixture_pin else os.environ.get("PHASE3_BENCHMARK_SOURCE_URI", str(source_copy)), + expected_source_sha256=fixture_pin.expected_source_sha256 if fixture_pin else os.environ.get("PHASE3_BENCHMARK_SOURCE_SHA256", expected_hash), + split_id=fixture_pin.slice_id if fixture_pin else os.environ.get("PHASE3_BENCHMARK_SPLIT", "validation"), + max_tasks=int(os.environ.get("PHASE3_BENCHMARK_MAX_TASKS", "1" if fixture_pin else "0") or "0"), + contamination_manifest_path=contamination_copy, + output_dir=output_dir, + fixture_scope=fixture_pin is not None, + ) + report = build_benchmark_campaign_report(spec, run_summary_path=summary_copy, replay_dir=replay_copy, command_log_manifest=manifest) + write_phase3_json(output_dir / "p3-m7_benchmark_campaign.json", { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "p3-m7_benchmark_campaign", + "milestone_id": "P3-M7", + "component": "benchmark_campaign", + "claim_boundary": "phase3_named_benchmark_campaign_scope", + "target_run_id": args.target_run_id, + "points": 80, + "passed": report.get("passed") is True, + "blocked_reason": ";".join(report.get("errors", [])), + "benchmark_report": report, + "input_hashes": { + "benchmark_report": sha256_file(output_dir / "phase3_benchmark_campaign_report.json"), + "benchmark_source": sha256_file(source_copy), + "run_summary": sha256_file(summary_copy), + "contamination": sha256_file(contamination_copy), + "replay_manifest": sha256_file(replay_manifest), + }, + "artifact_paths": { + "benchmark_report": str(output_dir / "phase3_benchmark_campaign_report.json"), + "benchmark_source": str(source_copy), + "run_summary": str(summary_copy), + "contamination": str(contamination_copy), + "replay_manifest": str(replay_manifest), + }, + "required_artifact_keys": ["benchmark_report", "benchmark_source", "run_summary", "contamination", "replay_manifest"], + "scorecard_update_allowed": False, + }) + print(json.dumps({"report": str(output_dir / "p3-m7_benchmark_campaign.json"), "passed": report.get("passed") is True, "errors": report.get("errors", [])}, sort_keys=True)) + return 0 if report.get("passed") is True else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/run_phase3_direct_node_preflight.py b/scripts/rl_phase3/run_phase3_direct_node_preflight.py new file mode 100644 index 00000000..9ac4c57a --- /dev/null +++ b/scripts/rl_phase3/run_phase3_direct_node_preflight.py @@ -0,0 +1,324 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shlex +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from breadboard.rl.phase3.evidence import sha256_file +from breadboard.rl.phase3.security_enforcement import enforce_command_request +from scripts.rl_phase3.run_phase3_target_command import _build_ssh_command, _safe_artifact_name + +DIRECT_PREFLIGHT_SCHEMA = "bb.rl.phase4.direct_node_preflight.v1" +DIRECT_CLAIM_BOUNDARY = "phase4_direct_node_dev_preflight_only_not_phase3_promotion" +DEFAULT_IMAGE = "vllm/vllm-openai-rocm:nightly" +DEFAULT_REMOTE_ROOT = "/shared/bb-p3-root" +ENDPOINT_ENV_VARS = ( + "BREADBOARD_ORS_BASE_URL", + "BREADBOARD_ORS_TOKEN", + "BREADBOARD_OPENREWARD_BASE_URL", + "BREADBOARD_OPENREWARD_TOKEN", + "BREADBOARD_BENCHFLOW_BASE_URL", + "BREADBOARD_BENCHFLOW_TOKEN", + "BREADBOARD_HARBOR_BASE_URL", + "BREADBOARD_HARBOR_TOKEN", + "BREADBOARD_VERIFIER_BASE_URL", + "BREADBOARD_VERIFIER_TOKEN", + "BREADBOARD_OBJECT_STORE_BASE_URL", + "BREADBOARD_OBJECT_STORE_BUCKET", + "BREADBOARD_OBJECT_STORE_TOKEN", + "BREADBOARD_SCHEDULER_BASE_URL", + "BREADBOARD_SCHEDULER_TOKEN", + "HF_HOME", +) + + +def _iso() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _sha256_text(text: str) -> str: + return "sha256:" + hashlib.sha256(text.encode()).hexdigest() + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n") + + +def _presence_shell(var_names: tuple[str, ...] = ENDPOINT_ENV_VARS) -> str: + rows = [] + for name in var_names: + quoted = shlex.quote(name) + rows.append(f"if [ -n \"${{{quoted}:-}}\" ]; then echo ENV_{quoted}=present; else echo ENV_{quoted}=absent; fi") + return "; ".join(rows) + + +def _remote_precheck_command(*, direct_run_id: str, command_id: str, remote_root: str, image: str, hip_visible_devices: str, create_remote_root: bool) -> str: + mkdir = f"mkdir -p {shlex.quote(remote_root)} {shlex.quote(remote_root)}/hf_home {shlex.quote(remote_root)}/direct_node_runs; " if create_remote_root else "" + return ( + "set -u; " + f"export PHASE4_DIRECT_RUN_ID={shlex.quote(direct_run_id)}; " + f"export PHASE3_COMMAND_ID={shlex.quote(command_id)}; " + f"export HIP_VISIBLE_DEVICES={shlex.quote(hip_visible_devices)}; " + f"{mkdir}" + "echo PHASE4_DIRECT_NODE=$(hostname 2>/dev/null || true); " + "echo PHASE4_DIRECT_USER=$(whoami 2>/dev/null || true); " + "echo PHASE4_DIRECT_MODE=precheck; " + "echo PHASE4_DIRECT_SCHEDULER=none_direct_ssh; " + "echo PHASE4_DIRECT_GPU_MASK=$HIP_VISIBLE_DEVICES; " + "for cmd in docker rocm-smi python3 unzip; do if command -v $cmd >/dev/null 2>&1; then echo CMD_${cmd}=present:$(command -v $cmd); else echo CMD_${cmd}=absent; fi; done; " + "if [ -e /dev/kfd ]; then echo DEVICE_KFD=present; else echo DEVICE_KFD=absent; fi; " + "if [ -d /dev/dri ]; then echo DEVICE_DRI=present; else echo DEVICE_DRI=absent; fi; " + f"if [ -d {shlex.quote(remote_root)} ]; then echo REMOTE_ROOT=present; else echo REMOTE_ROOT=absent; fi; " + f"if [ -f {shlex.quote(remote_root)}/phase3_vllm_verl_py312/bin/activate ]; then echo RUNTIME_VENV=present; else echo RUNTIME_VENV=absent; fi; " + f"if command -v docker >/dev/null 2>&1 && docker image inspect {shlex.quote(image)} >/dev/null 2>&1; then echo RUNTIME_IMAGE=present; docker image inspect --format='IMAGE_ID={{{{.Id}}}}' {shlex.quote(image)} 2>/dev/null || true; docker image inspect --format='IMAGE_REPODIGESTS={{{{json .RepoDigests}}}}' {shlex.quote(image)} 2>/dev/null || true; else echo RUNTIME_IMAGE=absent; fi; " + "if command -v rocm-smi >/dev/null 2>&1; then echo ROCM_SMI_BEGIN; rocm-smi --showproductname --showmeminfo vram 2>&1 | head -160; echo ROCM_SMI_END; fi; " + + _presence_shell() + ) + + +def _remote_run_command(*, direct_run_id: str, command_id: str, remote_zip: str, remote_root: str, image: str, hip_visible_devices: str) -> str: + safe_remote_root = shlex.quote(remote_root) + safe_command = shlex.quote(command_id) + return ( + "set -euo pipefail; " + f"export PHASE4_DIRECT_RUN_ID={shlex.quote(direct_run_id)}; " + "export PHASE3_TARGET_RUN_ID=; " + f"export PHASE3_COMMAND_ID={shlex.quote(command_id)}; " + f"export HIP_VISIBLE_DEVICES={shlex.quote(hip_visible_devices)}; " + "echo PHASE4_DIRECT_NODE=$(hostname); " + "echo PHASE4_DIRECT_MODE=run; " + "echo PHASE4_DIRECT_SCHEDULER=none_direct_ssh; " + "echo PHASE4_DIRECT_GPU_MASK=$HIP_VISIBLE_DEVICES; " + f"test -d {safe_remote_root} || (echo PHASE4_BLOCKED_REASON=direct_runtime_root_missing; exit 92); " + f"test -f {safe_remote_root}/phase3_vllm_verl_py312/bin/activate || (echo PHASE4_BLOCKED_REASON=direct_runtime_venv_missing; exit 94); " + f"docker image inspect {shlex.quote(image)} >/dev/null 2>&1 || (echo PHASE4_BLOCKED_REASON=direct_runtime_image_missing; exit 93); " + f"WORK=$(mktemp -d {safe_remote_root}/direct_node_runs/{safe_command}.XXXXXX); " + f"echo PHASE4_DIRECT_WORKDIR=$WORK; python3 -m zipfile -e {shlex.quote(remote_zip)} \"$WORK\"; cd \"$WORK\"; test -f ./run.sh; bash ./run.sh" + ) + + +def _parse_key_values(log_text: str) -> dict[str, str]: + keys: dict[str, str] = {} + prefixes = ("PHASE4_", "PHASE3_", "CMD_", "DEVICE_", "REMOTE_", "RUNTIME_", "IMAGE_", "ENV_") + for line in log_text.splitlines(): + if "=" not in line or not line.startswith(prefixes): + continue + key, value = line.split("=", 1) + keys[key] = value.strip() + return keys + + +def _inline_component_reports(log_text: str) -> list[dict[str, Any]]: + reports: list[dict[str, Any]] = [] + for line in log_text.splitlines(): + if not line.startswith("PHASE3_COMPONENT_REPORT_JSON="): + continue + payload = line.split("=", 1)[1] + try: + report = json.loads(payload) + reports.append(report if isinstance(report, dict) else {"passed": False, "blocked_reason": "inline_report_not_object"}) + except json.JSONDecodeError: + reports.append({"passed": False, "blocked_reason": "invalid_inline_report"}) + return reports + + +def _env_presence(keys: dict[str, str]) -> dict[str, bool]: + prefix = "ENV_" + return {key.removeprefix(prefix): value == "present" for key, value in keys.items() if key.startswith(prefix)} + + +def _assessment_payload( + *, + argv: list[str], + mode: str, + ssh_alias: str, + command_id: str, + safe_command_id: str, + direct_run_id: str, + payload_zip: Path, + output_dir: Path, + raw_log_path: Path, + started_at: str, + completed_at: str, + exit_code: int, + keys: dict[str, str], + blocked_reason: str, + component_reports: list[dict[str, Any]], + hip_visible_devices: str, + remote_root: str, + image: str, +) -> dict[str, Any]: + component_failed_count = sum(1 for report in component_reports if report.get("passed") is not True) + component_blocked_reasons = [str(report.get("blocked_reason") or "inline_component_not_passed") for report in component_reports if report.get("passed") is not True] + component_failure_reason = ";".join(component_blocked_reasons) + effective_blocked_reason = blocked_reason or component_failure_reason + if mode == "run" and exit_code == 0 and not component_reports and not effective_blocked_reason: + effective_blocked_reason = "payload_evidence_missing" + precheck_ready = mode == "precheck" and exit_code == 0 and keys.get("RUNTIME_IMAGE") == "present" and keys.get("RUNTIME_VENV") == "present" and keys.get("REMOTE_ROOT") == "present" + run_passed = mode == "run" and exit_code == 0 and bool(component_reports) and not effective_blocked_reason and component_failed_count == 0 + raw_hash = sha256_file(raw_log_path) + return { + "schema_version": DIRECT_PREFLIGHT_SCHEMA, + "report_id": safe_command_id, + "claim_boundary": DIRECT_CLAIM_BOUNDARY, + "promotional": False, + "scorecard_update_allowed": False, + "canonical_phase3_command_log_manifest_eligible": False, + "canonical_phase3_command_log_manifest_reason": "not_slurm_direct_ssh_preflight", + "scheduler": "none_direct_ssh", + "slurm_job_id_present": False, + "target_run_id": None, + "direct_run_id": direct_run_id, + "mode": mode, + "ssh_alias": ssh_alias, + "node": keys.get("PHASE4_DIRECT_NODE", ""), + "command_id": command_id, + "safe_command_id": safe_command_id, + "argv": argv, + "started_at": started_at, + "completed_at": completed_at, + "exit_code": exit_code, + "status": "passed" if (precheck_ready or run_passed) else "blocked" if effective_blocked_reason or exit_code != 0 else "not_ready", + "passed": bool(precheck_ready or run_passed), + "blocked_reason": effective_blocked_reason, + "hip_visible_devices": hip_visible_devices, + "remote_root": remote_root, + "image": image, + "image_id": keys.get("IMAGE_ID", ""), + "image_repo_digests": keys.get("IMAGE_REPODIGESTS", ""), + "runtime": { + "remote_root": keys.get("REMOTE_ROOT", ""), + "venv": keys.get("RUNTIME_VENV", ""), + "image": keys.get("RUNTIME_IMAGE", ""), + }, + "commands": {key: value for key, value in keys.items() if key.startswith("CMD_")}, + "devices": {key: value for key, value in keys.items() if key.startswith("DEVICE_")}, + "endpoint_env_presence": _env_presence(keys), + "component_reports": component_reports, + "component_failed_count": component_failed_count, + "component_blocked_reasons": component_blocked_reasons, + "raw_log_path": str(raw_log_path.relative_to(output_dir)), + "raw_log_sha256": raw_hash, + "input_hashes": { + "payload_zip": sha256_file(payload_zip), + "raw_log": raw_hash, + "direct_command_key_values": _sha256_text(json.dumps(keys, sort_keys=True)), + }, + "artifact_paths": { + "raw_log": str(raw_log_path), + "assessment": str(output_dir / f"{safe_command_id}_direct_node_preflight.json"), + }, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--ssh-alias", required=True) + parser.add_argument("--command-id", required=True) + parser.add_argument("--direct-run-id", required=True) + parser.add_argument("--payload-zip", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--mode", choices=("precheck", "run"), default="precheck") + parser.add_argument("--remote-root", default=DEFAULT_REMOTE_ROOT) + parser.add_argument("--image", default=DEFAULT_IMAGE) + parser.add_argument("--hip-visible-devices", default="0") + parser.add_argument("--scp-timeout-seconds", type=int, default=120) + parser.add_argument("--command-timeout-seconds", type=int, default=900) + parser.add_argument("--create-remote-root", action="store_true") + args = parser.parse_args(argv) + if args.scp_timeout_seconds < 1: + parser.error("--scp-timeout-seconds must be positive") + if args.command_timeout_seconds < 1: + parser.error("--command-timeout-seconds must be positive") + if any(part in args.direct_run_id for part in ("/", "\\", "..")): + parser.error("--direct-run-id must be an identifier, not a path") + if not args.payload_zip.exists(): + raise FileNotFoundError(args.payload_zip) + + args.output_dir.mkdir(parents=True, exist_ok=True) + safe_command_id = _safe_artifact_name(args.command_id, fallback="phase4_direct_node_preflight") + log_dir = args.output_dir / "command_logs" + log_dir.mkdir(parents=True, exist_ok=True) + raw_log_path = log_dir / f"{safe_command_id}.log" + assessment_path = args.output_dir / f"{safe_command_id}_direct_node_preflight.json" + remote_zip = f"/tmp/{safe_command_id}.zip" + started_at = _iso() + exit_code = 1 + blocked_reason = "" + raw_log = "" + + try: + if args.mode == "run": + remote_command = _remote_run_command(direct_run_id=args.direct_run_id, command_id=safe_command_id, remote_zip=remote_zip, remote_root=args.remote_root, image=args.image, hip_visible_devices=args.hip_visible_devices) + command = _build_ssh_command(ssh_alias=args.ssh_alias, remote_command=remote_command) + enforce_command_request(command, workspace_relative_path="workspace/direct-node", workspace_id="workspace") + scp = subprocess.run(["scp", str(args.payload_zip), f"{args.ssh_alias}:{remote_zip}"], check=False, text=True, capture_output=True, timeout=args.scp_timeout_seconds) + if scp.returncode != 0: + raw_log = (scp.stdout or "") + (scp.stderr or "") + exit_code = scp.returncode + blocked_reason = "payload_transfer_failed" + else: + result = subprocess.run(command, check=False, text=True, capture_output=True, env={**os.environ, "PHASE4_DIRECT_RUN_ID": args.direct_run_id}, timeout=args.command_timeout_seconds) + raw_log = (result.stdout or "") + (result.stderr or "") + exit_code = result.returncode + else: + remote_command = _remote_precheck_command(direct_run_id=args.direct_run_id, command_id=safe_command_id, remote_root=args.remote_root, image=args.image, hip_visible_devices=args.hip_visible_devices, create_remote_root=args.create_remote_root) + command = _build_ssh_command(ssh_alias=args.ssh_alias, remote_command=remote_command) + enforce_command_request(command, workspace_relative_path="workspace/direct-node", workspace_id="workspace") + result = subprocess.run(command, check=False, text=True, capture_output=True, env={**os.environ, "PHASE4_DIRECT_RUN_ID": args.direct_run_id}, timeout=args.command_timeout_seconds) + raw_log = (result.stdout or "") + (result.stderr or "") + exit_code = result.returncode + except subprocess.TimeoutExpired as exc: + raw_log = f"TimeoutExpired: {exc}\n" + exit_code = 124 + blocked_reason = "target_unreachable" + except Exception as exc: # noqa: BLE001 + raw_log = f"{exc.__class__.__name__}: {exc}\n" + exit_code = 1 + blocked_reason = exc.__class__.__name__ + + raw_log_path.write_text(raw_log) + keys = _parse_key_values(raw_log) + if not blocked_reason: + blocked_reason = keys.get("PHASE4_BLOCKED_REASON", "") + component_reports = _inline_component_reports(raw_log) + if not blocked_reason and exit_code != 0 and not component_reports: + blocked_reason = "remote_command_failed" + completed_at = _iso() + assessment = _assessment_payload( + argv=sys.argv if argv is None else argv, + mode=args.mode, + ssh_alias=args.ssh_alias, + command_id=args.command_id, + safe_command_id=safe_command_id, + direct_run_id=args.direct_run_id, + payload_zip=args.payload_zip, + output_dir=args.output_dir, + raw_log_path=raw_log_path, + started_at=started_at, + completed_at=completed_at, + exit_code=exit_code, + keys=keys, + blocked_reason=blocked_reason, + component_reports=component_reports, + hip_visible_devices=args.hip_visible_devices, + remote_root=args.remote_root, + image=args.image, + ) + _write_json(assessment_path, assessment) + print(json.dumps({"assessment": str(assessment_path), "passed": assessment["passed"], "status": assessment["status"], "blocked_reason": assessment["blocked_reason"]}, sort_keys=True)) + return 0 if assessment["passed"] else (exit_code or 1) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/run_phase3_harbor_local_lifecycle.py b/scripts/rl_phase3/run_phase3_harbor_local_lifecycle.py new file mode 100644 index 00000000..198479eb --- /dev/null +++ b/scripts/rl_phase3/run_phase3_harbor_local_lifecycle.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Mapping + +SCHEMA = "bb.phase3.harbor_local_lifecycle.v1" +REPORT_ID = "phase3_harbor_local_lifecycle" +CLAIM_BOUNDARY = "local_harbor_api_lifecycle_only" + +JsonPayload = Mapping[str, Any] | list[Any] | str | int | float | bool | None +Transport = Callable[[str, str, JsonPayload, float], tuple[int, JsonPayload]] + + +def _sha(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _json_bytes(payload: JsonPayload) -> bytes: + return json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + + +def _endpoint_identity(base_url: str) -> dict[str, Any]: + parsed = urllib.parse.urlparse(base_url) + return { + "scheme": parsed.scheme, + "hostname": parsed.hostname or "", + "port": parsed.port, + "is_loopback": (parsed.hostname or "").lower() in {"127.0.0.1", "localhost", "::1"}, + } + + +def _default_transport(url: str, method: str, payload: JsonPayload, timeout_s: float) -> tuple[int, JsonPayload]: + data = None if payload is None else _json_bytes(payload) + headers = {"Accept": "application/json"} + if data is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=data, method=method, headers=headers) + try: + with urllib.request.urlopen(request, timeout=timeout_s) as response: # noqa: S310 - operator-provided local Harbor URL. + raw = response.read() + if not raw: + return response.status, None + return response.status, json.loads(raw.decode()) + except urllib.error.HTTPError as exc: + raw = exc.read() + if raw: + try: + body: JsonPayload = json.loads(raw.decode()) + except json.JSONDecodeError: + body = raw.decode(errors="replace") + else: + body = str(exc) + return exc.code, body + + +@dataclass +class RouteRecorder: + base_url: str + timeout_s: float + transport: Transport + + def call(self, route_key: str, method: str, path: str, payload: JsonPayload = None) -> dict[str, Any]: + url = self.base_url.rstrip("/") + path + started = time.monotonic() + request_payload = payload + request_sha = _sha(_json_bytes(request_payload)) if request_payload is not None else "" + try: + status_code, response_payload = self.transport(url, method, request_payload, self.timeout_s) + error = "" + except Exception as exc: # noqa: BLE001 - report must capture local daemon failures. + status_code = 0 + response_payload = None + error = f"{exc.__class__.__name__}: {exc}" + response_sha = _sha(_json_bytes(response_payload)) if response_payload is not None else "" + passed = 200 <= int(status_code) < 300 and not error + return { + "route": route_key, + "method": method, + "path": path, + "status": "passed" if passed else "failed", + "status_code": status_code, + "duration_ms": round((time.monotonic() - started) * 1000, 3), + "request_sha256": request_sha, + "response_sha256": response_sha, + "error": error, + "response": response_payload, + } + + +def _route_status(route: dict[str, Any] | None) -> str: + if route is None: + return "skipped" + return str(route.get("status") or "failed") + + +def build_lifecycle_report( + *, + base_url: str, + target_run_id: str, + task_name: str | None, + answer: str, + exec_cmd: str, + timeout_s: float, + transport: Transport = _default_transport, +) -> dict[str, Any]: + recorder = RouteRecorder(base_url=base_url, timeout_s=timeout_s, transport=transport) + route_results: dict[str, dict[str, Any] | None] = {} + + route_results["health"] = recorder.call("health", "GET", "/health") + route_results["metrics"] = recorder.call("metrics", "GET", "/metrics.json") + route_results["list_tasks"] = recorder.call("list_tasks", "GET", "/list_tasks") + + selected_task = task_name or "" + list_response = route_results["list_tasks"].get("response") if route_results["list_tasks"] else None + if not selected_task and isinstance(list_response, list) and list_response: + selected_task = str(list_response[0]) + + if selected_task: + route_results["score"] = recorder.call("score", "POST", "/score", {"task_name": selected_task, "answer": answer}) + route_results["trial_create"] = recorder.call("trial_create", "POST", "/trial/create", {"task_name": selected_task, "ttl_sec": 30}) + else: + route_results["score"] = None + route_results["trial_create"] = None + + trial_id = "" + create_response = route_results["trial_create"].get("response") if route_results["trial_create"] else None + if isinstance(create_response, Mapping): + trial_id = str(create_response.get("trial_id") or "") + + if trial_id: + route_results["trial_exec"] = recorder.call("trial_exec", "POST", f"/trial/{trial_id}/exec", {"cmd": exec_cmd, "timeout_sec": 30}) + route_results["trial_get"] = recorder.call("trial_get", "GET", f"/trial/{trial_id}") + route_results["trial_stats"] = recorder.call("trial_stats", "GET", "/trial_stats") + route_results["trial_finalize"] = recorder.call("trial_finalize", "POST", f"/trial/{trial_id}/finalize", {"answer": answer}) + else: + route_results["trial_exec"] = None + route_results["trial_get"] = None + route_results["trial_stats"] = None + route_results["trial_finalize"] = None + + delete_trial_id = "" + if selected_task: + route_results["trial_create_for_delete"] = recorder.call("trial_create_for_delete", "POST", "/trial/create", {"task_name": selected_task, "ttl_sec": 30}) + delete_response = route_results["trial_create_for_delete"].get("response") if route_results["trial_create_for_delete"] else None + if isinstance(delete_response, Mapping): + delete_trial_id = str(delete_response.get("trial_id") or "") + else: + route_results["trial_create_for_delete"] = None + + if delete_trial_id: + route_results["trial_delete"] = recorder.call("trial_delete", "DELETE", f"/trial/{delete_trial_id}") + else: + route_results["trial_delete"] = None + + required_routes = [ + "health", + "metrics", + "list_tasks", + "score", + "trial_create", + "trial_exec", + "trial_get", + "trial_stats", + "trial_finalize", + "trial_create_for_delete", + "trial_delete", + ] + route_statuses = {key: _route_status(route_results.get(key)) for key in required_routes} + failed_routes = [key for key, status in route_statuses.items() if status != "passed"] + blocked_reason = "" if not failed_routes else "harbor_local_lifecycle_failed:" + ",".join(failed_routes) + passed = not failed_routes + + sanitized_routes = { + key: {field: value for field, value in route.items() if field != "response"} if route else None + for key, route in route_results.items() + } + return { + "schema_version": SCHEMA, + "report_id": REPORT_ID, + "claim_boundary": CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "promotional": False, + "scorecard_update_allowed": False, + "passed": passed, + "blocked_reason": blocked_reason, + "base_url_identity": _endpoint_identity(base_url), + "target_endpoint_proven": False, + "auth_proven": False, + "docker_execution_proven": route_statuses.get("trial_exec") == "passed", + "selected_task": selected_task, + "task_source": { + "HARBOR_DATASET": "operator_environment", + "HARBOR_EXTRA_TASKS_DIR": "operator_environment", + }, + "route_statuses": route_statuses, + "routes": sanitized_routes, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--phase-dir", required=True, type=Path) + parser.add_argument("--base-url", default="http://127.0.0.1:5050") + parser.add_argument("--target-run-id", default="local-harbor-lifecycle-diagnostic") + parser.add_argument("--task-name", default="") + parser.add_argument("--answer", default="") + parser.add_argument("--exec-cmd", default="true") + parser.add_argument("--timeout-s", type=float, default=30.0) + parser.add_argument("--output", type=Path) + parser.add_argument("--emit-component-json", action="store_true") + args = parser.parse_args() + + report = build_lifecycle_report( + base_url=args.base_url, + target_run_id=args.target_run_id, + task_name=args.task_name or None, + answer=args.answer, + exec_cmd=args.exec_cmd, + timeout_s=args.timeout_s, + ) + output = args.output or args.phase_dir / "runs" / "harbor_local_lifecycle" / "phase3_harbor_local_lifecycle.json" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + if args.emit_component_json: + component = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "p3_aux_harbor_local_lifecycle", + "component": "harbor_local_lifecycle", + "claim_boundary": report["claim_boundary"], + "target_run_id": args.target_run_id, + "points": 0, + "passed": report["passed"], + "blocked_reason": report["blocked_reason"], + "lifecycle_report": report, + "artifact_paths": {"lifecycle_report": str(output)}, + "required_artifact_keys": ["lifecycle_report"], + "scorecard_update_allowed": False, + } + print("PHASE3_COMPONENT_REPORT_JSON=" + json.dumps(component, sort_keys=True)) + print(json.dumps({"report": str(output), "passed": report["passed"], "blocked_reason": report["blocked_reason"]}, sort_keys=True)) + return 0 if report["passed"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/run_phase3_live_provider_reports.py b/scripts/rl_phase3/run_phase3_live_provider_reports.py new file mode 100644 index 00000000..28310158 --- /dev/null +++ b/scripts/rl_phase3/run_phase3_live_provider_reports.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from breadboard.rl.phase3.evidence import sha256_file, write_phase3_json +from breadboard.rl.phase3.integrations import HARBOR_BLOCKED_CLAIM_BOUNDARY, HARBOR_CLAIM_BOUNDARY, run_harbor_service_proof + + +def _component(*, milestone_id: str, component: str, points: int, claim_boundary: str, blocked_claim_boundary: str, target_run_id: str, provider_report_path: Path, provider_report: dict) -> dict: + passed = provider_report.get("passed") is True + report = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": f"{milestone_id.lower()}_{component}", + "milestone_id": milestone_id, + "component": component, + "claim_boundary": claim_boundary if passed else blocked_claim_boundary, + "target_run_id": target_run_id, + "points": points, + "passed": passed, + "blocked_reason": "" if passed else str(provider_report.get("blocked_reason") or "provider_report_failed"), + "provider_report": provider_report, + "provider_kind": provider_report.get("provider_kind") or provider_report.get("attestation_backend"), + "input_hashes": {"provider_report": sha256_file(provider_report_path)}, + "artifact_paths": {"provider_report": str(provider_report_path)}, + "required_artifact_keys": ["provider_report"], + "scorecard_update_allowed": False, + } + return report + + + + +def _p3m8_retired_provider_report(*, target_run_id: str) -> dict: + return { + "schema_version": "bb.rl.phase3.retired_provider_milestone.v1", + "report_id": "phase3_p3m8_retired_provider_milestone", + "claim_boundary": "phase3_retired_provider_milestone_pending_rubric_change_scope", + "target_run_id": target_run_id, + "provider_kind": "none", + "blocked_reason": "retired_provider_milestone_pending_accepted_rubric_change", + "scorecard_update_allowed": False, + "passed": False, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--phase-dir", required=True, type=Path) + parser.add_argument("--target-run-id", required=True) + parser.add_argument("--rows-jsonl", type=Path) + parser.add_argument("--env-package", type=Path) + args = parser.parse_args() + out = args.phase_dir / "runs" / "live_provider_reports" + out.mkdir(parents=True, exist_ok=True) + retired_p3m8 = _p3m8_retired_provider_report(target_run_id=args.target_run_id) + retired_p3m8_path = out / "p3m8_retired_provider_milestone.json" + write_phase3_json(retired_p3m8_path, retired_p3m8) + env_package = args.env_package or out / "harbor_env_package_required.tar" + harbor = run_harbor_service_proof(env_package, target_run_id=args.target_run_id) + harbor_path = out / "harbor_service_proof.json" + write_phase3_json(harbor_path, harbor) + p3m8 = _component( + milestone_id="P3-M8", + component="retired_provider_milestone", + points=70, + claim_boundary="phase3_retired_provider_milestone_pending_rubric_change_scope", + blocked_claim_boundary="phase3_retired_provider_milestone_pending_rubric_change_scope", + target_run_id=args.target_run_id, + provider_report_path=retired_p3m8_path, + provider_report=retired_p3m8, + ) + p3m9 = _component( + milestone_id="P3-M9", + component="harbor_nemo_gym", + points=60, + claim_boundary=HARBOR_CLAIM_BOUNDARY, + blocked_claim_boundary=HARBOR_BLOCKED_CLAIM_BOUNDARY, + target_run_id=args.target_run_id, + provider_report_path=harbor_path, + provider_report=harbor, + ) + write_phase3_json(out / "P3-M8_retired_provider_milestone.json", p3m8) + write_phase3_json(out / "P3-M9_harbor_nemo_gym.json", p3m9) + print(json.dumps({"P3-M8": p3m8["passed"], "P3-M9": p3m9["passed"], "output_dir": str(out)}, sort_keys=True)) + return 0 if p3m8["passed"] and p3m9["passed"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/run_phase3_nemo_agentloop_smoke.py b/scripts/rl_phase3/run_phase3_nemo_agentloop_smoke.py new file mode 100644 index 00000000..2bb4d1fd --- /dev/null +++ b/scripts/rl_phase3/run_phase3_nemo_agentloop_smoke.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +SMOKE_SCHEMA = "bb.rl.phase3.nemo_agentloop_smoke.v1" +SMOKE_ID = "phase3_nemo_agentloop_smoke" +SMOKE_BOUNDARY = "phase3_nemo_agentloop_smoke_wrapper_target_scope" +SMOKE_BLOCKED_BOUNDARY = "phase3_nemo_agentloop_smoke_blocked_scope" + + +def _sha(payload: bytes) -> str: + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _canonical_json(payload: Mapping[str, Any]) -> bytes: + return json.dumps(dict(payload), sort_keys=True, separators=(",", ":")).encode() + + +@dataclass(frozen=True) +class ToolCall: + name: str + arguments: dict[str, Any] + + +def _score_tool_call(expected: Mapping[str, Any], actual: ToolCall | None) -> float: + if expected.get("type") == "message": + return 1.0 if actual is None else 0.0 + if expected.get("type") != "function_call" or actual is None: + return 0.0 + expected_name = str(expected.get("name") or "") + expected_args = expected.get("arguments") if isinstance(expected.get("arguments"), Mapping) else {} + return 1.0 if actual.name == expected_name and dict(actual.arguments) == dict(expected_args) else 0.0 + + +def _row() -> dict[str, Any]: + return { + "id": "phase3-nemo-agentloop-smoke", + "messages": [ + {"role": "system", "content": "Call exactly one tool when the answer requires a weather lookup."}, + {"role": "user", "content": "What is the weather in Paris?"}, + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Return the current weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ], + "expected_action": {"type": "function_call", "name": "get_weather", "arguments": {"city": "Paris"}}, + } + + +def _dependency_status(wrapper_dir: Path) -> dict[str, Any]: + status = {"wrapper_dir": str(wrapper_dir), "nemo_gym_loop_path": str(wrapper_dir / "src" / "zyphra_verl" / "nemo_gym_loop.py")} + try: + source = (wrapper_dir / "src" / "zyphra_verl" / "nemo_gym_loop.py").read_text() + except OSError as exc: + status.update({"canonical_agentloop_source_present": False, "blocked_reason": exc.__class__.__name__}) + return status + required_terms = ("register(\"nemo_gym_tool_use\"", "ToolParser", "ToolCallComparator", "reward_score") + status["canonical_agentloop_source_present"] = all(term in source for term in required_terms) + status["nemo_gym_loop_sha256"] = _sha(source.encode()) + status["required_source_terms"] = list(required_terms) + try: + import verl # type: ignore # noqa: F401 + import nemo_gym # type: ignore # noqa: F401 + except Exception as exc: # noqa: BLE001 + status["canonical_runtime_imports_present"] = False + status["runtime_import_blocked_reason"] = exc.__class__.__name__ + else: + status["canonical_runtime_imports_present"] = True + status["runtime_import_blocked_reason"] = "" + return status + + +def build_smoke_report( + *, + wrapper_dir: Path, + target_run_id: str, + require_canonical_runtime: bool, + local_diagnostic: bool, + canonical_mode: bool = False, +) -> dict[str, Any]: + row = _row() + expected = row["expected_action"] + gold = ToolCall("get_weather", {"city": "Paris"}) + wrong_name = ToolCall("get_forecast", {"city": "Paris"}) + wrong_args = ToolCall("get_weather", {"city": "Lyon"}) + controls = { + "gold_reward": _score_tool_call(expected, gold), + "wrong_name_reward": _score_tool_call(expected, wrong_name), + "wrong_args_reward": _score_tool_call(expected, wrong_args), + "missing_call_reward": _score_tool_call(expected, None), + } + dependency = _dependency_status(wrapper_dir) + hashes = { + "row_sha256": _sha(_canonical_json(row)), + "messages_sha256": _sha(_canonical_json({"messages": row["messages"]})), + "tools_sha256": _sha(_canonical_json({"tools": row["tools"]})), + "expected_action_sha256": _sha(_canonical_json(expected)), + "gold_tool_call_sha256": _sha(_canonical_json({"name": gold.name, "arguments": gold.arguments})), + "negative_tool_call_sha256": _sha(_canonical_json({"name": wrong_name.name, "arguments": wrong_name.arguments})), + } + control_passed = controls == { + "gold_reward": 1.0, + "wrong_name_reward": 0.0, + "wrong_args_reward": 0.0, + "missing_call_reward": 0.0, + } + canonical_source = dependency.get("canonical_agentloop_source_present") is True + canonical_runtime = dependency.get("canonical_runtime_imports_present") is True + canonical_required = canonical_mode or require_canonical_runtime + canonical_agentloop_executed = False + canonical_tool_parser_used = False + canonical_comparator_used = False + canonical_reward_score_observed = False + diagnostic_passed = control_passed and canonical_source and (canonical_runtime or not require_canonical_runtime) + passed = ( + canonical_required + and control_passed + and canonical_source + and canonical_runtime + and canonical_agentloop_executed + and canonical_tool_parser_used + and canonical_comparator_used + and canonical_reward_score_observed + and not local_diagnostic + ) + blocked_reasons = [] + if not control_passed: + blocked_reasons.append("reward_control_failed") + if not canonical_source: + blocked_reasons.append("canonical_agentloop_source_missing") + if canonical_required and not canonical_runtime: + blocked_reasons.append("canonical_runtime_imports_missing") + if canonical_required and canonical_runtime and not canonical_agentloop_executed: + blocked_reasons.append("canonical_agentloop_execution_missing") + if not canonical_required: + blocked_reasons.append("diagnostic_only_not_promotional") + if local_diagnostic: + blocked_reasons.append("local_diagnostic_not_promotional") + return { + "schema_version": SMOKE_SCHEMA, + "report_id": SMOKE_ID, + "component": "nemo_gym_agentloop_smoke", + "claim_boundary": SMOKE_BOUNDARY if passed else SMOKE_BLOCKED_BOUNDARY, + "target_run_id": target_run_id, + "mode": "canonical" if canonical_required else "diagnostic", + "row": row, + "controls": controls, + "hashes": hashes, + "dependency_status": dependency, + "canonical_runtime_required": canonical_required, + "canonical_agentloop_executed": canonical_agentloop_executed, + "canonical_tool_parser_used": canonical_tool_parser_used, + "canonical_comparator_used": canonical_comparator_used, + "canonical_reward_score_observed": canonical_reward_score_observed, + "breadboard_toy_reward_used": True, + "diagnostic_passed": diagnostic_passed, + "local_diagnostic": local_diagnostic, + "promotional": False, + "scorecard_update_allowed": False, + "passed": passed, + "blocked_reason": ";".join(blocked_reasons), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--phase-dir", required=True, type=Path) + parser.add_argument("--wrapper-dir", required=True, type=Path) + parser.add_argument("--target-run-id", required=True) + parser.add_argument("--require-canonical-runtime", action="store_true") + parser.add_argument("--canonical", action="store_true") + parser.add_argument("--local-diagnostic", action="store_true") + parser.add_argument("--emit-component-json", action="store_true") + args = parser.parse_args() + report = build_smoke_report( + wrapper_dir=args.wrapper_dir, + target_run_id=args.target_run_id, + require_canonical_runtime=args.require_canonical_runtime, + local_diagnostic=args.local_diagnostic, + canonical_mode=args.canonical, + ) + output = args.phase_dir / "runs" / "nemo_agentloop_smoke" / "phase3_nemo_agentloop_smoke.json" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + if args.emit_component_json: + component = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "p3_aux_nemo_agentloop_smoke", + "component": "nemo_gym_agentloop_smoke", + "claim_boundary": report["claim_boundary"], + "target_run_id": args.target_run_id, + "points": 0, + "passed": report["passed"], + "blocked_reason": report["blocked_reason"], + "smoke_report": report, + "input_hashes": {"smoke_report": _sha(output.read_bytes())}, + "artifact_paths": {"smoke_report": str(output)}, + "required_artifact_keys": ["smoke_report"], + "scorecard_update_allowed": False, + } + print("PHASE3_COMPONENT_REPORT_JSON=" + json.dumps(component, sort_keys=True)) + print(json.dumps({"report": str(output), "passed": report["passed"], "blocked_reason": report["blocked_reason"]}, sort_keys=True)) + return 0 if report["passed"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/run_phase3_observability_scheduler_store.py b/scripts/rl_phase3/run_phase3_observability_scheduler_store.py new file mode 100644 index 00000000..ce83e3d3 --- /dev/null +++ b/scripts/rl_phase3/run_phase3_observability_scheduler_store.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from breadboard.rl.phase3.evidence import normalize_phase3_metric_sources, sha256_file, write_phase3_json +from breadboard.rl.phase3.observability_live import LOCAL_OBJECT_STORE_BACKENDS, build_live_observability_report + + + +def _load(path: Path | None) -> dict: + return json.loads(path.read_text()) if path and path.exists() else {} + + + + +def _object_store_blocker_state(path: Path | None) -> dict: + if not path or not path.exists(): + return { + "object_store_metrics_present": False, + "object_store_backend": "", + "object_store_is_local": None, + "production_object_store_endpoint_present": False, + } + metrics = _load(path) + backend = str(metrics.get("object_store") or metrics.get("backend") or "") + is_local = backend in LOCAL_OBJECT_STORE_BACKENDS + return { + "object_store_metrics_present": True, + "object_store_backend": backend, + "object_store_is_local": is_local, + "production_object_store_endpoint_present": bool(backend and not is_local), + } + + +def _p3m11_blocker_evidence(*, target_run_id: str, missing: list[str], object_store_metrics: Path | None) -> dict: + return { + "schema_version": "bb.rl.phase3.p3m11_blocker_evidence.v1", + "report_id": "p3m11_blocker_evidence", + "target_run_id": target_run_id, + "missing_inputs": missing, + "controller_env_verifier_base_url_present": bool(os.environ.get("BREADBOARD_VERIFIER_BASE_URL")), + "controller_env_verifier_token_present": bool(os.environ.get("BREADBOARD_VERIFIER_TOKEN")), + **_object_store_blocker_state(object_store_metrics), + } + + + + +def _component(*, target_run_id: str, output_dir: Path, passed: bool, blocked_reason: str, evidence: dict, artifact_paths: dict[str, Path]) -> dict: + input_hashes = {key: sha256_file(path) for key, path in artifact_paths.items()} + return { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "p3-m11_observability_scheduler_store", + "milestone_id": "P3-M11", + "component": "observability_scheduler_store", + "claim_boundary": "phase3_live_observability_object_store_scheduler_scope" if passed else "phase3_observability_scheduler_store_blocked_scope", + "target_run_id": target_run_id, + "points": 80, + "passed": passed, + "blocked_reason": blocked_reason, + "observability_evidence": evidence, + "input_hashes": input_hashes, + "artifact_paths": {key: str(path) for key, path in artifact_paths.items()}, + "required_artifact_keys": list(artifact_paths), + "scorecard_update_allowed": False, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--phase-dir", required=True, type=Path) + parser.add_argument("--target-run-id", required=True) + parser.add_argument("--slurm-metrics", type=Path) + parser.add_argument("--gpu-metrics", type=Path) + parser.add_argument("--verifier-metrics", type=Path) + parser.add_argument("--service-metrics", type=Path) + parser.add_argument("--object-store-metrics", type=Path) + parser.add_argument("--scheduler-metrics", type=Path) + parser.add_argument("--budget-caps", type=Path) + args = parser.parse_args() + output_dir = args.phase_dir / "runs" / "observability_scheduler_store_runner" + output_dir.mkdir(parents=True, exist_ok=True) + required_inputs = { + "slurm_metrics": args.slurm_metrics, + "gpu_metrics": args.gpu_metrics, + "verifier_metrics": args.verifier_metrics, + "service_metrics": args.service_metrics, + "object_store_metrics": args.object_store_metrics, + "scheduler_metrics": args.scheduler_metrics, + "budget_caps": args.budget_caps, + } + missing = [key for key, path in required_inputs.items() if not path or not path.exists()] + copied: dict[str, Path] = {} + for key, path in required_inputs.items(): + if path and path.exists(): + dst = output_dir / f"{key}.json" + dst.write_text(path.read_text()) + copied[key] = dst + if missing: + blocker = output_dir / "p3m11_blocker_evidence.json" + blocker_evidence = _p3m11_blocker_evidence(target_run_id=args.target_run_id, missing=missing, object_store_metrics=args.object_store_metrics) + write_phase3_json(blocker, blocker_evidence) + report = _component( + target_run_id=args.target_run_id, + output_dir=output_dir, + passed=False, + blocked_reason="missing live observability inputs: " + ",".join(missing), + evidence=blocker_evidence, + artifact_paths={"blocker_evidence": blocker, **copied}, + ) + write_phase3_json(output_dir / "P3-M11_observability_scheduler_store.json", report) + print(json.dumps({"passed": False, "blocked_reason": report["blocked_reason"], "report": str(output_dir / "P3-M11_observability_scheduler_store.json")}, sort_keys=True)) + return 2 + metrics = normalize_phase3_metric_sources( + { + "slurm": _load(args.slurm_metrics), + "gpu": _load(args.gpu_metrics), + "verifier": _load(args.verifier_metrics), + "service": _load(args.service_metrics), + "object_store": _load(args.object_store_metrics), + "scheduler": _load(args.scheduler_metrics), + } + ) + slurm = metrics["slurm"] + gpu = metrics["gpu"] + verifier = metrics["verifier"] + service = metrics["service"] + object_store = metrics["object_store"] + scheduler = metrics["scheduler"] + live = build_live_observability_report( + target_run_id=args.target_run_id, + slurm_metrics=slurm, + gpu_metrics=gpu, + verifier_metrics=verifier, + service_metrics=service, + object_store_metrics=object_store, + budget_caps=_load(args.budget_caps), + scheduler_metrics=scheduler, + ) + live_path = output_dir / "live_observability_report.json" + write_phase3_json(live_path, live) + semantic_errors = list(live.get("errors", [])) + report = _component( + target_run_id=args.target_run_id, + output_dir=output_dir, + passed=not semantic_errors, + blocked_reason=";".join(semantic_errors), + evidence=live, + artifact_paths={"live_observability_report": live_path, **copied}, + ) + write_phase3_json(output_dir / "P3-M11_observability_scheduler_store.json", report) + print(json.dumps({"passed": report["passed"], "blocked_reason": report["blocked_reason"], "report": str(output_dir / "P3-M11_observability_scheduler_store.json")}, sort_keys=True)) + return 0 if report["passed"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/run_phase3_self_hosted_harbor.py b/scripts/rl_phase3/run_phase3_self_hosted_harbor.py new file mode 100644 index 00000000..5a479ab1 --- /dev/null +++ b/scripts/rl_phase3/run_phase3_self_hosted_harbor.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import argparse +import json +import os +import sys +import tempfile +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from breadboard.rl.phase3.evidence import sha256_file, write_phase3_json +from breadboard.rl.phase3.integrations import HARBOR_BLOCKED_CLAIM_BOUNDARY, run_harbor_service_proof + + +SCHEMA = "bb.rl.phase3.self_hosted_harbor_facade.v1" +REPORT_ID = "phase3_self_hosted_harbor_facade" +LOCAL_FACADE_CLAIM_BOUNDARY = "local_harbor_compatible_facade_lifecycle_only" +TOKEN = "phase3-self-hosted-harbor-token" +TASK_NAME = "phase3-harbor-smoke" + + +class HarborFacadeHandler(BaseHTTPRequestHandler): + server_version = "Phase3SelfHostedHarbor/1.0" + + def _server_state(self) -> dict[str, Any]: + return self.server.state # type: ignore[attr-defined] + + def _send_json(self, status: int, payload: Any) -> None: + body = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _read_json(self) -> dict[str, Any]: + length = int(self.headers.get("Content-Length") or "0") + if length <= 0: + return {} + raw = self.rfile.read(length) + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return {} + return payload if isinstance(payload, dict) else {} + + def _authorized(self) -> bool: + expected = "Bearer " + str(self._server_state()["token"]) + return self.headers.get("Authorization") == expected + + def _require_auth(self) -> bool: + if self._authorized(): + return True + self._send_json(401, {"ok": False, "error": "unauthorized"}) + return False + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 - stdlib hook name. + return + + def do_GET(self) -> None: # noqa: N802 - stdlib hook name. + if not self._require_auth(): + return + state = self._server_state() + if self.path == "/health": + self._send_json(200, {"ok": True, "dataset": state["dataset"], "backend": "self_hosted_harbor_facade"}) + elif self.path == "/metrics.json": + self._send_json(200, {"ok": True, "requests": state["requests"], "trials": len(state["trials"])}) + elif self.path == "/list_tasks": + self._send_json(200, [state["task_name"]]) + elif self.path.startswith("/trial/"): + trial_id = self.path.removeprefix("/trial/") + trial = state["trials"].get(trial_id) + if not trial: + self._send_json(404, {"ok": False, "error": "trial_not_found"}) + else: + self._send_json(200, trial) + else: + self._send_json(404, {"ok": False, "error": "not_found"}) + + def do_POST(self) -> None: # noqa: N802 - stdlib hook name. + if not self._require_auth(): + return + state = self._server_state() + state["requests"] += 1 + payload = self._read_json() + if self.path == "/score": + passed = payload.get("task_name") == state["task_name"] + self._send_json(200, {"ok": True, "score": 1.0 if passed else 0.0, "passed": passed}) + elif self.path == "/trial/create": + if payload.get("task_name") != state["task_name"]: + self._send_json(404, {"ok": False, "error": "task_not_found"}) + return + trial_id = uuid.uuid4().hex + state["trials"][trial_id] = {"trial_id": trial_id, "task_name": state["task_name"], "status": "created", "stdout": "", "answer": "", "n_exec_calls": 0} + self._send_json(200, {"ok": True, "trial_id": trial_id, "task_name": state["task_name"]}) + elif self.path.startswith("/trial/") and self.path.endswith("/exec"): + trial_id = self.path.split("/")[2] + trial = state["trials"].get(trial_id) + if not trial: + self._send_json(404, {"ok": False, "error": "trial_not_found"}) + return + trial["status"] = "executed" + trial["stdout"] = "phase3-harbor-proof" + trial["cmd"] = payload.get("cmd", "") + trial["n_exec_calls"] = int(trial.get("n_exec_calls") or 0) + 1 + self._send_json(200, {"ok": True, "trial_id": trial_id, "stdout": trial["stdout"], "returncode": 0}) + elif self.path.startswith("/trial/") and self.path.endswith("/finalize"): + trial_id = self.path.split("/")[2] + trial = state["trials"].get(trial_id) + if not trial: + self._send_json(404, {"ok": False, "error": "trial_not_found"}) + return + trial["status"] = "finalized" + trial["answer"] = payload.get("answer", "") + self._send_json(200, {"ok": True, "trial_id": trial_id, "reward": 1.0, "status": trial["status"]}) + else: + self._send_json(404, {"ok": False, "error": "not_found"}) + + +def _start_server(*, token: str, task_name: str) -> tuple[ThreadingHTTPServer, str]: + server = ThreadingHTTPServer(("127.0.0.1", 0), HarborFacadeHandler) + server.state = {"token": token, "task_name": task_name, "dataset": "phase3-self-hosted", "requests": 0, "trials": {}} # type: ignore[attr-defined] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + host, port = server.server_address[:2] + return server, f"http://{host}:{port}" + + +def _component(*, target_run_id: str, provider_report_path: Path, provider_report: dict[str, Any]) -> dict[str, Any]: + facade_passed = provider_report.get("passed") is True + return { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "p3_aux_harbor_local_facade", + "milestone_id": "P3-M9", + "component": "harbor_local_facade", + "claim_boundary": HARBOR_BLOCKED_CLAIM_BOUNDARY, + "target_run_id": target_run_id, + "points": 0, + "passed": False, + "blocked_reason": "local_facade_not_target_harbor_nemo_evidence" if facade_passed else str(provider_report.get("blocked_reason") or "harbor_self_hosted_report_failed"), + "provider_report": provider_report, + "provider_kind": "local_harbor_facade", + "input_hashes": {"provider_report": sha256_file(provider_report_path)}, + "artifact_paths": {"provider_report": str(provider_report_path)}, + "required_artifact_keys": ["provider_report"], + "scorecard_update_allowed": False, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--phase-dir", required=True, type=Path) + parser.add_argument("--target-run-id", required=True) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--task-name", default=TASK_NAME) + args = parser.parse_args() + + out = args.output_dir or args.phase_dir / "runs" / "self_hosted_harbor" + out.mkdir(parents=True, exist_ok=True) + env_package = out / "self_hosted_harbor_env_package.tar" + env_package.write_bytes(b"phase3 self-hosted harbor env package\n") + + server, base_url = _start_server(token=TOKEN, task_name=args.task_name) + previous = {key: os.environ.get(key) for key in ("BREADBOARD_HARBOR_BASE_URL", "BREADBOARD_HARBOR_TOKEN", "BREADBOARD_HARBOR_TASK_NAME", "BREADBOARD_HARBOR_ALLOW_LOCAL")} + os.environ["BREADBOARD_HARBOR_BASE_URL"] = base_url + os.environ["BREADBOARD_HARBOR_TOKEN"] = TOKEN + os.environ["BREADBOARD_HARBOR_TASK_NAME"] = args.task_name + os.environ["BREADBOARD_HARBOR_ALLOW_LOCAL"] = "1" + try: + time.sleep(0.05) + provider_report = run_harbor_service_proof(env_package, target_run_id=args.target_run_id, task_name=args.task_name) + finally: + server.shutdown() + server.server_close() + for key, value in previous.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + provider_report["self_hosted"] = True + provider_report["self_hosted_base_url"] = base_url + provider_report["claim_boundary"] = LOCAL_FACADE_CLAIM_BOUNDARY + provider_report["promotional"] = False + provider_report["target_endpoint_proven"] = False + provider_report["auth_proven"] = False + provider_path = out / "harbor_service_proof.json" + write_phase3_json(provider_path, provider_report) + component = _component(target_run_id=args.target_run_id, provider_report_path=provider_path, provider_report=provider_report) + component_path = out / "P3-M9_harbor_nemo_gym.json" + write_phase3_json(component_path, component) + local_facade_passed = provider_report.get("passed") is True + summary = { + "schema_version": SCHEMA, + "report_id": REPORT_ID, + "target_run_id": args.target_run_id, + "base_url": base_url, + "task_name": args.task_name, + "provider_report": str(provider_path), + "component_report": str(component_path), + "local_facade_passed": local_facade_passed, + "passed": local_facade_passed, + "promotional": False, + "blocked_reason": "" if local_facade_passed else component["blocked_reason"], + } + write_phase3_json(out / "phase3_self_hosted_harbor_facade.json", summary) + print(json.dumps(summary, sort_keys=True)) + return 0 if local_facade_passed else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/run_phase3_target_command.py b/scripts/rl_phase3/run_phase3_target_command.py new file mode 100644 index 00000000..08051781 --- /dev/null +++ b/scripts/rl_phase3/run_phase3_target_command.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shlex +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + + +from breadboard.rl.phase3.evidence import PHASE3_COMMAND_LOG_MANIFEST_SCHEMA, PHASE3_TARGET_RUN_ID_PATTERN, sha256_file +from breadboard.rl.phase3.security_enforcement import enforce_command_request + + +def _inline_report_passed(report: object) -> bool: + return isinstance(report, dict) and report.get("passed") is True + + +def _iso() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _load_manifest(path: Path, target_run_id: str) -> dict: + empty = {"schema_version": PHASE3_COMMAND_LOG_MANIFEST_SCHEMA, "target_run_id": target_run_id, "commands": []} + if not path.exists(): + return empty + manifest = json.loads(path.read_text()) + if manifest.get("target_run_id") != target_run_id: + return empty + return manifest + + +def _load_attempts_manifest(path: Path, target_run_id: str) -> dict: + empty = { + "schema_version": "bb.rl.phase3.command_attempts_manifest.v1", + "target_run_id": target_run_id, + "attempts": [], + } + if not path.exists(): + return empty + manifest = json.loads(path.read_text()) + if manifest.get("target_run_id") != target_run_id: + return empty + return manifest + +def _write_manifest(path: Path, manifest: dict) -> None: + path.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n") + +def _build_remote_command( + *, + target_run_id: str, + command_id: str, + remote_zip: str, + partition: str, + job_name: str, + gres: str = "gpu:8", + mem: str | None = None, + nodelist: str | None = None, + constraint: str | None = None, + reservation: str | None = None, + qos: str | None = None, +) -> str: + slurm_payload = "echo PHASE3_NODE=$(hostname); echo PHASE3_SLURM_JOB_ID=${SLURM_JOB_ID:-}; ./run.sh" + srun_args = [ + f"--partition={shlex.quote(partition)}", + f"--job-name={shlex.quote(job_name)}", + f"--gres={shlex.quote(gres)}", + ] + if mem: + srun_args.append(f"--mem={shlex.quote(mem)}") + for flag, value in ( + ("--nodelist", nodelist), + ("--constraint", constraint), + ("--reservation", reservation), + ("--qos", qos), + ): + if value: + srun_args.append(f"{flag}={shlex.quote(value)}") + return ( + "set -euo pipefail; " + f"export PHASE3_TARGET_RUN_ID={shlex.quote(target_run_id)}; " + f"export PHASE3_COMMAND_ID={shlex.quote(command_id)}; " + f"mkdir -p /shared/bb-p3-${{USER:-root}}; " + f"WORK=$(mktemp -d /shared/bb-p3-${{USER:-root}}/{shlex.quote(command_id)}.XXXXXX); " + f"unzip -q {shlex.quote(remote_zip)} -d \"$WORK\"; " + "cd \"$WORK\"; " + "test -x ./run.sh; " + f"srun {' '.join(srun_args)} bash -lc {shlex.quote(slurm_payload)}" + ) + + +def _build_ssh_command(*, ssh_alias: str, remote_command: str) -> list[str]: + return ["ssh", ssh_alias, f"bash -lc {shlex.quote(remote_command)}"] + +def _safe_artifact_name(value: object, *, fallback: str) -> str: + raw = str(value or fallback) + stem = "".join(ch if ch.isalnum() or ch in {"_", "-"} else "_" for ch in raw).strip("._-") or fallback + if stem == raw: + return stem + digest = hashlib.sha256(raw.encode()).hexdigest()[:12] + return f"{stem}-{digest}" + + +def _valid_requested_target_run_id(target_run_id: str) -> bool: + return bool( + re.match(PHASE3_TARGET_RUN_ID_PATTERN, target_run_id) + or re.match(r"^\d{8}T\d{6}Z-slurm-pending$", target_run_id) + ) + + + +def _validated_slurm_option(value: str | None, *, name: str) -> str | None: + if value is None: + return None + if not re.fullmatch(r"[A-Za-z0-9_.:,=+\-\[\]]{1,256}", value): + raise ValueError(f"--{name} contains unsupported characters") + return value + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--ssh-alias", required=True) + parser.add_argument("--partition", required=True) + parser.add_argument("--job-name", required=True) + parser.add_argument("--command-id", required=True) + parser.add_argument("--target-run-id", required=True) + parser.add_argument("--payload-zip", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--gres", default="gpu:8") + parser.add_argument("--mem") + parser.add_argument("--nodelist") + parser.add_argument("--constraint") + parser.add_argument("--reservation") + parser.add_argument("--qos") + parser.add_argument("--scp-timeout-seconds", type=int, default=20) + args = parser.parse_args(argv) + try: + gres = _validated_slurm_option(args.gres, name="gres") + nodelist = _validated_slurm_option(args.nodelist, name="nodelist") + mem = _validated_slurm_option(args.mem, name="mem") + constraint = _validated_slurm_option(args.constraint, name="constraint") + reservation = _validated_slurm_option(args.reservation, name="reservation") + qos = _validated_slurm_option(args.qos, name="qos") + except ValueError as exc: + parser.error(str(exc)) + if args.scp_timeout_seconds < 1: + parser.error("--scp-timeout-seconds must be positive") + if not _valid_requested_target_run_id(args.target_run_id): + parser.error("--target-run-id must be concrete Phase 3 id or end in -slurm-pending") + args.output_dir.mkdir(parents=True, exist_ok=True) + safe_command_id = _safe_artifact_name(args.command_id, fallback="phase3_command") + log_dir = args.output_dir / "command_logs" + log_dir.mkdir(parents=True, exist_ok=True) + raw_log_path = log_dir / f"{safe_command_id}.log" + started_at = _iso() + remote_zip = f"/tmp/{safe_command_id}.zip" + remote_command = _build_remote_command( + target_run_id=args.target_run_id, + command_id=safe_command_id, + remote_zip=remote_zip, + partition=args.partition, + job_name=args.job_name, + gres=gres, + mem=mem, + nodelist=nodelist, + constraint=constraint, + reservation=reservation, + qos=qos, + ) + command = _build_ssh_command(ssh_alias=args.ssh_alias, remote_command=remote_command) + exit_code = 1 + slurm_job_id = "" + node = "" + final_target_run_id = args.target_run_id + blocked_reason = "" + inline_reports: list[dict] = [] + try: + enforce_command_request(command, workspace_relative_path="workspace/slurm", workspace_id="workspace") + if not args.payload_zip.exists(): + raise FileNotFoundError(args.payload_zip) + scp = subprocess.run(["scp", str(args.payload_zip), f"{args.ssh_alias}:{remote_zip}"], check=False, text=True, capture_output=True, timeout=args.scp_timeout_seconds) + if scp.returncode != 0: + raw_log_path.write_text((scp.stdout or "") + (scp.stderr or "")) + exit_code = scp.returncode + else: + run_env = {**os.environ, "PHASE3_TARGET_RUN_ID": args.target_run_id} + result = subprocess.run(command, check=False, text=True, capture_output=True, env=run_env, timeout=3600) + raw_log_path.write_text((result.stdout or "") + (result.stderr or "")) + exit_code = result.returncode + for line in (result.stdout or "").splitlines(): + if line.startswith("PHASE3_NODE="): + node = line.split("=", 1)[1].strip() + if line.startswith("PHASE3_SLURM_JOB_ID="): + slurm_job_id = line.split("=", 1)[1].strip() + if line.startswith("PHASE3_INTROSPECTION_REPORT=") or line.startswith("PHASE3_COMPONENT_REPORT_JSON="): + try: + inline_reports.append(json.loads(line.split("=", 1)[1])) + except json.JSONDecodeError: + blocked_reason = "invalid_inline_report" + if slurm_job_id and args.target_run_id.endswith("-pending"): + final_target_run_id = args.target_run_id.removesuffix("pending") + slurm_job_id + except subprocess.TimeoutExpired as exc: + blocked_reason = "target_unreachable" + raw_log_path.write_text(f"TimeoutExpired: {exc}\n") + exit_code = 124 + except Exception as exc: # noqa: BLE001 + blocked_reason = exc.__class__.__name__ + raw_log_path.write_text(f"{exc.__class__.__name__}: {exc}\n") + exit_code = 1 + completed_at = _iso() + manifest_path = args.output_dir / "phase3_command_log_manifest.json" + attempts_path = args.output_dir / "phase3_command_attempts_manifest.json" + component_failed_count = sum(1 for report in inline_reports if not _inline_report_passed(report)) + component_blocked_reasons = [ + str(report.get("blocked_reason") or "inline_component_not_passed") + for report in inline_reports + if not _inline_report_passed(report) and isinstance(report, dict) + ] + component_passed = component_failed_count == 0 + effective_blocked_reason = blocked_reason or ("inline_component_failed" if component_failed_count else "") + passed = exit_code == 0 and bool(slurm_job_id) and bool(node) and not effective_blocked_reason and component_passed + row = { + "command_id": args.command_id, + "argv": sys.argv if argv is None else argv, + "raw_log_path": str(raw_log_path.relative_to(args.output_dir)), + "raw_log_sha256": sha256_file(raw_log_path), + "slurm_job_id": slurm_job_id, + "target_run_id": final_target_run_id, + "node": node, + "started_at": started_at, + "completed_at": completed_at, + "exit_code": exit_code, + "status": "passed" if passed else "failed", + "blocked_reason": effective_blocked_reason, + "component_passed": component_passed, + "component_failed_count": component_failed_count, + "component_blocked_reasons": component_blocked_reasons, + } + if not blocked_reason: + for report in inline_reports: + report["target_run_id"] = final_target_run_id + report_id = str(report.get("report_id") or args.command_id) + safe_report_id = _safe_artifact_name(report_id, fallback=safe_command_id) + component_dir = _safe_artifact_name(report.get("component"), fallback=safe_report_id) + report_dir = args.output_dir / component_dir + report_dir.mkdir(parents=True, exist_ok=True) + report_path = report_dir / f"{safe_report_id}.json" + artifact_paths = report.setdefault("artifact_paths", {}) + if isinstance(artifact_paths, dict): + artifact_paths.setdefault("component_report_json", str(report_path.resolve())) + artifact_paths.setdefault("command_log", str(raw_log_path.resolve())) + report_path.write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + if blocked_reason == "target_unreachable": + blocked_report = { + "schema_version": "bb.rl.phase3.target_command_blocked.v1", + "report_id": f"{args.command_id}_blocked", + "target_run_id": final_target_run_id, + "blocked_reason": "target_unreachable", + "scorecard_update_allowed": False, + "passed": False, + } + (args.output_dir / f"{safe_command_id}_blocked.json").write_text(json.dumps(blocked_report, sort_keys=True, indent=2) + "\n") + if passed: + manifest = _load_manifest(manifest_path, final_target_run_id) + manifest["target_run_id"] = final_target_run_id + manifest["commands"] = [existing for existing in manifest.get("commands", []) if existing.get("command_id") != args.command_id] + manifest["commands"].append(row) + _write_manifest(manifest_path, manifest) + if attempts_path.exists(): + attempts = _load_attempts_manifest(attempts_path, final_target_run_id) + attempts["target_run_id"] = final_target_run_id + attempts["attempts"] = [existing for existing in attempts.get("attempts", []) if existing.get("command_id") != args.command_id] + if attempts["attempts"]: + _write_manifest(attempts_path, attempts) + else: + attempts_path.unlink() + else: + if manifest_path.exists(): + manifest = _load_manifest(manifest_path, final_target_run_id) + manifest["commands"] = [existing for existing in manifest.get("commands", []) if existing.get("command_id") != args.command_id] + if manifest["commands"]: + _write_manifest(manifest_path, manifest) + else: + manifest_path.unlink() + attempts = _load_attempts_manifest(attempts_path, final_target_run_id) + attempts["target_run_id"] = final_target_run_id + attempts["attempts"] = [existing for existing in attempts.get("attempts", []) if existing.get("command_id") != args.command_id] + attempts["attempts"].append(row) + _write_manifest(attempts_path, attempts) + return 0 if passed else (exit_code or 1) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/run_phase3_target_validation.py b/scripts/rl_phase3/run_phase3_target_validation.py new file mode 100644 index 00000000..d4e34ad3 --- /dev/null +++ b/scripts/rl_phase3/run_phase3_target_validation.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--ssh-alias", required=True) + parser.add_argument("--partition", required=True) + parser.add_argument("--require-8-mi300x", action="store_true") + parser.add_argument("--trainer-backends", required=True) + parser.add_argument("--output-dir", required=True, type=Path) + args = parser.parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + report = { + "schema_version": "bb.rl.phase3.target_validation.v1", + "report_id": "phase3_target_validation_blocked", + "target_run_id": "", + "ssh_alias": args.ssh_alias, + "partition": args.partition, + "trainer_backends": args.trainer_backends.split(","), + "blocked_reason": "target_validation_requires_live_slurm_submission", + "scorecard_update_allowed": False, + "passed": False, + } + (args.output_dir / "phase3_target_validation_blocked.json").write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + print(json.dumps(report, sort_keys=True)) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/run_verl_trainer_update.py b/scripts/rl_phase3/run_verl_trainer_update.py new file mode 100644 index 00000000..e91490c1 --- /dev/null +++ b/scripts/rl_phase3/run_verl_trainer_update.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +_repo_root = Path(__file__).resolve().parents[2] if len(Path(__file__).resolve().parents) > 2 else Path.cwd() +sys.path.insert(0, str(_repo_root)) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run a real Phase 3 VeRL smoke trainer update on the target runtime.") + parser.add_argument("--backend", required=True, choices=["verl_ppo", "verl_grpo"]) + parser.add_argument("--target-run-id", required=True) + parser.add_argument("--introspection-report", required=True, type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--model-ref", default=os.environ.get("PHASE3_TRAINER_MODEL_PATH", "Qwen/Qwen2.5-0.5B-Instruct")) + args = parser.parse_args() + report_text = args.introspection_report.read_text() + if '"passed": true' not in report_text and '"passed":true' not in report_text: + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / f"{args.backend}_trainer_update_blocked.json").write_text( + '{"schema_version":"bb.rl.phase3.verl_trainer_update.v1","blocked_reason":"introspection_not_passed","scorecard_update_allowed":false,"passed":false}\n' + ) + return 2 + os.environ["PHASE3_TRAINER_BACKEND"] = args.backend + os.environ["PHASE3_TARGET_RUN_ID"] = args.target_run_id + os.environ["PHASE3_TRAINER_MODEL_PATH"] = args.model_ref + try: + from scripts.rl_phase3.target_verl_smoke_train import main as smoke_main + except ModuleNotFoundError: + from target_verl_smoke_train import main as smoke_main + + return smoke_main() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/target_phase4_native_inference_lane.py b/scripts/rl_phase3/target_phase4_native_inference_lane.py new file mode 100644 index 00000000..ca9a0331 --- /dev/null +++ b/scripts/rl_phase3/target_phase4_native_inference_lane.py @@ -0,0 +1,457 @@ +from __future__ import annotations + +import asyncio +import hashlib +import importlib +import json +import os +import socket +import subprocess +import sys +import time +import traceback +from pathlib import Path +from typing import Any + +MODEL = os.environ.get("PHASE4_NATIVE_INFERENCE_MODEL", "Qwen/Qwen2.5-0.5B-Instruct") +PORT = int(os.environ.get("PHASE4_NATIVE_INFERENCE_PORT", "18008")) +BASE_URL = f"http://127.0.0.1:{PORT}" +SERVER_TIMEOUT_SECONDS = int(os.environ.get("PHASE4_NATIVE_INFERENCE_SERVER_TIMEOUT", "900")) +CLAIM_BOUNDARY = "phase4_native_breadboard_sub_inference_lane_slurm_smoke_scope" +INFERENCE_OWNER = "breadboard_native_sub_inference_lane" + + +def sha_bytes(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def sha_path(path: Path) -> str: + return sha_bytes(path.read_bytes()) if path.exists() else "" + + +def json_hash(payload: Any) -> str: + return sha_bytes(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()) + + +def component_report(payload: dict[str, Any]) -> dict[str, Any]: + return { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "phase4_native_breadboard_inference_lane_attempt", + "component": "native_breadboard_inference_lane", + "claim_boundary": payload["claim_boundary"], + "target_run_id": payload["target_run_id"], + "points": 0, + "passed": payload["passed"], + "blocked_reason": payload["blocked_reason"], + "attempt_report": payload, + "input_hashes": {"attempt_report_inline": json_hash(payload)}, + "artifact_paths": {}, + "required_artifact_keys": [], + "scorecard_update_allowed": False, + "promotional": False, + } + + +def wait_for_server(proc: subprocess.Popen, timeout_seconds: int) -> tuple[bool, str]: + import urllib.request + + deadline = time.time() + timeout_seconds + last_error = "" + while time.time() < deadline: + if proc.poll() is not None: + return False, f"server_exited:{proc.returncode}:{last_error}" + try: + with urllib.request.urlopen(BASE_URL + "/v1/models", timeout=5) as response: + if response.status == 200: + return True, "" + last_error = f"status:{response.status}" + except Exception as exc: # noqa: BLE001 + last_error = f"{exc.__class__.__name__}:{exc}" + time.sleep(5) + return False, f"server_timeout:{last_error}" + + +class CountingComparator: + def __init__(self, inner): + self.inner = inner + self.compare_tool_call_calls = 0 + + def compare_tool_call(self, *args, **kwargs): + self.compare_tool_call_calls += 1 + return self.inner.compare_tool_call(*args, **kwargs) + + +class RealRolloutTokenizerProxy: + def __init__(self, tokenizer): + self._tokenizer = tokenizer + self.decode_calls = 0 + self.chat_template_calls = 0 + self.eos_token_id = tokenizer.eos_token_id + self.pad_token_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id + + def apply_chat_template(self, *args, **kwargs): + self.chat_template_calls += 1 + return self._tokenizer.apply_chat_template(*args, **kwargs) + + def decode(self, *args, **kwargs): + self.decode_calls += 1 + return self._tokenizer.decode(*args, **kwargs) + + def batch_decode(self, *args, **kwargs): + self.decode_calls += len(args[0]) if args else 1 + return self._tokenizer.batch_decode(*args, **kwargs) + + def encode(self, *args, **kwargs): + return self._tokenizer.encode(*args, **kwargs) + + def __getattr__(self, name): + return getattr(self._tokenizer, name) + + +async def main() -> int: + wrapper = Path(sys.argv[1]).resolve() + target_run_id = os.environ.get("PHASE3_TARGET_RUN_ID", "") + slurm_job_id = os.environ.get("SLURM_JOB_ID", "") or os.environ.get("PHASE3_SLURM_JOB_ID", "") + node = os.environ.get("SLURMD_NODENAME", "") or socket.gethostname() + if slurm_job_id and target_run_id.endswith("-slurm-pending"): + target_run_id = f"{target_run_id[:-len('-slurm-pending')]}-slurm-{slurm_job_id}" + report: dict[str, Any] = { + "schema_version": "bb.rl.phase4.native_breadboard_inference_lane_attempt.v1", + "report_id": "phase4_native_breadboard_inference_lane_attempt", + "component": "native_breadboard_inference_lane", + "target_run_id": target_run_id, + "slurm_job_id": slurm_job_id, + "node": node, + "claim_boundary": CLAIM_BOUNDARY, + "scorecard_update_allowed": False, + "promotional": False, + "breadboard_native_lane_used": False, + "inference_owner": "", + "engine_owner": "vllm_openai_server", + "compatibility_bridge": "verl.workers.rollout.llm_server.LLMServerClient", + "runtime": {}, + "wrapper": {}, + "checks": {}, + "errors": [], + "model": MODEL, + "server_base_url": BASE_URL, + } + source_path = wrapper / "src/zyphra_verl/nemo_gym_loop.py" + source = source_path.read_text() if source_path.exists() else "" + report["wrapper"] = { + "path": str(wrapper), + "nemo_gym_loop_sha256": sha_bytes(source.encode()) if source else "", + "deps_yaml_sha256": sha_path(wrapper / "deps.yaml"), + "source_terms_present": {term: (term in source) for term in ["register(\"nemo_gym_tool_use\"", "ToolParser", "ToolCallComparator", "reward_score"]}, + } + sys.path.insert(0, str(wrapper / "src")) + nemo_dir = os.environ.get("ZYPHRA_NEMO_GYM_DIR", "") + if nemo_dir: + sys.path.insert(0, nemo_dir) + proc = None + server_stdout_file = None + server_stderr_file = None + log_dir = Path(os.environ.get("PHASE4_NATIVE_INFERENCE_LOG_DIR", "/workspace")) + server_stdout_path = log_dir / "phase4_native_vllm_stdout.log" + server_stderr_path = log_dir / "phase4_native_vllm_stderr.log" + native_request_log_path = log_dir / "phase4_native_breadboard_requests.jsonl" + try: + for name in ("torch", "verl", "vllm", "nemo_gym", "zyphra_verl", "breadboard.rl.phase4.native_inference"): + try: + mod = importlib.import_module(name) + report["runtime"][name] = {"present": True, "version": str(getattr(mod, "__version__", "")), "file": str(getattr(mod, "__file__", ""))} + except Exception as exc: # noqa: BLE001 + report["runtime"][name] = {"present": False, "error": type(exc).__name__, "message": str(exc), "file": ""} + from breadboard.rl.phase4.wrapper_identity import runtime_module_provenance + + provenance = runtime_module_provenance( + wrapper, + { + "zyphra_verl": str(report["runtime"].get("zyphra_verl", {}).get("file", "")), + "verl": str(report["runtime"].get("verl", {}).get("file", "")), + "nemo_gym": str(report["runtime"].get("nemo_gym", {}).get("file", "")), + }, + ) + identity_manifest_path = wrapper / "wrapper_identity.json" + identity_manifest = {} + if identity_manifest_path.exists(): + identity_manifest = json.loads(identity_manifest_path.read_text()) + report["wrapper"]["identity_manifest_sha256"] = sha_path(identity_manifest_path) + report["wrapper"]["identity_manifest_passed"] = identity_manifest.get("passed") is True + report["checks"]["runtime_module_provenance"] = provenance + report["checks"]["runtime_module_provenance_passed"] = provenance["passed"] + try: + import torch + + report["runtime"]["torch_cuda"] = { + "available": bool(torch.cuda.is_available()), + "device_count": int(torch.cuda.device_count()) if torch.cuda.is_available() else 0, + "device_names": [torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())] if torch.cuda.is_available() else [], + } + except Exception as exc: # noqa: BLE001 + report["runtime"]["torch_cuda"] = {"error": type(exc).__name__, "message": str(exc)} + + server_cmd = [ + sys.executable, + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + MODEL, + "--host", + "127.0.0.1", + "--port", + str(PORT), + "--tensor-parallel-size", + "1", + "--gpu-memory-utilization", + os.environ.get("PHASE4_NATIVE_INFERENCE_GPU_UTIL", "0.35"), + "--max-model-len", + os.environ.get("PHASE4_NATIVE_INFERENCE_MAX_MODEL_LEN", "2048"), + "--no-enable-log-requests", + "--trust-remote-code", + ] + report["checks"]["server_command"] = server_cmd + server_stdout_path.parent.mkdir(parents=True, exist_ok=True) + server_stdout_file = server_stdout_path.open("w", encoding="utf-8") + server_stderr_file = server_stderr_path.open("w", encoding="utf-8") + report["checks"]["vllm_server_stdout_path"] = str(server_stdout_path) + report["checks"]["vllm_server_stderr_path"] = str(server_stderr_path) + report["checks"]["native_request_log_path"] = str(native_request_log_path) + proc = subprocess.Popen(server_cmd, stdout=server_stdout_file, stderr=server_stderr_file, text=True) + ready, ready_reason = wait_for_server(proc, SERVER_TIMEOUT_SECONDS) + report["checks"]["vllm_openai_server_ready"] = ready + if not ready: + report["errors"].append({"stage": "start_vllm_server", "type": "ServerNotReady", "message": ready_reason}) + raise RuntimeError(ready_reason) + + from breadboard.rl.phase4.native_inference import BREADBOARD_NATIVE_INFERENCE_OWNER, NativeInferenceLane, sha256_file + from omegaconf import OmegaConf + import ray + from transformers import AutoTokenizer + from verl.workers.rollout.llm_server import GlobalRequestLoadBalancer, LLMServerClient + from verl.workers.rollout.replica import TokenOutput + + base_tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True) + if base_tokenizer.pad_token_id is None: + base_tokenizer.pad_token = base_tokenizer.eos_token + tokenizer = RealRolloutTokenizerProxy(base_tokenizer) + + @ray.remote(num_cpus=0) + class BreadboardNativeInferenceActor: + def __init__(self, model: str, base_url: str, request_log_path: str, target_run_id: str): + actor_tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True) + if actor_tokenizer.pad_token_id is None: + actor_tokenizer.pad_token = actor_tokenizer.eos_token + self.lane = NativeInferenceLane( + model_ref=model, + base_url=base_url, + tokenizer=actor_tokenizer, + request_log_path=Path(request_log_path), + target_run_id=target_run_id, + ) + + def generate(self, request_id, prompt_ids, sampling_params, **kwargs): + del kwargs + native_response = self.lane.generate_completion( + upstream_request_id=request_id, + prompt_ids=list(prompt_ids), + sampling_params=dict(sampling_params), + ) + return TokenOutput( + token_ids=native_response.posthoc_token_ids, + log_probs=[0.0] * len(native_response.posthoc_token_ids), + num_preempted=0, + extra_fields={ + "breadboard_native_lane_used": True, + "inference_owner": BREADBOARD_NATIVE_INFERENCE_OWNER, + "breadboard_request_id": native_response.request_id, + "breadboard_response_id": native_response.response_id, + "model_ref": native_response.model_ref, + "raw_generation_text": native_response.output_text[:500], + "native_output_text_sha256": native_response.output_text_sha256, + "backend_token_texts_sha256": native_response.backend_token_texts_sha256, + "backend_token_text_count": len(native_response.backend_token_texts), + "backend_token_logprobs_sha256": native_response.backend_token_logprobs_sha256, + "backend_token_logprob_count": len(native_response.backend_token_logprobs), + "backend_token_ids_sha256": native_response.backend_token_ids_sha256, + "backend_token_id_count": len(native_response.backend_token_ids), + "posthoc_transport_token_ids_sha256": native_response.posthoc_token_ids_sha256, + "posthoc_transport_token_count": len(native_response.posthoc_token_ids), + "native_http_status": native_response.http_status, + "native_latency_ms": native_response.latency_ms, + "backend_completion_id": native_response.backend_completion_id, + }, + ) + + def status(self): + return self.lane.status() + + if not ray.is_initialized(): + ray.init(num_cpus=2, include_dashboard=False, ignore_reinit_error=True, logging_level="ERROR") + actor = BreadboardNativeInferenceActor.remote(MODEL, BASE_URL, str(native_request_log_path), target_run_id) + load_balancer = GlobalRequestLoadBalancer.remote(servers={BASE_URL: actor}) + server_manager = LLMServerClient(config=OmegaConf.create({}), load_balancer_handle=load_balancer) + report["checks"]["server_manager_class"] = f"{server_manager.__class__.__module__}.{server_manager.__class__.__name__}" + report["checks"]["server_manager_role"] = "compatibility_transport_for_breadboard_native_lane" + + loop_mod = importlib.import_module("zyphra_verl.nemo_gym_loop") + verifier, fn_call_cls = loop_mod._load_canonical_verifier() + report["checks"]["canonical_verifier_loaded"] = True + report["checks"]["canonical_verifier_module"] = str(getattr(verifier, "__file__", "")) + tool_parser_cls = getattr(loop_mod, "ToolParser") + tool_parser_obj = tool_parser_cls.get_tool_parser("hermes", tokenizer) + loop_cls = getattr(loop_mod, "NeMoGymToolUseLoop") + loop = loop_cls.__new__(loop_cls) + loop.response_length = 128 + loop.rollout_config = type("RolloutConfig", (), {"response_length": 128, "prompt_length": 1024, "multi_turn": type("MultiTurn", (), {"format": "hermes"})()})() + loop.tokenizer = tokenizer + loop.processor = None + loop.server_manager = server_manager + loop.loop = asyncio.get_running_loop() + loop.apply_chat_template_kwargs = {} + loop.system_prompt = [] + loop.tool_parser = tool_parser_obj + loop._FnCall = fn_call_cls + loop._ExpectedFunctionCall = verifier.ExpectedFunctionCall + comparator = CountingComparator(verifier.ToolCallComparator(config=verifier.ToolCallComparatorConfig(word_count_similarity_threshold=0.1))) + loop._comparator = comparator + + messages = [{"role": "user", "content": "Use the available tool to answer: what is the weather in Paris? Call get_weather with city Paris."}] + tools = [{"type": "function", "function": {"name": "get_weather", "description": "Return weather for a city.", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}] + expected = {"type": "function_call", "name": "get_weather", "arguments": {"city": "Paris"}} + output = await loop.run( + {"max_tokens": 128, "temperature": 0.0}, + raw_prompt=messages, + extra_info={"tools": tools, "expected_action": expected}, + ) + actor_status = ray.get(actor.status.remote()) + extra_fields = dict(getattr(output, "extra_fields", {}) or {}) + report["breadboard_native_lane_used"] = bool(extra_fields.get("breadboard_native_lane_used")) + report["inference_owner"] = str(extra_fields.get("inference_owner") or "") + report["checks"].update( + { + "agentloop_class": f"{loop_cls.__module__}.{loop_cls.__name__}", + "agentloop_run_invoked": True, + "native_lane_actor_status": actor_status, + "native_lane_generate_calls": actor_status.get("generate_calls", 0), + "native_request_id": extra_fields.get("breadboard_request_id", ""), + "native_response_id": extra_fields.get("breadboard_response_id", ""), + "native_model_ref": extra_fields.get("model_ref", ""), + "native_generation_text_observed": bool(extra_fields.get("raw_generation_text")), + "native_backend_token_texts_observed": int(extra_fields.get("backend_token_text_count") or 0) > 0, + "native_backend_token_ids_observed": int(extra_fields.get("backend_token_id_count") or 0) > 0, + "native_backend_token_logprobs_observed": int(extra_fields.get("backend_token_logprob_count") or 0) > 0, + "native_output_text_sha256": extra_fields.get("native_output_text_sha256", ""), + "native_backend_token_texts_sha256": extra_fields.get("backend_token_texts_sha256", ""), + "native_backend_token_ids_sha256": extra_fields.get("backend_token_ids_sha256", ""), + "native_backend_token_logprobs_sha256": extra_fields.get("backend_token_logprobs_sha256", ""), + "posthoc_transport_token_ids_sha256": extra_fields.get("posthoc_transport_token_ids_sha256", ""), + "native_request_log_sha256": actor_status.get("request_log_sha256", ""), + "tokenizer_chat_template_calls": tokenizer.chat_template_calls, + "tokenizer_decode_calls": tokenizer.decode_calls, + "raw_generation_text": extra_fields.get("raw_generation_text", ""), + "reward_score": float(getattr(output, "reward_score")) if getattr(output, "reward_score", None) is not None else None, + "reward_score_observed": getattr(output, "reward_score", None) is not None, + "comparator_compare_tool_call_calls": comparator.compare_tool_call_calls, + "canonical_comparator_reward_observed": comparator.compare_tool_call_calls > 0, + "canonical_tool_parser_used_observed": tokenizer.decode_calls > 0, + "canonical_agentloop_run_method_invoked": True, + "metrics": dict(getattr(output, "metrics", {}) or {}), + } + ) + if native_request_log_path.exists(): + report["checks"]["native_request_log_tail"] = native_request_log_path.read_text(errors="replace")[-4000:] + report["checks"]["native_request_log_sha256"] = sha256_file(native_request_log_path) + except Exception as exc: # noqa: BLE001 + report["errors"].append({"stage": "native_breadboard_inference_lane", "type": type(exc).__name__, "message": str(exc), "traceback": traceback.format_exc(limit=14)}) + finally: + if proc is not None and proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + proc.kill() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + pass + if proc is not None: + for handle in (server_stdout_file, server_stderr_file): + if handle is not None and not handle.closed: + handle.close() + stdout = server_stdout_path.read_text(errors="replace") if server_stdout_path.exists() else "" + stderr = server_stderr_path.read_text(errors="replace") if server_stderr_path.exists() else "" + report["checks"]["vllm_server_stdout_sha256"] = sha_bytes(stdout.encode()) + report["checks"]["vllm_server_stderr_sha256"] = sha_bytes(stderr.encode()) + report["checks"]["vllm_server_stdout_tail"] = stdout[-4000:] + report["checks"]["vllm_server_stderr_tail"] = stderr[-4000:] + report["checks"]["vllm_server_returncode"] = proc.returncode + + checks = report["checks"] + passed = bool( + checks.get("vllm_openai_server_ready") is True + and checks.get("server_manager_class") == "verl.workers.rollout.llm_server.LLMServerClient" + and checks.get("server_manager_role") == "compatibility_transport_for_breadboard_native_lane" + and checks.get("agentloop_run_invoked") is True + and report.get("breadboard_native_lane_used") is True + and report.get("inference_owner") == INFERENCE_OWNER + and checks.get("native_lane_generate_calls", 0) >= 1 + and bool(checks.get("native_request_id")) + and bool(checks.get("native_response_id")) + and checks.get("native_model_ref") == MODEL + and checks.get("native_generation_text_observed") is True + and (checks.get("native_backend_token_texts_observed") is True or checks.get("native_backend_token_ids_observed") is True) + and bool(checks.get("native_request_log_sha256")) + and checks.get("canonical_tool_parser_used_observed") is True + and checks.get("canonical_comparator_reward_observed") is True + and checks.get("reward_score_observed") is True + and checks.get("runtime_module_provenance_passed") is True + and report.get("wrapper", {}).get("identity_manifest_passed") is True + ) + blockers = [] + if not checks.get("vllm_openai_server_ready"): + blockers.append("vllm_openai_server_not_ready") + if checks.get("server_manager_class") != "verl.workers.rollout.llm_server.LLMServerClient": + blockers.append("verl_llm_server_client_compatibility_bridge_not_used") + if checks.get("server_manager_role") != "compatibility_transport_for_breadboard_native_lane": + blockers.append("native_lane_compatibility_bridge_not_marked") + if not checks.get("agentloop_run_invoked"): + blockers.append("canonical_agentloop_run_not_invoked") + if report.get("breadboard_native_lane_used") is not True: + blockers.append("breadboard_native_lane_not_used") + if report.get("inference_owner") != INFERENCE_OWNER: + blockers.append("breadboard_native_inference_owner_not_observed") + if checks.get("native_lane_generate_calls", 0) < 1: + blockers.append("native_lane_generate_not_observed") + if not checks.get("native_request_id"): + blockers.append("native_request_id_missing") + if not checks.get("native_response_id"): + blockers.append("native_response_id_missing") + if checks.get("native_model_ref") != MODEL: + blockers.append("native_model_ref_missing") + if checks.get("native_generation_text_observed") is not True: + blockers.append("native_generation_text_not_observed") + if checks.get("native_backend_token_texts_observed") is not True and checks.get("native_backend_token_ids_observed") is not True: + blockers.append("native_backend_token_output_not_observed") + if not checks.get("native_request_log_sha256"): + blockers.append("native_request_log_hash_missing") + if not checks.get("canonical_tool_parser_used_observed"): + blockers.append("canonical_tool_parser_use_not_observed") + if not checks.get("canonical_comparator_reward_observed"): + blockers.append("canonical_comparator_reward_not_observed") + if not checks.get("reward_score_observed"): + blockers.append("reward_score_not_observed") + if checks.get("runtime_module_provenance_passed") is not True: + provenance = checks.get("runtime_module_provenance") or {} + blockers.extend(provenance.get("blockers") or ["runtime_module_provenance_missing"]) + if report.get("wrapper", {}).get("identity_manifest_passed") is not True: + blockers.append("wrapper_identity_manifest_missing_or_failed") + report["passed"] = passed + report["blocked_reason"] = "" if passed else ";".join(blockers or ["native_breadboard_inference_lane_failed"]) + print("PHASE3_COMPONENT_REPORT_JSON=" + json.dumps(component_report(report), sort_keys=True, separators=(",", ":"))) + print(json.dumps({"passed": passed, "blocked_reason": report["blocked_reason"], "checks": checks}, sort_keys=True)) + return 0 if passed else 2 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/rl_phase3/target_verl_smoke_train.py b/scripts/rl_phase3/target_verl_smoke_train.py new file mode 100644 index 00000000..21b5060e --- /dev/null +++ b/scripts/rl_phase3/target_verl_smoke_train.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pandas as pd + + +def sha_path(path: Path) -> str: + h = hashlib.sha256() + if path.is_file(): + h.update(path.read_bytes()) + elif path.exists(): + for child in sorted(p for p in path.rglob("*") if p.is_file()): + h.update(str(child.relative_to(path)).encode()) + h.update(child.read_bytes()) + return "sha256:" + h.hexdigest() + + +def write_dataset(path: Path, rows: int, *, grpo: bool) -> None: + data = [] + for index in range(rows): + group = index // 2 if grpo else index + data.append({ + "data_source": "phase3_smoke_math", + "prompt": [{"role": "user", "content": f"Return only the integer {group + 1}."}], + "reward_model": {"style": "exact", "ground_truth": str(group + 1)}, + "extra_info": {"index": index, "split": "train", "group_id": str(group)}, + }) + df = pd.DataFrame(data) + path.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(path) + + +def write_reward(path: Path) -> None: + path.write_text( + "def compute_score(data_source, solution_str, ground_truth, extra_info=None):\n" + " text = str(solution_str).strip()\n" + " truth = str(ground_truth).strip()\n" + " return {'score': 1.0 if truth in text else 0.0, 'exact_match': truth in text}\n" + ) + + +def main() -> int: + backend = os.environ.get("PHASE3_TRAINER_BACKEND", "verl_ppo") + target_run_id = os.environ.get("PHASE3_TARGET_RUN_ID", "") + model_ref = os.environ.get("PHASE3_TRAINER_MODEL_PATH", "Qwen/Qwen2.5-0.5B-Instruct") + root = Path("/shared/bb-p3-root/phase3_trainer_runs") / f"{target_run_id or 'no-target'}-{int(time.time())}" / backend + root.mkdir(parents=True, exist_ok=True) + train = root / "train.parquet" + val = root / "val.parquet" + reward = root / "reward.py" + grpo = backend == "verl_grpo" + write_dataset(train, 8 if not grpo else 8, grpo=grpo) + write_dataset(val, 2 if not grpo else 2, grpo=grpo) + write_reward(reward) + ckpt_dir = root / "checkpoints" + before_sha = sha_path(ckpt_dir) + adv = "grpo" if grpo else "gae" + rollout_n = os.environ.get("PHASE3_ROLLOUT_N", "2" if grpo else "1") + rollout_name = os.environ.get("PHASE3_ROLLOUT_NAME", "vllm") + n_gpus_per_node = os.environ.get("PHASE3_N_GPUS_PER_NODE", "8") + entrypoint = os.environ.get("PHASE3_TRAINER_ENTRYPOINT", "verl.trainer.main_ppo" if grpo else "verl.trainer.main_ppo_sync") + command = [ + sys.executable, "-m", entrypoint, + f"data.train_files={train}", + f"data.val_files={val}", + "data.train_batch_size=8", + "data.val_batch_size=2", + "data.max_prompt_length=64", + "data.max_response_length=16", + "data.dataloader_num_workers=0", + "data.filter_overlong_prompts=False", + "data.truncation=right", + f"actor_rollout_ref.model.path={model_ref}", + "+actor_rollout_ref.model.override_config.attn_implementation=eager", + "actor_rollout_ref.model.trust_remote_code=True", + f"actor_rollout_ref.rollout.name={rollout_name}", + f"actor_rollout_ref.rollout.n={rollout_n}", + "actor_rollout_ref.rollout.tensor_model_parallel_size=1", + "actor_rollout_ref.rollout.mode=async", + "actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=1", + "actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=1", + "actor_rollout_ref.actor.ppo_mini_batch_size=8", + "actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=1", + "actor_rollout_ref.actor.fsdp_config.use_torch_compile=False", + ] + if not grpo: + command.extend([ + f"critic.model.path={model_ref}", + "+critic.model.override_config.attn_implementation=eager", + "critic.model.trust_remote_code=True", + "critic.ppo_micro_batch_size_per_gpu=1", + "critic.ppo_mini_batch_size=8", + "critic.optim.lr=1e-6", + ]) + command.extend([ + "reward.num_workers=1", + f"reward.custom_reward_function.path={reward}", + "reward.custom_reward_function.name=compute_score", + f"algorithm.adv_estimator={adv}", + "trainer.project_name=bb_phase3", + f"trainer.experiment_name={backend}", + "trainer.nnodes=1", + f"trainer.n_gpus_per_node={n_gpus_per_node}", + "trainer.total_epochs=1", + "trainer.total_training_steps=1", + "trainer.save_freq=1", + "trainer.test_freq=-1", + "trainer.val_before_train=False", + "trainer.logger=console", + f"trainer.default_local_dir={ckpt_dir}", + ]) + env = dict(os.environ) + env["HF_HOME"] = "/shared/bb-p3-root/hf_home" + visible = env.get("ROCR_VISIBLE_DEVICES", env.get("HIP_VISIBLE_DEVICES", env.get("CUDA_VISIBLE_DEVICES", ""))) + if visible: + env["HIP_VISIBLE_DEVICES"] = visible + env.pop("ROCR_VISIBLE_DEVICES", None) + env.pop("CUDA_VISIBLE_DEVICES", None) + env.pop("RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES", None) + started = time.time() + stdout_path = root / "trainer_stdout.log" + stderr_path = root / "trainer_stderr.log" + timed_out = False + with stdout_path.open("w") as stdout_handle, stderr_path.open("w") as stderr_handle: + try: + result = subprocess.run(command, text=True, stdout=stdout_handle, stderr=stderr_handle, env=env, timeout=2700, check=False) + returncode = result.returncode + except subprocess.TimeoutExpired as exc: + timed_out = True + returncode = 124 + if exc.stdout: + stdout_handle.write(exc.stdout if isinstance(exc.stdout, str) else exc.stdout.decode(errors="replace")) + if exc.stderr: + stderr_handle.write(exc.stderr if isinstance(exc.stderr, str) else exc.stderr.decode(errors="replace")) + stderr_handle.write(f"\nPHASE3_TRAINER_TIMEOUT: {exc}\n") + after_sha = sha_path(ckpt_dir) + changed = before_sha != after_sha and ckpt_dir.exists() + metrics = { + "optimizer_step_count": 1 if returncode == 0 and changed else 0, + "device_count": 8, + "weight_update_performed": returncode == 0 and changed, + "duration_seconds": time.time() - started, + "returncode": returncode, + "timed_out": timed_out, + } + (root / "metrics.json").write_text(json.dumps(metrics, sort_keys=True, indent=2) + "\n") + report = { + "schema_version": "bb.rl.phase3.verl_trainer_update.v1", + "report_id": f"phase3_{backend}_trainer_update", + "component": f"phase3_{backend}_trainer_update", + "claim_boundary": "phase3_verl_ppo_grpo_weight_update_named_target_scope", + "target_run_id": target_run_id, + "trainer_backend": backend, + "model_ref": model_ref, + "entrypoint": entrypoint, + "algorithm_adv_estimator": adv, + "rollout_name": rollout_name, + "n_gpus_per_node": int(n_gpus_per_node), + "optimizer_step_count": metrics["optimizer_step_count"], + "checkpoint_before_sha256": before_sha, + "checkpoint_after_sha256": after_sha, + "checkpoint_changed": changed, + "weight_update_performed": metrics["weight_update_performed"], + "device_count": int(n_gpus_per_node), + "artifact_paths": {"run_dir": str(root), "metrics": str(root / "metrics.json"), "stdout": str(root / "trainer_stdout.log"), "stderr": str(root / "trainer_stderr.log")}, + "input_hashes": {"train": sha_path(train), "val": sha_path(val), "reward": sha_path(reward)}, + "blocked_reason": "" if returncode == 0 and changed else ("verl_trainer_update_timeout" if timed_out else "verl_trainer_update_failed"), + "scorecard_update_allowed": False, + "passed": returncode == 0 and changed, + } + print("PHASE3_COMPONENT_REPORT_JSON=" + json.dumps(report, sort_keys=True, separators=(",", ":"))) + return 0 if report["passed"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rl_phase3/validate_phase3_component.py b/scripts/rl_phase3/validate_phase3_component.py new file mode 100644 index 00000000..87586077 --- /dev/null +++ b/scripts/rl_phase3/validate_phase3_component.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--component", required=True) + parser.add_argument("--phase-dir", required=True, type=Path) + parser.add_argument("--require-passed", action="store_true") + args = parser.parse_args() + matches = list((args.phase_dir / "runs").rglob(f"*{args.component}*.json")) + if not matches: + print(json.dumps({"errors": ["component report not found"]})) + return 2 + report = json.loads(matches[0].read_text()) + errors = [] + if args.require_passed and report.get("passed") is not True: + errors.append("passed must be true") + if report.get("scorecard_update_allowed") is not False: + errors.append("scorecard_update_allowed must be false") + print(json.dumps({"report": str(matches[0]), "errors": errors}, sort_keys=True)) + return 0 if not errors else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..d146cbc2 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test package namespace for RL helper imports.""" diff --git a/tests/rl/__init__.py b/tests/rl/__init__.py new file mode 100644 index 00000000..78a0d2ec --- /dev/null +++ b/tests/rl/__init__.py @@ -0,0 +1 @@ +"""RL Phase 1 test helpers.""" diff --git a/tests/rl/adapters/__init__.py b/tests/rl/adapters/__init__.py new file mode 100644 index 00000000..3f6ba9f1 --- /dev/null +++ b/tests/rl/adapters/__init__.py @@ -0,0 +1 @@ +"""Adapter test helpers.""" diff --git a/tests/rl/adapters/test_adapter_probe_reports.py b/tests/rl/adapters/test_adapter_probe_reports.py new file mode 100644 index 00000000..a8b7551e --- /dev/null +++ b/tests/rl/adapters/test_adapter_probe_reports.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from breadboard.rl.adapters.benchflow import build_benchflow_fixture_probe_report +from breadboard.rl.adapters.ors import build_ors_fixture_probe_report +from breadboard.rl.adapters.prime_verifiers import build_prime_verifiers_fixture_probe_report +from breadboard.rl.adapters.probe import validate_adapter_probe_report +from breadboard.rl.adapters.verl import build_verl_jsonl_probe_report + + +def test_required_adapter_probe_reports_validate() -> None: + reports = [ + build_benchflow_fixture_probe_report(), + build_ors_fixture_probe_report(), + build_verl_jsonl_probe_report(), + build_prime_verifiers_fixture_probe_report(), + ] + + assert {report.adapter_id for report in reports} == { + "benchflow.fixture.v1", + "ors.fixture.v1", + "verl.jsonl_probe.v1", + "prime_verifiers.fixture.v1", + } + for report in reports: + assert validate_adapter_probe_report(report) == [] + assert report.claim_boundary == "adapter_probe_not_production_integration" + assert report.preserved_fields + assert report.lost_fields or report.unsupported_fields + assert report.data_boundary == "fixture_or_probe_only" + assert report.fidelity_notes + assert report.promotion_requirements + assert set(report.preserved_fields).issubset(report.field_mapping) + + +def test_verl_report_points_to_m7_jsonl_not_trainer_support() -> None: + report = build_verl_jsonl_probe_report() + + assert report.support_level == "jsonl_probe" + assert "verl_DataProto_object" in report.lost_fields + assert "ppo_grpo_trainer_execution" in report.unsupported_fields + assert report.field_mapping["input_ids"] == "VeRLProbeRow.input_ids" + assert "DataProto" in " ".join(report.promotion_requirements) diff --git a/tests/rl/adapters/test_adapter_probe_schema.py b/tests/rl/adapters/test_adapter_probe_schema.py new file mode 100644 index 00000000..d6f2836e --- /dev/null +++ b/tests/rl/adapters/test_adapter_probe_schema.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from breadboard.rl.adapters.probe import AdapterProbeReport, validate_adapter_probe_report + + +def test_adapter_probe_requires_preserved_fields() -> None: + report = AdapterProbeReport( + adapter_id="x", + adapter_kind="fixture", + support_level="fixture_probe", + workload_family="swe", + preserved_fields=[], + ) + + assert "preserved_fields must be non-empty" in validate_adapter_probe_report(report) + + +def test_supported_report_cannot_have_lost_fields() -> None: + report = AdapterProbeReport( + adapter_id="x", + adapter_kind="fixture", + support_level="supported", + workload_family="swe", + preserved_fields=["task_id"], + lost_fields=["x"], + claim_boundary="production_supported", + ) + + assert "support_level=supported requires no lost_fields or unsupported_fields" in validate_adapter_probe_report(report) + + +def test_production_boundary_requires_supported_level() -> None: + report = AdapterProbeReport( + adapter_id="x", + adapter_kind="fixture", + support_level="fixture_probe", + workload_family="swe", + preserved_fields=["task_id"], + claim_boundary="production_supported", + ) + + assert "production_supported claim_boundary requires support_level=supported" in validate_adapter_probe_report(report) + + +def test_probe_report_requires_mapping_and_promotion_notes() -> None: + report = AdapterProbeReport( + adapter_id="x", + adapter_kind="fixture", + support_level="fixture_probe", + workload_family="swe", + preserved_fields=["task_id"], + lost_fields=["live_runtime"], + source_artifacts=["fixture.json"], + field_mapping={}, + fidelity_notes=[], + promotion_requirements=[], + ) + + errors = validate_adapter_probe_report(report) + + assert "field_mapping must be non-empty" in errors + assert "field_mapping missing preserved field: task_id" in errors + assert "non-supported reports must include fidelity_notes" in errors + assert "non-supported reports must include promotion_requirements" in errors + + +def test_complete_fixture_probe_report_validates() -> None: + report = AdapterProbeReport( + adapter_id="x", + adapter_kind="fixture", + support_level="fixture_probe", + workload_family="swe", + preserved_fields=["task_id"], + lost_fields=["live_runtime"], + source_artifacts=["fixture.json"], + field_mapping={"task_id": "fixture.task.id"}, + fidelity_notes=["Static fixture only."], + promotion_requirements=["Run against the real service."], + ) + + assert validate_adapter_probe_report(report) == [] diff --git a/tests/rl/docs/__init__.py b/tests/rl/docs/__init__.py new file mode 100644 index 00000000..2c56a1a8 --- /dev/null +++ b/tests/rl/docs/__init__.py @@ -0,0 +1 @@ +"""Documentation validation tests.""" diff --git a/tests/rl/docs/test_rl_phase1_docs.py b/tests/rl/docs/test_rl_phase1_docs.py new file mode 100644 index 00000000..df23fa26 --- /dev/null +++ b/tests/rl/docs/test_rl_phase1_docs.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[3] +WORKSPACE_ROOT = REPO_ROOT.parent +DOCS = REPO_ROOT / "docs" / "rl_phase1" +PHASE_DIR = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" + + +def test_m10_operator_docs_exist() -> None: + expected = [ + "README.md", + "env_package_ir.md", + "swe_hardening.md", + "verl_export_contract.md", + "runtime_pool_runbook.md", + "replay_admission.md", + "support_ladder.md", + "manual_qc_guide.md", + "decision_ledger.md", + "demo_script.md", + "m12_transfer_pack.md", + ] + + for name in expected: + assert (DOCS / name).exists(), name + + +def test_docs_preserve_claim_boundary() -> None: + combined = "\n".join(path.read_text(encoding="utf-8") for path in DOCS.glob("*.md")) + + assert "controlled SWE toy" in combined + assert "not production" in combined.lower() or "no production" in combined.lower() + assert "not DataProto" in combined + assert "MI300X" in combined + + +def test_handoff_names_critical_files_and_next_work() -> None: + handoff = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_HANDOFF.md" + text = handoff.read_text(encoding="utf-8") + + assert "Current verified score: 1000 / 1000" in text + assert "breadboard/rl/env_package/" in text + assert "breadboard/rl/security/" in text + assert "docs_tmp/ZYPHRA/RL_PHASE_1/runs/" in text + assert "M12 target validation has passed" in text diff --git a/tests/rl/e2e/__init__.py b/tests/rl/e2e/__init__.py new file mode 100644 index 00000000..acd38a9d --- /dev/null +++ b/tests/rl/e2e/__init__.py @@ -0,0 +1 @@ +"""End-to-end test helpers.""" diff --git a/tests/rl/e2e/test_secondary_math_pilot.py b/tests/rl/e2e/test_secondary_math_pilot.py new file mode 100644 index 00000000..df067406 --- /dev/null +++ b/tests/rl/e2e/test_secondary_math_pilot.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.export import build_projection_manifest +from breadboard.rl.replay import compare_replay_parity +from breadboard.rl.session import create_local_session +from breadboard.rl.trace import build_graph_from_session_events, validate_graph_invariants + + +REPO_ROOT = Path(__file__).resolve().parents[3] +MATH_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "math_console_toy" / "env_package.yaml" + + +def test_math_console_toy_env_package_validates() -> None: + package = load_env_package(MATH_TOY) + + assert package.package_id == "bb.math_console_toy.v1alpha" + assert package.runtime.backend == "local_process" + + +def test_math_console_toy_lifecycle_replay_and_projection_proof() -> None: + package = load_env_package(MATH_TOY) + session = create_local_session(package, "math_toy_001") + session.reset() + session.step({"tool": "submit_answer", "answer": "42"}) + evaluation = session.evaluate() + graph = build_graph_from_session_events( + graph_id="math_toy_001.graph", + session_id=session.session_id, + events=session.events, + ) + projection = build_projection_manifest( + graph=graph, + target_format="math_console_jsonl_probe", + preserved_fields=["task_id", "reward", "event_id"], + lost_fields=["full_python_console_transcript"], + included_node_kinds={"step", "evaluate"}, + ) + replay = compare_replay_parity(graph, graph) + + assert evaluation.reward == 1.0 + assert validate_graph_invariants(graph) == [] + assert replay.passed is True + assert projection.lost_fields == ["full_python_console_transcript"] + assert projection.metadata["canonical_truth"] == "breadboard_graph_replay_runtime" diff --git a/tests/rl/e2e/test_swe_hardened_probe.py b/tests/rl/e2e/test_swe_hardened_probe.py new file mode 100644 index 00000000..c186d5ae --- /dev/null +++ b/tests/rl/e2e/test_swe_hardened_probe.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.e2e import run_controlled_swe_probe + + +REPO_ROOT = Path(__file__).resolve().parents[3] +SWE_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "swe_toy_patch" / "env_package.yaml" + + +def test_accepted_rows_have_hardening_replay_and_projection_evidence(tmp_path) -> None: + run = run_controlled_swe_probe( + package_path=SWE_TOY, + output_dir=tmp_path, + run_id="test_m6", + limit=10, + ) + accepted = [row for row in run.rows if row.row_status == "accepted"] + + assert accepted + for row in accepted: + assert row.hardening_status == "passed" + assert row.replay_status == "passed" + assert row.exportable_debug is True + assert row.projection_id + assert (tmp_path / "row_evidence" / f"{row.task_id}.json").exists() diff --git a/tests/rl/e2e/test_swe_quarantine_export.py b/tests/rl/e2e/test_swe_quarantine_export.py new file mode 100644 index 00000000..314a530f --- /dev/null +++ b/tests/rl/e2e/test_swe_quarantine_export.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.e2e import run_controlled_swe_probe + + +REPO_ROOT = Path(__file__).resolve().parents[3] +SWE_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "swe_toy_patch" / "env_package.yaml" + + +def test_quarantined_and_rejected_rows_do_not_become_trainable_or_debug_exportable() -> None: + run = run_controlled_swe_probe(package_path=SWE_TOY, run_id="test_m6", limit=10) + + for row in run.rows: + assert row.trainable is False + if row.row_status in {"quarantined", "rejected"}: + assert row.exportable_debug is False + assert row.blocked_reasons or row.findings diff --git a/tests/rl/e2e/test_swe_toy_probe.py b/tests/rl/e2e/test_swe_toy_probe.py new file mode 100644 index 00000000..3afdd38e --- /dev/null +++ b/tests/rl/e2e/test_swe_toy_probe.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.e2e import run_controlled_swe_probe + + +REPO_ROOT = Path(__file__).resolve().parents[3] +SWE_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "swe_toy_patch" / "env_package.yaml" + + +def test_controlled_swe_toy_probe_runs_10_tasks_with_metrics_and_qc(tmp_path) -> None: + run = run_controlled_swe_probe( + package_path=SWE_TOY, + output_dir=tmp_path, + run_id="test_m6", + limit=10, + ) + + assert len(run.rows) == 10 + assert sum(row.row_status == "accepted" for row in run.rows) >= 7 + assert any(row.row_status == "rejected" for row in run.rows) + assert any(row.row_status == "quarantined" for row in run.rows) + assert "total_ms" in run.metrics_summary + assert run.metrics_summary["total_ms"]["p95"] >= run.metrics_summary["total_ms"]["p50"] + assert run.qc_report["accepted_sample"] + assert run.qc_report["rejected_sample"] + assert run.qc_report["quarantined_sample"] + assert (tmp_path / "run_ledger.jsonl").exists() + assert (tmp_path / "metrics_summary.json").exists() + assert (tmp_path / "qc_report.json").exists() + + +def test_controlled_swe_toy_claim_names_source() -> None: + run = run_controlled_swe_probe(package_path=SWE_TOY, run_id="test_m6", limit=10) + + assert run.source_claim == "controlled_swe_toy_slice" + assert "swe_rebench" not in run.source_claim + assert "swe_gym" not in run.source_claim diff --git a/tests/rl/env_package/test_export_support_level.py b/tests/rl/env_package/test_export_support_level.py new file mode 100644 index 00000000..a8aca219 --- /dev/null +++ b/tests/rl/env_package/test_export_support_level.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path + +from breadboard.rl.env_package.hash import canonical_env_package_hash +from breadboard.rl.env_package.validate import load_yaml_mapping, validate_env_package_mapping + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_supported_export_requires_support_evidence_refs() -> None: + payload = deepcopy(load_yaml_mapping(PYTHON_TOY)) + payload["exports"]["support_level"] = "supported" + payload["package_hash"] = canonical_env_package_hash(payload) + + assert "exports.support_level=supported requires support_evidence_refs" in validate_env_package_mapping(payload) + + +def test_non_trainable_export_cannot_use_trainable_status() -> None: + payload = deepcopy(load_yaml_mapping(PYTHON_TOY)) + payload["exports"]["trainability_status"] = "rl_candidate" + payload["package_hash"] = canonical_env_package_hash(payload) + + assert "non-trainable exports cannot use trainable trainability_status" in validate_env_package_mapping(payload) + + +def test_trainable_unknown_contamination_scope_fails() -> None: + payload = deepcopy(load_yaml_mapping(PYTHON_TOY)) + payload["exports"]["trainable"] = True + payload["exports"]["trainability_status"] = "rl_candidate" + payload["provenance"]["contamination_scope"] = "unknown" + payload["package_hash"] = canonical_env_package_hash(payload) + + assert "trainable packages must not use unknown contamination_scope" in validate_env_package_mapping(payload) + + +def test_trainable_replay_disabled_fails() -> None: + payload = deepcopy(load_yaml_mapping(PYTHON_TOY)) + payload["exports"]["trainable"] = True + payload["exports"]["trainability_status"] = "rl_candidate" + payload["replay"]["replay_required"] = False + payload["package_hash"] = canonical_env_package_hash(payload) + + assert "trainable packages require replay.replay_required" in validate_env_package_mapping(payload) + + +def test_trainable_protected_contamination_scope_fails() -> None: + payload = deepcopy(load_yaml_mapping(PYTHON_TOY)) + payload["exports"]["trainable"] = True + payload["exports"]["trainability_status"] = "rl_candidate" + payload["provenance"]["contamination_scope"] = "dev_hidden" + payload["package_hash"] = canonical_env_package_hash(payload) + + assert "protected contamination scopes cannot be marked trainable" in validate_env_package_mapping(payload) diff --git a/tests/rl/env_package/test_hardening_required_for_swe.py b/tests/rl/env_package/test_hardening_required_for_swe.py new file mode 100644 index 00000000..05fa0d8c --- /dev/null +++ b/tests/rl/env_package/test_hardening_required_for_swe.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path + +from breadboard.rl.env_package.validate import load_yaml_mapping, validate_env_package_mapping + + +REPO_ROOT = Path(__file__).resolve().parents[3] +SWE_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "swe_toy_patch" / "env_package.yaml" + + +def test_swe_package_requires_hardening_policy() -> None: + payload = deepcopy(load_yaml_mapping(SWE_TOY)) + payload["hardening"] = None + + assert "SWE packages require hardening policy" in validate_env_package_mapping(payload) + + +def test_untrusted_runtime_requires_hardening_policy() -> None: + payload = deepcopy(load_yaml_mapping(SWE_TOY)) + payload["package_id"] = "bb.patch_toy.v1alpha" + payload["tasksets"][0]["source_kind"] = "local_fixture" + payload["harness"]["interaction_mode"] = "multi_turn" + payload["hardening"] = None + + assert "untrusted runtime packages require hardening policy" in validate_env_package_mapping(payload) + + +def test_hardening_forbids_root_agent() -> None: + payload = deepcopy(load_yaml_mapping(SWE_TOY)) + payload["runtime"]["agent_user"] = "root" + + assert "hardening.agent_non_root_required forbids runtime.agent_user=root" in validate_env_package_mapping(payload) + + +def test_hardening_requires_verifier_isolation() -> None: + payload = deepcopy(load_yaml_mapping(SWE_TOY)) + payload["verifier"]["isolated_from_agent"] = False + + errors = validate_env_package_mapping(payload) + assert "hardening.verifier_isolated_required requires verifier.isolated_from_agent=true" in errors diff --git a/tests/rl/env_package/test_hash_stability.py b/tests/rl/env_package/test_hash_stability.py new file mode 100644 index 00000000..2317dc11 --- /dev/null +++ b/tests/rl/env_package/test_hash_stability.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path + +from breadboard.rl.env_package.hash import canonical_env_package_hash +from breadboard.rl.env_package.validate import load_yaml_mapping, validate_env_package_mapping + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_canonical_hash_matches_declared_hash() -> None: + payload = load_yaml_mapping(PYTHON_TOY) + + assert validate_env_package_mapping(payload) == [] + assert payload["package_hash"] == canonical_env_package_hash(payload) + + +def test_canonical_hash_ignores_declared_package_hash_value() -> None: + payload = load_yaml_mapping(PYTHON_TOY) + mutated = deepcopy(payload) + mutated["package_hash"] = "sha256:not-the-real-hash" + + assert canonical_env_package_hash(payload) == canonical_env_package_hash(mutated) + + +def test_canonical_hash_is_stable_under_mapping_order_changes() -> None: + payload = load_yaml_mapping(PYTHON_TOY) + reordered = { + "exports": payload["exports"], + "schema_version": payload["schema_version"], + "runtime": payload["runtime"], + "splits": payload["splits"], + "provenance": payload["provenance"], + "version": payload["version"], + "package_id": payload["package_id"], + "package_hash": payload["package_hash"], + "tasksets": payload["tasksets"], + "harness": payload["harness"], + "verifier": payload["verifier"], + "reward": payload["reward"], + "renderer": payload["renderer"], + "hardening": payload["hardening"], + "replay": payload["replay"], + } + + assert canonical_env_package_hash(payload) == canonical_env_package_hash(reordered) + + +def test_canonical_hash_changes_under_semantic_edit() -> None: + payload = load_yaml_mapping(PYTHON_TOY) + edited = deepcopy(payload) + edited["runtime"]["backend"] = "docker" + + assert canonical_env_package_hash(payload) != canonical_env_package_hash(edited) + + +def test_declared_hash_mismatch_fails_validation() -> None: + payload = load_yaml_mapping(PYTHON_TOY) + payload["package_hash"] = "sha256:bad" + + assert "package_hash does not match canonical EnvPackage hash" in validate_env_package_mapping(payload) diff --git a/tests/rl/env_package/test_runtime_envelope.py b/tests/rl/env_package/test_runtime_envelope.py new file mode 100644 index 00000000..d38223ae --- /dev/null +++ b/tests/rl/env_package/test_runtime_envelope.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path + +from breadboard.rl.env_package.hash import canonical_env_package_hash +from breadboard.rl.env_package.validate import load_yaml_mapping, validate_env_package_mapping + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_full_network_requires_allowlist_reason() -> None: + payload = deepcopy(load_yaml_mapping(PYTHON_TOY)) + payload["runtime"]["network"] = "full" + payload["package_hash"] = canonical_env_package_hash(payload) + + assert "runtime.network=full requires network_allowlist_reason" in validate_env_package_mapping(payload) + + +def test_full_network_with_allowlist_reason_validates() -> None: + payload = deepcopy(load_yaml_mapping(PYTHON_TOY)) + payload["runtime"]["network"] = "full" + payload["runtime"]["network_allowlist_reason"] = "fixture-localhost-only" + payload["package_hash"] = canonical_env_package_hash(payload) + + assert validate_env_package_mapping(payload) == [] diff --git a/tests/rl/env_package/test_schema_validation.py b/tests/rl/env_package/test_schema_validation.py new file mode 100644 index 00000000..42eb6a02 --- /dev/null +++ b/tests/rl/env_package/test_schema_validation.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path + +import pytest + +from breadboard.rl.env_package.schema import EnvPackage, SCHEMA_VERSION +from breadboard.rl.env_package.validate import load_env_package, load_yaml_mapping, validate_env_package_mapping + + +REPO_ROOT = Path(__file__).resolve().parents[3] +EXAMPLES = REPO_ROOT / "examples" / "rl_env_packages" +PYTHON_TOY = EXAMPLES / "python_console_toy" / "env_package.yaml" +SWE_TOY = EXAMPLES / "swe_toy_patch" / "env_package.yaml" + + +def load_example(path: Path) -> dict: + return load_yaml_mapping(path) + + +def assert_invalid(payload: dict, expected_fragment: str) -> None: + errors = validate_env_package_mapping(payload) + assert any(expected_fragment in error for error in errors), errors + + +def test_golden_packages_validate_and_load_as_env_packages() -> None: + for path in [PYTHON_TOY, SWE_TOY]: + package = load_env_package(path) + assert isinstance(package, EnvPackage) + assert package.schema_version == SCHEMA_VERSION + assert package.package_hash + assert package.to_dict()["package_hash"] == package.package_hash + + +def test_missing_required_field_fails_with_specific_error() -> None: + payload = load_example(PYTHON_TOY) + payload.pop("verifier") + + assert_invalid(payload, "missing required field: verifier") + assert_invalid(payload, "verifier must be a mapping") + + +def test_missing_package_hash_fails() -> None: + payload = load_example(PYTHON_TOY) + payload.pop("package_hash") + + assert_invalid(payload, "missing required field: package_hash") + assert_invalid(payload, "package_hash must be non-empty") + + +def test_env_package_from_dict_raises_on_invalid_payload() -> None: + payload = load_example(PYTHON_TOY) + payload["schema_version"] = "wrong" + + with pytest.raises(ValueError, match="schema_version"): + EnvPackage.from_dict(payload) + + +def test_split_must_be_allowed_by_owning_taskset() -> None: + payload = deepcopy(load_example(PYTHON_TOY)) + payload["splits"]["not_allowed"] = deepcopy(payload["splits"]["train_probe"]) + payload["splits"]["not_allowed"]["split_id"] = "not_allowed" + payload["splits"]["not_allowed"]["split_hash"] = "sha256:not-allowed" + + assert_invalid(payload, "splits.not_allowed is not listed in taskset allowed_splits") diff --git a/tests/rl/env_package/test_split_visibility.py b/tests/rl/env_package/test_split_visibility.py new file mode 100644 index 00000000..cc871abd --- /dev/null +++ b/tests/rl/env_package/test_split_visibility.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from copy import deepcopy +from pathlib import Path + +from breadboard.rl.env_package.validate import load_yaml_mapping, validate_env_package_mapping + + +REPO_ROOT = Path(__file__).resolve().parents[3] +SWE_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "swe_toy_patch" / "env_package.yaml" + + +def test_protected_split_cannot_be_trainer_visible() -> None: + payload = deepcopy(load_yaml_mapping(SWE_TOY)) + payload["splits"]["protected_probe"]["trainer_visible"] = True + + errors = validate_env_package_mapping(payload) + assert "splits.protected_probe protected split cannot be trainer_visible" in errors + + +def test_protected_split_cannot_be_optimizer_visible() -> None: + payload = deepcopy(load_yaml_mapping(SWE_TOY)) + payload["splits"]["protected_probe"]["optimizer_visible"] = True + + errors = validate_env_package_mapping(payload) + assert "splits.protected_probe protected split cannot be optimizer_visible" in errors diff --git a/tests/rl/export/__init__.py b/tests/rl/export/__init__.py new file mode 100644 index 00000000..343393bd --- /dev/null +++ b/tests/rl/export/__init__.py @@ -0,0 +1 @@ +"""Export test helpers.""" diff --git a/tests/rl/export/test_projection_manifest.py b/tests/rl/export/test_projection_manifest.py new file mode 100644 index 00000000..aa430e74 --- /dev/null +++ b/tests/rl/export/test_projection_manifest.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from breadboard.rl.export import build_projection_manifest +from breadboard.rl.trace import build_graph_from_session_events +from tests.rl.session.helpers import build_successful_toy_session + + +def test_projection_manifest_records_preserved_and_lost_fields() -> None: + session = build_successful_toy_session() + graph = build_graph_from_session_events( + graph_id="toy.graph", + session_id=session.session_id, + events=session.events, + ) + + manifest = build_projection_manifest( + graph=graph, + target_format="jsonl_transition_probe", + preserved_fields=["node_kind", "reward", "event_id"], + lost_fields=["full_runtime_state"], + included_node_kinds={"step", "evaluate"}, + ) + + assert manifest.source_graph_id == graph.graph_id + assert "full_runtime_state" in manifest.lost_fields + assert len(manifest.included_node_ids) == 2 + assert len(manifest.excluded_node_ids) == 2 + assert manifest.metadata["canonical_truth"] == "breadboard_graph_replay_runtime" diff --git a/tests/rl/export/test_token_record_export.py b/tests/rl/export/test_token_record_export.py new file mode 100644 index 00000000..68e89527 --- /dev/null +++ b/tests/rl/export/test_token_record_export.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from breadboard.rl.export.token_record import ( + TOKEN_RECORD_EXPORT_SCHEMA, + build_token_record_export_payload, + validate_token_record_export_payload, +) +from breadboard.rl.renderer.schema import RenderedTurnRecord +from tests.rl.renderer.helpers import cloned_payload + + +def test_token_record_export_payload_validates() -> None: + record = RenderedTurnRecord.from_dict(cloned_payload()) + payload = build_token_record_export_payload(record) + + assert payload["schema_version"] == TOKEN_RECORD_EXPORT_SCHEMA + assert payload["projection_boundary"]["canonical_truth"] == "breadboard_graph_replay_runtime" + assert payload["projection_boundary"]["trainer_specific"] is False + assert payload["projection_boundary"]["verl_support_claim"] is False + assert validate_token_record_export_payload(payload) == [] + + +def test_token_record_export_rejects_trainer_specific_boundary() -> None: + record = RenderedTurnRecord.from_dict(cloned_payload()) + payload = build_token_record_export_payload(record) + payload["projection_boundary"]["trainer_specific"] = True + + errors = validate_token_record_export_payload(payload) + assert "projection_boundary.trainer_specific must be false" in errors + + +def test_token_record_export_rejects_m2_verl_support_claim() -> None: + record = RenderedTurnRecord.from_dict(cloned_payload()) + payload = build_token_record_export_payload(record) + payload["projection_boundary"]["verl_support_claim"] = True + + errors = validate_token_record_export_payload(payload) + assert "projection_boundary.verl_support_claim must be false for M2" in errors diff --git a/tests/rl/export/test_verl_logprob_trainability.py b/tests/rl/export/test_verl_logprob_trainability.py new file mode 100644 index 00000000..6ffa92b9 --- /dev/null +++ b/tests/rl/export/test_verl_logprob_trainability.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from breadboard.rl.export import build_verl_probe_rows_from_m6_summary, validate_verl_probe_row +from breadboard.rl.export.schema import VerlProbeRow + + +WORKSPACE_ROOT = Path(__file__).resolve().parents[4] +M6_SUMMARY = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m6_controlled_swe_toy" / "run_summary.json" + + +def accepted_row() -> VerlProbeRow: + summary = json.loads(M6_SUMMARY.read_text(encoding="utf-8")) + return next(row for row in build_verl_probe_rows_from_m6_summary(summary) if row.trainable_candidate) + + +def test_trainable_candidate_requires_completion_logprobs() -> None: + payload = accepted_row().to_dict() + payload.pop("completion_logprobs") + + errors = validate_verl_probe_row(VerlProbeRow.from_dict(payload)) + assert "trainable_candidate requires completion_logprobs" in errors + + +def test_trainable_candidate_requires_passed_hardening_and_replay() -> None: + payload = accepted_row().to_dict() + payload["admission"]["hardening_status"] = "quarantined" + + errors = validate_verl_probe_row(VerlProbeRow.from_dict(payload)) + assert "trainable_candidate requires hardening_status=passed" in errors diff --git a/tests/rl/export/test_verl_nested_metadata.py b/tests/rl/export/test_verl_nested_metadata.py new file mode 100644 index 00000000..ee939ca4 --- /dev/null +++ b/tests/rl/export/test_verl_nested_metadata.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from breadboard.rl.export import build_verl_probe_rows_from_m6_summary, validate_verl_probe_row +from breadboard.rl.export.schema import VerlProbeRow + + +WORKSPACE_ROOT = Path(__file__).resolve().parents[4] +M6_SUMMARY = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m6_controlled_swe_toy" / "run_summary.json" + + +def _rows(): + summary = json.loads(M6_SUMMARY.read_text(encoding="utf-8")) + return build_verl_probe_rows_from_m6_summary(summary) + + +def test_verl_probe_rows_include_required_nested_metadata() -> None: + for row in _rows(): + assert validate_verl_probe_row(row) == [] + assert row.policy["policy_staleness"]["staleness_status"] == "not_stale_offline_probe" + assert "stop_ids" in row.renderer + assert row.reward["verifier_hash"].startswith("sha256:") + assert row.runtime["state_refs"] + assert row.runtime["artifact_refs"] + assert row.admission["eligible_exports"] + + +def test_verl_probe_row_requires_policy_staleness() -> None: + payload = _rows()[0].to_dict() + payload["policy"].pop("policy_staleness") + + errors = validate_verl_probe_row(VerlProbeRow.from_dict(payload)) + assert "policy.policy_staleness must be present" in errors + + +def test_verl_probe_row_requires_renderer_stop_ids() -> None: + payload = _rows()[0].to_dict() + payload["renderer"].pop("stop_ids") + + errors = validate_verl_probe_row(VerlProbeRow.from_dict(payload)) + assert "renderer.stop_ids must be present" in errors + + +def test_verl_probe_row_requires_explicit_logprob_status_for_non_trainable_rows() -> None: + payload = next(row for row in _rows() if not row.trainable_candidate).to_dict() + payload.pop("completion_logprob_status") + + errors = validate_verl_probe_row(VerlProbeRow.from_dict(payload)) + assert "missing completion_logprobs requires explicit unavailable status" in errors + + +def test_verl_probe_row_requires_runtime_artifact_refs() -> None: + payload = _rows()[0].to_dict() + payload["runtime"].pop("artifact_refs") + + errors = validate_verl_probe_row(VerlProbeRow.from_dict(payload)) + assert "runtime.artifact_refs must be present" in errors diff --git a/tests/rl/export/test_verl_parquet_probe.py b/tests/rl/export/test_verl_parquet_probe.py new file mode 100644 index 00000000..17af4881 --- /dev/null +++ b/tests/rl/export/test_verl_parquet_probe.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from breadboard.rl.export import ( + build_verl_probe_rows_from_m6_summary, + smoke_consume_verl_probe_parquet, + write_verl_probe_parquet, +) + + +WORKSPACE_ROOT = Path(__file__).resolve().parents[4] +M6_SUMMARY = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m6_controlled_swe_toy" / "run_summary.json" + + +def test_verl_probe_parquet_smoke_consumer_passes(tmp_path) -> None: + summary = json.loads(M6_SUMMARY.read_text(encoding="utf-8")) + rows = build_verl_probe_rows_from_m6_summary(summary) + output = tmp_path / "verl_probe.parquet" + write_verl_probe_parquet(rows, output) + + report = smoke_consume_verl_probe_parquet(output) + + assert output.exists() + assert output.stat().st_size > 0 + assert report["tensorizable"] is True + assert report["row_count"] == 10 + assert report["trainable_candidate_count"] == 7 + assert report["errors"] == [] + assert "not DataProto" in report["compatibility_target"] diff --git a/tests/rl/export/test_verl_projection_manifest.py b/tests/rl/export/test_verl_projection_manifest.py new file mode 100644 index 00000000..d54abf9c --- /dev/null +++ b/tests/rl/export/test_verl_projection_manifest.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from breadboard.rl.export import ( + build_verl_probe_projection_manifest, + build_verl_probe_rows_from_m6_summary, + validate_verl_probe_projection_manifest, + write_verl_probe_projection_manifest, +) + + +WORKSPACE_ROOT = Path(__file__).resolve().parents[4] +M6_SUMMARY = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m6_controlled_swe_toy" / "run_summary.json" + + +def _rows(): + summary = json.loads(M6_SUMMARY.read_text(encoding="utf-8")) + return build_verl_probe_rows_from_m6_summary(summary) + + +def test_verl_probe_projection_manifest_records_projection_boundary() -> None: + rows = _rows() + manifest = build_verl_probe_projection_manifest(rows, target_formats=["jsonl", "parquet"]) + + assert validate_verl_probe_projection_manifest(manifest, rows) == [] + assert manifest["canonical_truth"] == "breadboard_graph_replay_runtime" + assert manifest["claim_boundary"] == "verl_shaped_probe_not_trainer_ready" + assert manifest["row_count"] == 10 + assert manifest["trainable_candidate_count"] == 7 + assert "trainer_dataproto_object" in manifest["lost_fields"] + assert set(manifest["source_projection_manifest_ids"]) == {row.projection_manifest_id for row in rows} + + +def test_verl_probe_projection_manifest_file_round_trips(tmp_path) -> None: + rows = _rows() + output = tmp_path / "projection_manifest.json" + manifest = write_verl_probe_projection_manifest(rows, output, target_formats=["jsonl", "parquet"]) + loaded = json.loads(output.read_text(encoding="utf-8")) + + assert loaded == manifest + assert validate_verl_probe_projection_manifest(loaded, rows) == [] diff --git a/tests/rl/export/test_verl_quarantine_blocks_trainable.py b/tests/rl/export/test_verl_quarantine_blocks_trainable.py new file mode 100644 index 00000000..b9680d26 --- /dev/null +++ b/tests/rl/export/test_verl_quarantine_blocks_trainable.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from breadboard.rl.export import build_verl_probe_rows_from_m6_summary + + +WORKSPACE_ROOT = Path(__file__).resolve().parents[4] +M6_SUMMARY = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m6_controlled_swe_toy" / "run_summary.json" + + +def test_quarantined_and_rejected_m6_rows_are_not_trainable_candidates() -> None: + summary = json.loads(M6_SUMMARY.read_text(encoding="utf-8")) + rows = build_verl_probe_rows_from_m6_summary(summary) + + for row in rows: + if row.admission["row_status"] in {"quarantined", "rejected"}: + assert row.trainable_candidate is False + assert row.completion_logprobs is None diff --git a/tests/rl/export/test_verl_required_fields.py b/tests/rl/export/test_verl_required_fields.py new file mode 100644 index 00000000..5f433f41 --- /dev/null +++ b/tests/rl/export/test_verl_required_fields.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from breadboard.rl.export import build_verl_probe_rows_from_m6_summary +from breadboard.rl.export.schema import VerlProbeRow + + +WORKSPACE_ROOT = Path(__file__).resolve().parents[4] +M6_SUMMARY = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m6_controlled_swe_toy" / "run_summary.json" + + +def load_rows(): + summary = json.loads(M6_SUMMARY.read_text(encoding="utf-8")) + return build_verl_probe_rows_from_m6_summary(summary) + + +def test_verl_probe_rows_include_required_identity_and_policy_fields() -> None: + rows = load_rows() + + assert rows + for row in rows: + assert row.rollout_id + assert row.trajectory_id + assert row.episode_id + assert row.task_id + assert row.env_package_hash.startswith("sha256:") + assert row.policy["policy_id"] + + +def test_missing_required_field_fails_from_dict() -> None: + payload = load_rows()[0].to_dict() + payload.pop("task_id") + + with pytest.raises(ValueError, match="task_id"): + VerlProbeRow.from_dict(payload) diff --git a/tests/rl/export/test_verl_smoke_consumer.py b/tests/rl/export/test_verl_smoke_consumer.py new file mode 100644 index 00000000..47c60c69 --- /dev/null +++ b/tests/rl/export/test_verl_smoke_consumer.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from breadboard.rl.export import ( + build_verl_probe_rows_from_m6_summary, + smoke_consume_verl_probe_jsonl, + write_verl_probe_jsonl, +) + + +WORKSPACE_ROOT = Path(__file__).resolve().parents[4] +M6_SUMMARY = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m6_controlled_swe_toy" / "run_summary.json" + + +def test_verl_probe_jsonl_smoke_consumer_passes(tmp_path) -> None: + summary = json.loads(M6_SUMMARY.read_text(encoding="utf-8")) + rows = build_verl_probe_rows_from_m6_summary(summary) + output = tmp_path / "verl_probe.jsonl" + write_verl_probe_jsonl(rows, output) + + report = smoke_consume_verl_probe_jsonl(output) + + assert report["tensorizable"] is True + assert report["row_count"] == 10 + assert report["trainable_candidate_count"] == 7 + assert report["errors"] == [] + assert "not DataProto" in report["compatibility_target"] diff --git a/tests/rl/export/test_verl_token_mask_alignment.py b/tests/rl/export/test_verl_token_mask_alignment.py new file mode 100644 index 00000000..b45366dc --- /dev/null +++ b/tests/rl/export/test_verl_token_mask_alignment.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from breadboard.rl.export import build_verl_probe_rows_from_m6_summary, validate_verl_probe_row +from breadboard.rl.export.schema import VerlProbeRow + + +WORKSPACE_ROOT = Path(__file__).resolve().parents[4] +M6_SUMMARY = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m6_controlled_swe_toy" / "run_summary.json" + + +def first_row() -> VerlProbeRow: + summary = json.loads(M6_SUMMARY.read_text(encoding="utf-8")) + return build_verl_probe_rows_from_m6_summary(summary)[0] + + +def test_valid_verl_probe_row_token_masks_align() -> None: + assert validate_verl_probe_row(first_row()) == [] + + +def test_verl_probe_row_rejects_loss_mask_mismatch() -> None: + payload = first_row().to_dict() + payload["loss_mask"] = [True] + + errors = validate_verl_probe_row(VerlProbeRow.from_dict(payload)) + assert "loss_mask length must equal input_ids length" in errors + + +def test_verl_probe_row_rejects_input_ids_mismatch() -> None: + payload = first_row().to_dict() + payload["input_ids"] = [1, 2, 3, 999] + + errors = validate_verl_probe_row(VerlProbeRow.from_dict(payload)) + assert "input_ids must equal prompt_ids + completion_ids" in errors diff --git a/tests/rl/m12/__init__.py b/tests/rl/m12/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/tests/rl/m12/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/rl/m12/test_m12_bootstrap.py b/tests/rl/m12/test_m12_bootstrap.py new file mode 100644 index 00000000..cf672d4d --- /dev/null +++ b/tests/rl/m12/test_m12_bootstrap.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import json +from pathlib import Path +import subprocess +import sys + +from breadboard.rl.m12 import ( + validate_m12_bootstrap_dry_run_report, + write_m12_bootstrap_dry_run_report, + write_m12_transfer_archive, +) + + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def test_m12_bootstrap_dry_run_report_executes_generated_bootstrap(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + output_path = tmp_path / "bootstrap" / "m12_bootstrap_dry_run_report.json" + + report = write_m12_bootstrap_dry_run_report( + repo_root=REPO_ROOT, + workspace_root=REPO_ROOT.parent, + transfer_prep_dir=tmp_path / "prep", + output_path=output_path, + ) + + assert output_path.exists() + assert report["report_id"] == "bb_zyphra_rl_phase1_m12_bootstrap_dry_run_report_v1" + assert report["claim_boundary"] == "target_bootstrap_dry_run_not_m12_validation" + assert report["status"] == "passed" + assert report["scorecard_update_allowed"] is False + assert report["m12_points_awarded"] is False + assert report["repo_head_verified"] is True + assert report["dirty_checkout_check_observed"] is True + assert report["dirty_checkout_mode"] in {"clean", "override"} + assert report["target_commands_skipped"] is True + assert report["exit_code"] == 0 + assert report["input_hashes"]["bootstrap_script"].startswith("sha256:") + assert report["input_hashes"]["transfer_manifest"].startswith("sha256:") + assert report["input_hashes"]["archive_manifest"].startswith("sha256:") + assert report["input_hashes"]["overlay_dry_run_report"].startswith("sha256:") + assert report["overlay"]["status"] == "passed" + assert report["overlay"]["dry_run"] is True + assert report["overlay"]["written_count"] == 0 + assert validate_m12_bootstrap_dry_run_report(report) == [] + + +def test_m12_bootstrap_dry_run_cli_writes_non_scoring_report(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + output_path = tmp_path / "out" / "m12_bootstrap_dry_run_report.json" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_bootstrap_dry_run.py", + "--repo-root", + str(REPO_ROOT), + "--workspace-root", + str(REPO_ROOT.parent), + "--transfer-prep-dir", + str(tmp_path / "prep"), + "--output", + str(output_path), + "--require-pass", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0 + assert "status=passed" in result.stdout + assert "dirty_checkout_mode=" in result.stdout + assert "target_commands_skipped=True" in result.stdout + report = json.loads(output_path.read_text(encoding="utf-8")) + assert report["status"] == "passed" + assert report["dirty_checkout_check_observed"] is True + assert report["dirty_checkout_mode"] in {"clean", "override"} + assert report["target_commands_skipped"] is True + assert report["input_hashes"]["bootstrap_script"].startswith("sha256:") + assert report["input_hashes"]["overlay_dry_run_report"].startswith("sha256:") + assert report["overlay"]["written_count"] == 0 + assert report["scorecard_update_allowed"] is False + + +def test_m12_bootstrap_validator_rejects_missing_input_hashes(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + output_path = tmp_path / "bootstrap" / "m12_bootstrap_dry_run_report.json" + report = write_m12_bootstrap_dry_run_report( + repo_root=REPO_ROOT, + workspace_root=REPO_ROOT.parent, + transfer_prep_dir=tmp_path / "prep", + output_path=output_path, + ) + + report["input_hashes"]["bootstrap_script"] = None + report["input_hashes"].pop("overlay_dry_run_report") + + errors = validate_m12_bootstrap_dry_run_report(report) + + assert "input_hashes.bootstrap_script must start with sha256:" in errors + assert "input_hashes.overlay_dry_run_report must start with sha256:" in errors + + +def test_m12_bootstrap_validator_rejects_stale_input_hashes(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + output_path = tmp_path / "bootstrap" / "m12_bootstrap_dry_run_report.json" + report = write_m12_bootstrap_dry_run_report( + repo_root=REPO_ROOT, + workspace_root=REPO_ROOT.parent, + transfer_prep_dir=tmp_path / "prep", + output_path=output_path, + ) + + report["input_hashes"]["transfer_manifest"] = "sha256:" + ("0" * 64) + + errors = validate_m12_bootstrap_dry_run_report(report) + + assert "input_hashes.transfer_manifest does not match current file" in errors + + +def test_m12_bootstrap_validator_rejects_overlay_summary_drift(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + output_path = tmp_path / "bootstrap" / "m12_bootstrap_dry_run_report.json" + report = write_m12_bootstrap_dry_run_report( + repo_root=REPO_ROOT, + workspace_root=REPO_ROOT.parent, + transfer_prep_dir=tmp_path / "prep", + output_path=output_path, + ) + + report["overlay"]["would_write_count"] += 1 + + errors = validate_m12_bootstrap_dry_run_report(report) + + assert "overlay.would_write_count must match overlay_dry_run_report" in errors + + +def test_m12_bootstrap_validator_rejects_stale_overlay_report_file(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + output_path = tmp_path / "bootstrap" / "m12_bootstrap_dry_run_report.json" + report = write_m12_bootstrap_dry_run_report( + repo_root=REPO_ROOT, + workspace_root=REPO_ROOT.parent, + transfer_prep_dir=tmp_path / "prep", + output_path=output_path, + ) + overlay_path = Path(report["overlay_dry_run_report"]) + overlay = json.loads(overlay_path.read_text(encoding="utf-8")) + overlay["existing_destination_count"] += 1 + overlay_path.write_text(json.dumps(overlay, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_m12_bootstrap_dry_run_report(report) + + assert "input_hashes.overlay_dry_run_report does not match current file" in errors + assert ( + "overlay_dry_run_report.existing_destination_count must equal entries with exists=true" + in errors + ) diff --git a/tests/rl/m12/test_m12_command_logs.py b/tests/rl/m12/test_m12_command_logs.py new file mode 100644 index 00000000..8c5bde41 --- /dev/null +++ b/tests/rl/m12/test_m12_command_logs.py @@ -0,0 +1,940 @@ +from __future__ import annotations + +import hashlib +import json +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest + +from breadboard.rl.m12 import record_command_log_result, validate_command_log_manifest +from breadboard.rl.m12.final_report import REQUIRED_COMMAND_LOG_IDS + + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return "sha256:" + digest.hexdigest() + + +def _write_wrapper_style_log( + log_path: Path, + *, + command_id: str, + command: str = "python -m pytest -q", + exit_code: int = 0, + started_at: str = "2026-06-18T00:00:00Z", + completed_at: str = "2026-06-18T00:00:01Z", + target_run_id: str = "m12-target-run-test", + body: str | None = None, +) -> None: + log_path.write_text( + "\n".join( + [ + f"# command_id: {command_id}", + f"# target_run_id: {target_run_id}", + f"# command: {command}", + f"# argv_json: {json.dumps(shlex.split(command), ensure_ascii=True)}", + f"# started_at: {started_at}", + body if body is not None else f"{command_id} ok", + f"# completed_at: {completed_at}", + f"# exit_code: {exit_code}", + "", + ] + ), + encoding="utf-8", + ) + + +def _write_valid_logged_manifest(tmp_path: Path, command_id: str = "target_preflight") -> Path: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + log_dir.mkdir() + log_path = log_dir / f"{command_id}.log" + _write_wrapper_style_log(log_path, command_id=command_id) + record_command_log_result( + manifest_path=manifest_path, + command_id=command_id, + command="python -m pytest -q", + log_path=log_path, + exit_code=0, + started_at="2026-06-18T00:00:00Z", + completed_at="2026-06-18T00:00:01Z", + target_run_id="m12-target-run-test", + ) + return manifest_path + + +def _write_full_valid_logged_manifest(tmp_path: Path) -> Path: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + log_dir.mkdir() + for command_id in REQUIRED_COMMAND_LOG_IDS: + log_path = log_dir / f"{command_id}.log" + _write_wrapper_style_log(log_path, command_id=command_id) + record_command_log_result( + manifest_path=manifest_path, + command_id=command_id, + command="python -m pytest -q", + log_path=log_path, + exit_code=0, + started_at="2026-06-18T00:00:00Z", + completed_at="2026-06-18T00:00:01Z", + target_run_id="m12-target-run-test", + ) + return manifest_path + + +def test_logged_command_cli_records_hash_verified_manifest(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_logged_command.py", + "--manifest", + str(manifest_path), + "--log-dir", + str(log_dir), + "--command-id", + "target_preflight", + "--target-run-id", + "m12-target-run-test", + "--", + sys.executable, + "-c", + "print('m12 log ok')", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0 + assert "m12 log ok" in result.stdout + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + assert manifest["claim_boundary"] == "target_command_logs_not_scorecard_update" + assert manifest["scorecard_update_allowed"] is False + assert manifest["m12_points_awarded"] is False + assert entry["status"] == "passed" + assert entry["exit_code"] == 0 + assert entry["target_run_id"] == "m12-target-run-test" + assert entry["argv"] == [sys.executable, "-c", "print('m12 log ok')"] + assert entry["attempts"][0]["argv"] == [sys.executable, "-c", "print('m12 log ok')"] + assert entry["log_path"] == "logs/target_preflight.log" + assert entry["sha256"].startswith("sha256:") + assert manifest["target_run_ids"] == ["m12-target-run-test"] + assert manifest["latest_target_run_id"] == "m12-target-run-test" + assert (log_dir / "target_preflight.log").exists() + assert "# argv_json: " in (log_dir / "target_preflight.log").read_text(encoding="utf-8") + assert ( + validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + == [] + ) + + +def test_logged_command_cli_records_failure_and_exits_nonzero(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_logged_command.py", + "--manifest", + str(manifest_path), + "--log-dir", + str(log_dir), + "--command-id", + "target_preflight", + "--", + sys.executable, + "-c", + "import sys; print('m12 fail'); sys.exit(7)", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 7 + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + assert entry["status"] == "failed" + assert entry["exit_code"] == 7 + assert "command did not pass: target_preflight" in validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + +def test_logged_command_cli_separates_trailer_after_output_without_newline(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_logged_command.py", + "--manifest", + str(manifest_path), + "--log-dir", + str(log_dir), + "--command-id", + "target_preflight", + "--target-run-id", + "m12-target-run-test", + "--", + sys.executable, + "-c", + "import sys; sys.stdout.write('no trailing newline')", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0 + log_text = (log_dir / "target_preflight.log").read_text(encoding="utf-8") + assert "no trailing newline\n# completed_at:" in log_text + assert ( + validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + == [] + ) + + +def test_logged_command_cli_records_spawn_failure_and_exits_nonzero(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_logged_command.py", + "--manifest", + str(manifest_path), + "--log-dir", + str(log_dir), + "--command-id", + "target_preflight", + "--target-run-id", + "m12-target-run-test", + "--", + "definitely-not-a-real-m12-command", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 127 + assert "spawn_error: FileNotFoundError" in result.stderr + log_path = log_dir / "target_preflight.log" + assert log_path.exists() + assert "spawn_error: FileNotFoundError" in log_path.read_text(encoding="utf-8") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + assert entry["status"] == "failed" + assert entry["exit_code"] == 127 + assert entry["target_run_id"] == "m12-target-run-test" + assert entry["argv"] == ["definitely-not-a-real-m12-command"] + assert entry["log_path"] == "logs/target_preflight.log" + assert entry["sha256"].startswith("sha256:") + assert "spawn_error: FileNotFoundError" in entry["notes"] + assert "command did not pass: target_preflight" in validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + +def test_logged_command_cli_rejects_unsafe_target_run_id_before_running(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_logged_command.py", + "--manifest", + str(manifest_path), + "--log-dir", + str(log_dir), + "--command-id", + "target_preflight", + "--target-run-id", + "../bad-target-run", + "--", + sys.executable, + "-c", + "print('should not run')", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode != 0 + assert "invalid target run id" in result.stderr + assert "should not run" not in result.stdout + assert not manifest_path.exists() + assert not (log_dir / "target_preflight.log").exists() + + +def test_logged_command_cli_exports_target_run_id_to_child_environment(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_logged_command.py", + "--manifest", + str(manifest_path), + "--log-dir", + str(log_dir), + "--command-id", + "target_preflight", + "--target-run-id", + "m12-target-run-test", + "--", + sys.executable, + "-c", + "import os; print(os.environ.get('M12_TARGET_RUN_ID'))", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0 + assert "m12-target-run-test" in result.stdout + log_path = log_dir / "target_preflight.log" + assert "m12-target-run-test" in log_path.read_text(encoding="utf-8") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + assert entry["target_run_id"] == "m12-target-run-test" + + +def test_logged_command_cli_rejects_unsafe_log_dir_before_running(tmp_path) -> None: + manifest_path = tmp_path / "manifest" / "command_log_manifest.json" + outside_log_dir = tmp_path.parent / f"{tmp_path.name}_outside_logs" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_logged_command.py", + "--manifest", + str(manifest_path), + "--log-dir", + str(outside_log_dir), + "--command-id", + "target_preflight", + "--target-run-id", + "m12-target-run-test", + "--", + sys.executable, + "-c", + "print('should not run')", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode != 0 + assert "invalid log path" in result.stderr + assert "should not run" not in result.stdout + assert not manifest_path.exists() + assert not outside_log_dir.exists() + + +def test_logged_command_cli_preserves_rerun_attempt_logs(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + base_command = [ + sys.executable, + "scripts/rl_phase1/run_m12_logged_command.py", + "--manifest", + str(manifest_path), + "--log-dir", + str(log_dir), + "--command-id", + "target_preflight", + "--", + sys.executable, + "-c", + ] + + first = subprocess.run( + [*base_command, "print('first target attempt')"], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + second = subprocess.run( + [*base_command, "print('second target attempt')"], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert first.returncode == 0 + assert second.returncode == 0 + assert (log_dir / "target_preflight.log").exists() + assert (log_dir / "target_preflight.attempt-002.log").exists() + assert "first target attempt" in (log_dir / "target_preflight.log").read_text(encoding="utf-8") + assert "second target attempt" in (log_dir / "target_preflight.attempt-002.log").read_text(encoding="utf-8") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + assert entry["status"] == "passed" + assert entry["log_path"] == "logs/target_preflight.attempt-002.log" + assert len(entry["attempts"]) == 2 + assert entry["attempts"][0]["log_path"] == "logs/target_preflight.log" + assert entry["attempts"][1]["log_path"] == "logs/target_preflight.attempt-002.log" + assert entry["attempts"][0]["argv"] == [sys.executable, "-c", "print('first target attempt')"] + assert entry["attempts"][1]["argv"] == [sys.executable, "-c", "print('second target attempt')"] + assert entry["argv"] == [sys.executable, "-c", "print('second target attempt')"] + assert ( + validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + == [] + ) + + +def test_command_log_manifest_validator_rejects_completed_row_without_attempts(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + del entry["attempts"] + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "completed command entry must preserve attempts: target_preflight" in errors + + +def test_command_log_manifest_validator_rejects_unsafe_attempt_metadata(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + entry["attempts"][0]["log_path"] = "../target_preflight.log" + entry["attempts"][0]["target_run_id"] = "../bad-target-run" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "invalid attempt log_path for target_preflight attempt 1: log_path must not contain parent-directory traversal" in errors + assert "invalid target_run_id for target_preflight attempt 1: target_run_id must contain only letters, numbers, dot, underscore, colon, and hyphen" in errors + assert "latest attempt mismatch for target_preflight: log_path" in errors + assert "latest attempt mismatch for target_preflight: target_run_id" in errors + + +def test_command_log_manifest_validator_rejects_latest_attempt_drift(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + entry["status"] = "failed" + entry["exit_code"] = 99 + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "latest attempt mismatch for target_preflight: status" in errors + assert "latest attempt mismatch for target_preflight: exit_code" in errors + + +def test_command_log_manifest_validator_rejects_status_exit_code_incongruence(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + entry["status"] = "passed" + entry["exit_code"] = 4 + entry["attempts"][0]["status"] = "passed" + entry["attempts"][0]["exit_code"] = 4 + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=False, + ) + + assert "invalid status/exit_code for target_preflight: passed commands must have exit_code 0" in errors + assert ( + "invalid status/exit_code for target_preflight attempt 1: passed commands must have exit_code 0" + in errors + ) + + +def test_command_log_manifest_validator_rejects_unknown_command_rows(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["commands"].append( + { + "command_id": "operator_diagnostic", + "required": False, + "description": "unexpected target command row", + "status": "pending", + "exit_code": None, + "log_path": None, + "sha256": None, + "started_at": None, + "completed_at": None, + "notes": "", + } + ) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "unknown command entry: operator_diagnostic" in errors + + +def test_command_log_manifest_validator_rejects_invalid_argv_and_latest_drift(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + entry["argv"] = ["python", "-m", "pytest", "-q"] + entry["attempts"][0]["argv"] = [] + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "invalid argv for target_preflight attempt 1: argv must be a non-empty list" in errors + assert "latest attempt mismatch for target_preflight: argv" in errors + + +def test_command_log_manifest_validator_rejects_command_argv_incongruence(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + entry["argv"] = ["python", "-m", "pytest", "tests/rl", "-q"] + entry["attempts"][0]["argv"] = ["python", "-m", "pytest", "tests/rl", "-q"] + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert ( + "invalid command/argv for target_preflight attempt 1: command must equal shlex.join(argv)" + in errors + ) + + +def test_command_log_manifest_validator_rejects_raw_log_header_drift(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + log_path = tmp_path / "logs" / "target_preflight.log" + log_text = log_path.read_text(encoding="utf-8") + log_path.write_text( + log_text.replace("# command: python -m pytest -q", "# command: python -m pytest tests/rl -q"), + encoding="utf-8", + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + new_hash = _sha256_file(log_path) + entry["sha256"] = new_hash + entry["attempts"][0]["sha256"] = new_hash + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "raw log header mismatch for target_preflight attempt 1: command" in errors + + +def test_command_log_manifest_validator_rejects_raw_log_header_layout_drift(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + log_path = tmp_path / "logs" / "target_preflight.log" + lines = log_path.read_text(encoding="utf-8").splitlines() + command_line = lines.pop(2) + lines.insert(5, command_line) + log_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + new_hash = _sha256_file(log_path) + entry["sha256"] = new_hash + entry["attempts"][0]["sha256"] = new_hash + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "raw log header layout mismatch for target_preflight attempt 1: preamble" in errors + + +def test_command_log_manifest_validator_rejects_duplicate_raw_log_header_keys(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + log_dir.mkdir() + log_path = log_dir / "target_preflight.log" + _write_wrapper_style_log( + log_path, + command_id="target_preflight", + body="normal output\n# command: python -m pytest -q\nmore output", + ) + record_command_log_result( + manifest_path=manifest_path, + command_id="target_preflight", + command="python -m pytest -q", + log_path=log_path, + exit_code=0, + started_at="2026-06-18T00:00:00Z", + completed_at="2026-06-18T00:00:01Z", + target_run_id="m12-target-run-test", + ) + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "raw log header duplicate key for target_preflight attempt 1: command" in errors + + +def test_logged_command_cli_rejects_unsafe_command_id_before_running(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_logged_command.py", + "--manifest", + str(manifest_path), + "--log-dir", + str(log_dir), + "--command-id", + "../target_preflight", + "--", + sys.executable, + "-c", + "print('should not run')", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode != 0 + assert "invalid command id" in result.stderr + assert "should not run" not in result.stdout + assert not manifest_path.exists() + assert not (tmp_path / "target_preflight.log").exists() + + +def test_record_command_log_result_rejects_unsafe_command_id_before_manifest_write(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_path = tmp_path / "logs" / "target_preflight.log" + log_path.parent.mkdir() + log_path.write_text("ok\n", encoding="utf-8") + + with pytest.raises(ValueError, match="command_id must contain only"): + record_command_log_result( + manifest_path=manifest_path, + command_id="../target_preflight", + command="python -m pytest -q", + log_path=log_path, + exit_code=0, + started_at="2026-06-18T00:00:00Z", + completed_at="2026-06-18T00:00:01Z", + target_run_id="m12-target-run-test", + ) + + assert not manifest_path.exists() + + +def test_record_command_log_result_rejects_log_path_outside_manifest_tree(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + outside_dir = tmp_path.parent / f"{tmp_path.name}_outside_logs" + outside_dir.mkdir() + log_path = outside_dir / "target_preflight.log" + log_path.write_text("ok\n", encoding="utf-8") + + with pytest.raises(ValueError, match="log_path must be relative"): + record_command_log_result( + manifest_path=manifest_path, + command_id="target_preflight", + command="python -m pytest -q", + log_path=log_path, + exit_code=0, + started_at="2026-06-18T00:00:00Z", + completed_at="2026-06-18T00:00:01Z", + target_run_id="m12-target-run-test", + ) + + assert not manifest_path.exists() + + +def test_command_log_manifest_validator_rejects_duplicate_command_entries(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + manifest["commands"].append(dict(entry)) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "duplicate command entry: target_preflight" in errors + + +def test_command_log_manifest_validator_rejects_boundary_drift(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["claim_boundary"] = "scorecard_update_allowed" + manifest["scorecard_update_allowed"] = True + manifest["m12_points_awarded"] = True + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "claim_boundary must remain target_command_logs_not_scorecard_update" in errors + assert "scorecard_update_allowed must be false" in errors + assert "m12_points_awarded must be false" in errors + + +def test_command_log_manifest_validator_rejects_required_flag_drift(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + required_entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + optional_entry = next(item for item in manifest["commands"] if item["command_id"] == "final_report") + required_entry["required"] = False + optional_entry["required"] = True + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "required flag mismatch for target_preflight" in errors + assert "required flag mismatch for final_report" in errors + + +def test_command_log_manifest_validator_rejects_unsafe_manifest_command_id(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["commands"].append({"command_id": "../escape", "status": "pending"}) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "invalid command_id '../escape': command_id must contain only letters, numbers, dot, underscore, and hyphen" in errors + + +def test_command_log_manifest_validator_rejects_invalid_target_run_ids(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + entry["target_run_id"] = "../bad" + manifest["target_run_ids"] = ["m12-target-run-test", "../bad"] + manifest["latest_target_run_id"] = "../bad" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "invalid target_run_id for target_preflight: target_run_id must contain only letters, numbers, dot, underscore, colon, and hyphen" in errors + assert "invalid manifest target_run_id '../bad': target_run_id must contain only letters, numbers, dot, underscore, colon, and hyphen" in errors + assert "invalid latest_target_run_id: target_run_id must contain only letters, numbers, dot, underscore, colon, and hyphen" in errors + + +def test_command_log_manifest_validator_rejects_unsafe_log_paths(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + entry["log_path"] = "../target_preflight.log" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "invalid log_path for target_preflight: log_path must not contain parent-directory traversal" in errors + + entry["log_path"] = str((tmp_path / "logs" / "target_preflight.log").resolve()) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "invalid log_path for target_preflight: log_path must be relative to the command-log manifest directory" in errors + + +def test_command_log_manifest_validator_rejects_stale_target_run_id_summary(tmp_path) -> None: + manifest_path = _write_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["target_run_ids"] = [] + manifest["latest_target_run_id"] = "m12-target-run-missing" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "target_run_ids must equal sorted target_run_id values from command rows" in errors + assert "latest_target_run_id must be present in target_run_ids" in errors + + +def test_command_log_manifest_validator_rejects_stale_required_summary_flags(tmp_path) -> None: + manifest_path = _write_full_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert manifest["all_required_logs_archived"] is True + assert manifest["all_required_commands_passed"] is True + manifest["all_required_logs_archived"] = False + manifest["all_required_commands_passed"] = False + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest(manifest_path, require_passed=True, verify_hashes=True) + + assert "all_required_logs_archived must match required command log rows" in errors + assert "all_required_commands_passed must match required command statuses" in errors + + +def test_command_log_manifest_validator_rejects_narrowed_required_command_ids(tmp_path) -> None: + manifest_path = _write_full_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["required_command_ids"] = ["target_preflight"] + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "required_command_ids must equal canonical M12 required command IDs" in errors + + +def test_command_log_manifest_validator_rejects_missing_required_command_ids(tmp_path) -> None: + manifest_path = _write_full_valid_logged_manifest(tmp_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + del manifest["required_command_ids"] + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_command_log_manifest( + manifest_path, + required_command_ids=["target_preflight"], + require_passed=True, + verify_hashes=True, + ) + + assert "required_command_ids must equal canonical M12 required command IDs" in errors diff --git a/tests/rl/m12/test_m12_evidence_consistency.py b/tests/rl/m12/test_m12_evidence_consistency.py new file mode 100644 index 00000000..436de816 --- /dev/null +++ b/tests/rl/m12/test_m12_evidence_consistency.py @@ -0,0 +1,790 @@ +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import yaml + +from breadboard.rl.m12 import ( + build_m12_evidence_consistency_report, + validate_m12_evidence_consistency_report, +) + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PHASE_DIR = REPO_ROOT.parent / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" + + +REQUIRED_PHASE_FILES = [ + "BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml", + "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md", + "BB_ZYPHRA_RL_PHASE_1_M12_VALIDATION_REPORT.md", + "BB_ZYPHRA_RL_PHASE_1_HANDOFF.md", + "runs/m12_transfer_prep/m12_transfer_summary.json", + "runs/m12_transfer_prep/m12_transfer_manifest.json", + "runs/m12_transfer_prep/m12_transfer_archive_manifest.json", + "runs/m12_transfer_prep/m12_archive_verify_report.json", + "runs/m12_transfer_prep/m12_transfer_evidence_pack.tar.gz", + "runs/m12_transfer_prep/m12_transfer_evidence_pack.tar.gz.sha256", + "runs/m12_overlay_apply_probe/m12_overlay_apply_report.json", + "runs/m12_bootstrap_dry_run/m12_bootstrap_dry_run_report.json", + "runs/m12_target_preflight/m12_preflight_report.json", + "runs/m12_transfer_prep/m12_readiness_summary.json", + "runs/m12_final_report/m12_final_report.json", + "runs/m12_final_report/m12_remediation_summary.json", + "runs/m12_promotion_audit/m12_promotion_audit.json", +] + + +def _copy_required_phase_files(dst: Path) -> None: + for rel in REQUIRED_PHASE_FILES: + src = PHASE_DIR / rel + target = dst / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, target) + + +def test_m12_evidence_consistency_current_target_validated_state_passes() -> None: + report = build_m12_evidence_consistency_report(phase_dir=PHASE_DIR) + + assert report["report_id"] == "bb_zyphra_rl_phase1_m12_evidence_consistency_v1" + assert report["claim_boundary"] == "m12_evidence_consistency_not_scorecard_update" + assert report["scorecard_update_allowed"] is False + assert report["m12_points_awarded"] is False + assert report["consistent"] is True + assert report["errors"] == [] + assert report["counts"]["scorecard_current_verified_points"] == 1000 + assert report["counts"]["m12_verified_points"] == 80 + assert report["counts"]["transfer_artifacts"] == 28 + assert report["counts"]["archive_entries"] == 277 + assert report["counts"]["overlay_would_write"] == 277 + assert report["counts"]["bootstrap_overlay_would_write"] == 277 + assert report["counts"]["overlay_existing_destinations"] > 0 + assert report["counts"]["transfer_commands"] == 10 + assert report["counts"]["transfer_expected_outputs"] == 15 + assert report["counts"]["archive_verify_entries"] == 277 + assert report["counts"]["local_final_missing_gates"] == 0 + assert report["counts"]["local_remediation_summary_actions"] == 0 + assert report["counts"]["local_promotion_missing_requirements"] == 0 + assert report["checks"]["claim_ledger"]["has_final_report_manifest_validation_claim"] is True + assert report["checks"]["claim_ledger"]["has_promotion_row_equality_claim"] is True + assert report["checks"]["claim_ledger"]["has_promotion_explicit_score_inputs_claim"] is True + assert report["checks"]["claim_ledger"]["has_overlay_report_self_consistency_claim"] is True + assert report["checks"]["claim_ledger"]["has_bootstrap_overlay_file_validation_claim"] is True + assert report["checks"]["claim_ledger"]["has_promotion_final_report_path_gate_claim"] is True + assert report["checks"]["claim_ledger"]["has_promotion_control_input_path_gates_claim"] is True + assert report["checks"]["claim_ledger"]["has_promotion_output_path_gate_claim"] is True + assert report["checks"]["claim_ledger"]["has_promotion_target_script_path_gate_claim"] is True + assert report["checks"]["claim_ledger"]["has_target_run_log_reuse_guard_claim"] is True + assert report["checks"]["claim_ledger"]["has_target_closeout_artifact_reuse_guard_claim"] is True + assert report["checks"]["claim_ledger"]["has_cli_log_dir_preexecution_claim"] is True + assert report["checks"]["claim_ledger"]["has_unknown_command_row_rejection_claim"] is True + assert report["checks"]["scorecard"]["allowed_claim_records_cli_log_dir_preexecution"] is True + assert report["checks"]["scorecard"]["allowed_claim_records_unknown_command_row_rejection"] is True + assert report["checks"]["scorecard"]["allowed_claim_records_overlay_report_self_consistency"] is True + assert report["checks"]["scorecard"]["allowed_claim_records_bootstrap_overlay_file_validation"] is True + assert report["checks"]["scorecard"]["allowed_claim_records_promotion_final_report_path_gate"] is True + assert report["checks"]["scorecard"]["allowed_claim_records_promotion_control_input_path_gates"] is True + assert report["checks"]["scorecard"]["allowed_claim_records_promotion_output_path_gate"] is True + assert report["checks"]["scorecard"]["allowed_claim_records_promotion_target_script_path_gates"] is True + assert report["checks"]["scorecard"]["allowed_claim_records_target_run_log_reuse_guard"] is True + assert report["checks"]["scorecard"]["allowed_claim_records_target_closeout_artifact_reuse_guard"] is True + assert report["checks"]["m12_report"]["states_manifest_validation_gate"] is True + assert report["checks"]["m12_report"]["states_promotion_row_equality_gate"] is True + assert report["checks"]["m12_report"]["states_promotion_explicit_score_inputs"] is True + assert report["checks"]["m12_report"]["states_cli_log_dir_preexecution"] is True + assert report["checks"]["m12_report"]["states_unknown_command_row_rejection"] is True + assert report["checks"]["m12_report"]["states_promotion_final_report_path_gate"] is True + assert report["checks"]["m12_report"]["states_promotion_control_input_path_gates"] is True + assert report["checks"]["m12_report"]["states_promotion_output_path_gate"] is True + assert report["checks"]["m12_report"]["states_promotion_target_script_path_gate"] is True + assert report["checks"]["m12_report"]["states_target_run_log_reuse_guard"] is True + assert report["checks"]["m12_report"]["states_target_closeout_artifact_reuse_guard"] is True + assert report["checks"]["handoff"]["states_manifest_validation_gate"] is True + assert report["checks"]["handoff"]["states_promotion_row_equality_gate"] is True + assert report["checks"]["handoff"]["states_promotion_explicit_score_inputs"] is True + assert report["checks"]["handoff"]["states_cli_log_dir_preexecution"] is True + assert report["checks"]["handoff"]["states_unknown_command_row_rejection"] is True + assert report["checks"]["handoff"]["states_overlay_report_self_consistency"] is True + assert report["checks"]["handoff"]["states_bootstrap_overlay_file_validation"] is True + assert report["checks"]["handoff"]["states_promotion_final_report_path_gate"] is True + assert report["checks"]["handoff"]["states_promotion_control_input_path_gates"] is True + assert report["checks"]["handoff"]["states_promotion_output_path_gate"] is True + assert report["checks"]["handoff"]["states_promotion_target_script_path_gate"] is True + assert report["checks"]["handoff"]["states_target_run_log_reuse_guard"] is True + assert report["checks"]["handoff"]["states_target_closeout_artifact_reuse_guard"] is True + assert report["checks"]["local_consistency_boundary"]["consistency_report_not_transfer_artifact"] is True + assert report["checks"]["local_consistency_boundary"]["bootstrap_report_not_transfer_artifact"] is True + assert report["checks"]["local_consistency_boundary"]["consistency_checker_not_target_command"] is True + assert report["checks"]["local_consistency_boundary"]["bootstrap_dry_run_not_target_command"] is True + assert report["checks"]["archive"]["repo_root_path_portable"] is True + assert report["checks"]["archive"]["source_paths_portable"] is True + assert report["checks"]["archive"]["archive_paths_portable"] is True + assert report["checks"]["archive_verify_report"]["validator_passed"] is True + assert report["checks"]["archive_verify_report"]["status_passed"] is True + assert report["checks"]["archive_verify_report"]["archive_sha_matches_manifest"] is True + assert report["checks"]["archive_verify_report"]["entry_count_matches_manifest"] is True + assert report["checks"]["transfer"]["readiness_summary_validator_passed"] is True + assert report["checks"]["transfer"]["transfer_summary_validator_passed"] is True + assert report["checks"]["transfer"]["readiness_summary_fail_closed"] is True + assert report["checks"]["overlay_apply"]["dry_run_only"] is True + assert report["checks"]["overlay_apply"]["would_write_matches_archive"] is True + assert report["checks"]["overlay_apply"]["would_write_matches_entries"] is True + assert report["checks"]["overlay_apply"]["written_count_bounded"] is True + assert report["checks"]["overlay_apply"]["existing_destination_count_matches_entries"] is True + assert report["checks"]["bootstrap_dry_run"]["repo_head_verified"] is True + assert report["checks"]["bootstrap_dry_run"]["target_commands_skipped"] is True + assert report["checks"]["bootstrap_dry_run"]["input_hashes_present"] is True + assert report["checks"]["bootstrap_dry_run"]["input_hashes_current"] is True + assert report["checks"]["bootstrap_dry_run"]["overlay_would_write_matches_archive"] is True + assert report["checks"]["bootstrap_dry_run"]["overlay_summary_matches_overlay_file"] is True + assert report["checks"]["bootstrap_dry_run"]["overlay_file_validator_passed"] is True + assert report["checks"]["final_report"]["missing_gates_empty"] is True + assert report["checks"]["final_report"]["target_artifact_paths_match"] is True + assert report["checks"]["final_report"]["target_run_ids_match_command_logs"] is True + assert report["checks"]["final_report"]["single_target_run_id_recorded"] is True + assert report["checks"]["remediation_summary"]["validator_passed"] is True + assert report["checks"]["remediation_summary"]["score_eligible_matches_final_report"] is True + assert report["checks"]["remediation_summary"]["missing_gate_count_matches_final_report"] is True + assert report["checks"]["remediation_summary"]["remediation_count_matches_final_report"] is True + assert report["checks"]["remediation_summary"]["action_gate_count_matches_final_report"] is True + assert report["checks"]["remediation_summary"]["target_hardware_action_absent_after_pass"] is True + assert report["checks"]["remediation_summary"]["command_log_action_absent_after_pass"] is True + assert report["checks"]["scorecard"]["m12_final_report_result_records_target_artifact_path_gate"] is True + assert report["checks"]["transfer"]["promotion_audit_explicit_score_inputs"] is True + assert report["checks"]["transfer"]["promotion_audit_explicit_target_paths"] is True + assert report["checks"]["transfer"]["target_run_log_reuse_guard"] is True + assert report["checks"]["transfer"]["target_closeout_artifact_reuse_guard"] is True + assert report["checks"]["promotion_audit"]["review_ready_true"] is True + assert report["checks"]["promotion_audit"]["final_report_path_gate_satisfied"] is True + assert report["checks"]["promotion_audit"]["command_log_manifest_path_gate_satisfied"] is True + assert report["checks"]["promotion_audit"]["scorecard_path_gate_satisfied"] is True + assert report["checks"]["promotion_audit"]["claim_ledger_path_gate_satisfied"] is True + assert report["checks"]["promotion_audit"]["output_path_gate_satisfied"] is True + assert report["checks"]["promotion_audit"]["m12_state_pre_review_unawarded"] is True + assert validate_m12_evidence_consistency_report(report) == [] + + +def test_m12_evidence_consistency_report_rejects_stale_summary_fields() -> None: + report = build_m12_evidence_consistency_report(phase_dir=PHASE_DIR) + report["checks"]["scorecard"]["current_points_1000"] = False + + errors = validate_m12_evidence_consistency_report(report) + + assert "errors must include every failed embedded check" in errors + assert "consistent must match evidence-consistency errors" not in errors + + stale_error_report = build_m12_evidence_consistency_report(phase_dir=PHASE_DIR) + stale_error_report["consistent"] = False + stale_error_report["errors"] = ["scorecard.current_points_1000"] + errors = validate_m12_evidence_consistency_report(stale_error_report) + assert "errors must not include passed embedded checks" in errors + + unknown_error_report = build_m12_evidence_consistency_report(phase_dir=PHASE_DIR) + unknown_error_report["consistent"] = False + unknown_error_report["errors"] = ["synthetic.unknown"] + errors = validate_m12_evidence_consistency_report(unknown_error_report) + assert "errors contains unknown entries" in errors + + +def test_m12_evidence_consistency_report_requires_boolean_checks() -> None: + report = build_m12_evidence_consistency_report(phase_dir=PHASE_DIR) + report["checks"]["scorecard"]["current_points_1000"] = "yes" + + errors = validate_m12_evidence_consistency_report(report) + + assert "checks.scorecard.current_points_1000 must be boolean" in errors + assert "errors must include every failed embedded check" in errors + + +def test_m12_evidence_consistency_cli_writes_report(tmp_path) -> None: + output_path = tmp_path / "m12_evidence_consistency.json" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(PHASE_DIR), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0 + assert output_path.exists() + assert "consistent=True" in result.stdout + written = json.loads(output_path.read_text(encoding="utf-8")) + assert written["consistent"] is True + assert written["scorecard_update_allowed"] is False + + +def test_m12_evidence_consistency_detects_scorecard_regression_after_promotion(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + scorecard_path = tmp_path / "BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" + scorecard = yaml.safe_load(scorecard_path.read_text(encoding="utf-8")) + scorecard["current_verified_points"] = 920 + scorecard["status"] = "m12_transfer_prepared_target_preflight_blocked" + for milestone in scorecard["milestones"]: + if milestone["id"] == "M12": + milestone["verified_points"] = 0 + milestone["status"] = "transfer_prepared_target_preflight_blocked" + scorecard_path.write_text(yaml.safe_dump(scorecard, sort_keys=False), encoding="utf-8") + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert written["consistent"] is False + assert "scorecard.current_points_1000" in written["errors"] + assert "scorecard.m12_awarded" in written["errors"] + assert "scorecard.m12_status_completed" in written["errors"] + +def test_m12_evidence_consistency_detects_stale_transfer_manifest_hashes(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + transfer_manifest_path = tmp_path / "runs/m12_transfer_prep/m12_transfer_manifest.json" + transfer_manifest = json.loads(transfer_manifest_path.read_text(encoding="utf-8")) + stale_artifact = next( + artifact + for artifact in transfer_manifest["artifacts"] + if artifact["path"] == "requirements.txt" + ) + stale_artifact["sha256"] = "sha256:" + ("0" * 64) + transfer_manifest_path.write_text(json.dumps(transfer_manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "archive.transfer_manifest_file_artifact_hashes_current" in written["errors"] + + +def test_m12_evidence_consistency_detects_stale_readiness_summary(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + readiness_path = tmp_path / "runs/m12_transfer_prep/m12_readiness_summary.json" + readiness = json.loads(readiness_path.read_text(encoding="utf-8")) + readiness["artifact_count"] += 1 + readiness["target_script_fail_closed"]["preflight_requires_pass"] = False + readiness_path.write_text(json.dumps(readiness, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "transfer.readiness_summary_validator_passed" in written["errors"] + assert "transfer.readiness_summary_matches_manifest_artifacts" in written["errors"] + assert "transfer.readiness_summary_fail_closed" in written["errors"] + assert any( + error.startswith("readiness_summary_validator.artifact_count must match") + for error in written["errors"] + ) + + +def test_m12_evidence_consistency_detects_stale_overlay_report(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + overlay_path = tmp_path / "runs/m12_overlay_apply_probe/m12_overlay_apply_report.json" + overlay = json.loads(overlay_path.read_text(encoding="utf-8")) + overlay["existing_destination_count"] += 1 + overlay_path.write_text(json.dumps(overlay, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "overlay_apply.existing_destination_count_matches_entries" in written["errors"] + assert any( + error == "overlay_validator.existing_destination_count must equal entries with exists=true" + for error in written["errors"] + ) + + +def test_m12_evidence_consistency_detects_missing_overlay_report_claim(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + claim_ledger_path = tmp_path / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + claim_ledger = claim_ledger_path.read_text(encoding="utf-8") + claim_ledger_path.write_text( + claim_ledger.replace( + "transfer overlay report status, error-list, write-count, and existing-destination self-consistency", + "transfer overlay report claim removed", + ), + encoding="utf-8", + ) + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "claim_ledger.has_overlay_report_self_consistency_claim" in written["errors"] + + +def test_m12_evidence_consistency_detects_missing_bootstrap_overlay_file_claim(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + claim_ledger_path = tmp_path / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + claim_ledger = claim_ledger_path.read_text(encoding="utf-8") + claim_ledger_path.write_text( + claim_ledger.replace( + "bootstrap dry-run reports validate the referenced overlay dry-run report", + "bootstrap dry-run referenced-overlay claim removed", + ), + encoding="utf-8", + ) + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "claim_ledger.has_bootstrap_overlay_file_validation_claim" in written["errors"] + + +def test_m12_evidence_consistency_detects_missing_promotion_final_report_path_gate_claim(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + claim_ledger_path = tmp_path / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + claim_ledger = claim_ledger_path.read_text(encoding="utf-8") + claim_ledger_path.write_text( + claim_ledger.replace( + "promotion-audit canonical final-report input-path gating", + "promotion-audit final-report path claim removed", + ), + encoding="utf-8", + ) + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "claim_ledger.has_promotion_final_report_path_gate_claim" in written["errors"] + + +def test_m12_evidence_consistency_detects_missing_promotion_control_input_path_gate_claim(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + claim_ledger_path = tmp_path / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + claim_ledger = claim_ledger_path.read_text(encoding="utf-8") + claim_ledger_path.write_text( + claim_ledger.replace( + "promotion-audit canonical control-input path gating", + "promotion-audit control input path claim removed", + ), + encoding="utf-8", + ) + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "claim_ledger.has_promotion_control_input_path_gates_claim" in written["errors"] + + +def test_m12_evidence_consistency_detects_missing_promotion_output_path_gate_claim(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + claim_ledger_path = tmp_path / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + claim_ledger = claim_ledger_path.read_text(encoding="utf-8") + claim_ledger_path.write_text( + claim_ledger.replace( + "promotion-audit canonical output-path gating", + "promotion-audit output path claim removed", + ), + encoding="utf-8", + ) + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "claim_ledger.has_promotion_output_path_gate_claim" in written["errors"] + + +def test_m12_evidence_consistency_detects_missing_promotion_target_script_path_gate_claim(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + claim_ledger_path = tmp_path / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + claim_ledger = claim_ledger_path.read_text(encoding="utf-8") + claim_ledger_path.write_text( + claim_ledger.replace( + "target-script promotion-audit explicit target-path gating", + "target-script promotion audit target path claim removed", + ), + encoding="utf-8", + ) + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "claim_ledger.has_promotion_target_script_path_gate_claim" in written["errors"] + + +def test_m12_evidence_consistency_detects_missing_target_run_log_reuse_guard_claim(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + claim_ledger_path = tmp_path / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + claim_ledger = claim_ledger_path.read_text(encoding="utf-8") + claim_ledger_path.write_text( + claim_ledger.replace( + "target-run command-log reuse guard", + "target run command log reuse claim removed", + ), + encoding="utf-8", + ) + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "claim_ledger.has_target_run_log_reuse_guard_claim" in written["errors"] + + +def test_m12_evidence_consistency_detects_nonportable_transfer_repo_root(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + transfer_manifest_path = tmp_path / "runs/m12_transfer_prep/m12_transfer_manifest.json" + transfer_manifest = json.loads(transfer_manifest_path.read_text(encoding="utf-8")) + transfer_manifest["repo"]["root"] = str(REPO_ROOT) + transfer_manifest["repo"]["root_path_portable"] = False + transfer_manifest_path.write_text(json.dumps(transfer_manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "archive.repo_root_path_portable" in written["errors"] + + +def test_m12_evidence_consistency_detects_stale_archive_verify_report(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + archive_verify_path = tmp_path / "runs/m12_transfer_prep/m12_archive_verify_report.json" + archive_verify = json.loads(archive_verify_path.read_text(encoding="utf-8")) + archive_verify["archive_sha256"] = "sha256:" + ("0" * 64) + archive_verify["included_entry_count"] += 1 + archive_verify_path.write_text(json.dumps(archive_verify, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "archive_verify_report.archive_sha_matches_manifest" in written["errors"] + assert "archive_verify_report.entry_count_matches_manifest" in written["errors"] + assert any( + error.startswith("archive_verify_report_validator.archive_sha256 must match archive manifest") + for error in written["errors"] + ) + + +def test_m12_evidence_consistency_detects_ineligible_final_report(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + final_report_path = tmp_path / "runs/m12_final_report/m12_final_report.json" + final_report = json.loads(final_report_path.read_text(encoding="utf-8")) + final_report["m12_score_eligible"] = False + final_report_path.write_text(json.dumps(final_report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "final_report.score_eligible_true" in written["errors"] + + +def test_m12_evidence_consistency_detects_stale_remediation_summary(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + remediation_summary_path = tmp_path / "runs/m12_final_report/m12_remediation_summary.json" + remediation_summary = json.loads(remediation_summary_path.read_text(encoding="utf-8")) + remediation_summary["missing_gate_count"] += 1 + remediation_summary_path.write_text( + json.dumps(remediation_summary, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "remediation_summary.missing_gate_count_matches_final_report" in written["errors"] + assert any( + error.startswith("remediation_summary_validator.missing_gate_count must match") + for error in written["errors"] + ) + + +def test_m12_evidence_consistency_detects_stale_scorecard_final_report_result(tmp_path) -> None: + _copy_required_phase_files(tmp_path) + scorecard_path = tmp_path / "BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" + scorecard = yaml.safe_load(scorecard_path.read_text(encoding="utf-8")) + for milestone in scorecard["milestones"]: + if milestone["id"] != "M12": + continue + for evidence in milestone["evidence"]: + if "build_m12_final_report.py" in str(evidence.get("command") or ""): + evidence["result"] = ( + str(evidence["result"]) + .replace("artifact_paths_match_target_defaults,", "") + .replace(",artifact_paths_match_target_defaults", "") + ) + break + scorecard_path.write_text(yaml.safe_dump(scorecard, sort_keys=False), encoding="utf-8") + + output_path = tmp_path / "out" / "m12_evidence_consistency.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/check_m12_evidence_consistency.py", + "--phase-dir", + str(tmp_path), + "--output", + str(output_path), + "--require-consistent", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + written = json.loads(output_path.read_text(encoding="utf-8")) + assert "scorecard.m12_final_report_result_records_target_artifact_path_gate" in written["errors"] diff --git a/tests/rl/m12/test_m12_final_report.py b/tests/rl/m12/test_m12_final_report.py new file mode 100644 index 00000000..361de99a --- /dev/null +++ b/tests/rl/m12/test_m12_final_report.py @@ -0,0 +1,1182 @@ +from __future__ import annotations + +import json +import hashlib +import shlex +import subprocess +import sys +from pathlib import Path + +from breadboard.rl.m12 import ( + build_m12_final_report, + record_command_log_result, + summarize_m12_final_report_remediations, + validate_m12_final_report, + validate_m12_final_report_remediation_summary, + write_m12_final_report, +) +from breadboard.rl.m12.final_report import REQUIRED_COMMAND_LOG_COMMANDS, REQUIRED_COMMAND_LOG_IDS, TARGET_ARTIFACT_PATHS +from breadboard.rl.m12.transfer import ( + COMMAND_LOG_MANIFEST_TEMPLATE, + LOAD_LADDER_REPORT_TEMPLATE, + SOAK_REPORT_TEMPLATE, +) + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PHASE_RUNS = REPO_ROOT.parent / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" + + +def _fingerprint_sha256(fingerprint: dict) -> str: + payload = dict(fingerprint) + payload.pop("sha256", None) + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _write_wrapper_style_log( + log_path: Path, + *, + command_id: str, + command: str, + exit_code: int = 0, + started_at: str = "2026-06-17T00:00:00Z", + completed_at: str = "2026-06-17T00:00:01Z", + target_run_id: str = "m12-target-run-test", +) -> None: + log_path.write_text( + "\n".join( + [ + f"# command_id: {command_id}", + f"# target_run_id: {target_run_id}", + f"# command: {command}", + f"# argv_json: {json.dumps(shlex.split(command), ensure_ascii=True)}", + f"# started_at: {started_at}", + f"{command_id} ok", + f"# completed_at: {completed_at}", + f"# exit_code: {exit_code}", + "", + ] + ), + encoding="utf-8", + ) + + +def _write_complete_command_log_manifest(tmp_path: Path, *, target_run_id: str = "m12-target-run-test") -> Path: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + log_dir.mkdir() + for command_id in REQUIRED_COMMAND_LOG_IDS: + log_path = log_dir / f"{command_id}.log" + _write_wrapper_style_log( + log_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + target_run_id=target_run_id, + ) + record_command_log_result( + manifest_path=manifest_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + log_path=log_path, + exit_code=0, + started_at="2026-06-17T00:00:00Z", + completed_at="2026-06-17T00:00:01Z", + target_run_id=target_run_id, + ) + return manifest_path + + +def _local_report() -> dict: + return build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + ) + + +def test_local_m12_final_report_is_not_score_eligible() -> None: + report = _local_report() + + assert report["report_id"] == "bb_zyphra_rl_phase1_m12_final_report_v1" + assert report["claim_boundary"] == "m12_target_validation_candidate_not_scorecard_update" + assert report["scorecard_update_allowed"] is False + assert report["m12_score_eligible"] is False + assert "artifact_paths_match_target_defaults" in report["missing_gates"] + assert report["artifact_path_policy"]["paths_match_target_defaults"] is False + assert "archive_verify_report_present" in report["missing_gates"] + assert "archive_verify_report_readable" in report["missing_gates"] + assert "archive_verify_report_valid" in report["missing_gates"] + assert "archive_verify_status_passed" in report["missing_gates"] + assert "preflight_passed" not in report["missing_gates"] + assert "preflight_report_valid" not in report["missing_gates"] + assert "swe_run_summary_valid" not in report["missing_gates"] + assert "verl_smoke_report_valid" not in report["missing_gates"] + assert "ray_probe_report_valid" not in report["missing_gates"] + assert "warm_vs_cold_report_valid" not in report["missing_gates"] + assert "preflight_runtime_fingerprint_present" not in report["missing_gates"] + assert "target_hardware" not in report["missing_gates"] + assert "verl_available" not in report["missing_gates"] + assert "inference_engine_available" not in report["missing_gates"] + assert "container_runtime_available" not in report["missing_gates"] + assert "ray_probe_distributed" in report["missing_gates"] + assert "load_ladder_report_present" in report["missing_gates"] + assert "soak_report_present" in report["missing_gates"] + assert "command_log_manifest_present" in report["missing_gates"] + assert "target_artifact_run_id_binding" in report["missing_gates"] + assert [item["gate"] for item in report["missing_gate_remediations"]] == report["missing_gates"] + remediations_by_gate = {item["gate"]: item for item in report["missing_gate_remediations"]} + assert ( + remediations_by_gate["archive_verify_report_present"]["target_action_id"] + == "target_transfer_archive_verify" + ) + assert ( + remediations_by_gate["archive_verify_report_present"]["required_artifact_path"] + == TARGET_ARTIFACT_PATHS["archive_verify_report"] + ) + assert remediations_by_gate["load_ladder_report_present"]["target_action_id"] == "target_load_ladder" + assert remediations_by_gate["soak_report_present"]["target_action_id"] == "target_soak" + assert remediations_by_gate["command_log_manifest_present"]["target_action_id"] == "m12_test_commands.sh" + assert report["load_ladder"]["present"] is False + assert report["soak"]["present"] is False + assert report["command_logs"]["present"] is False + assert report["archive_verify"]["present"] is False + assert report["archive_verify"]["validation_errors"] == [] + assert report["preflight"]["validation_errors"] == [] + assert report["preflight"]["inference_engine_feasibility"]["decision"] == "available" + assert report["preflight"]["container_runtimes"] + assert "status_output" in report["preflight"]["ray_cluster"] + assert report["swe_probe"]["validation_errors"] == [] + assert report["verl_export"]["validation_errors"] == [] + assert report["ray_probe"]["validation_errors"] == [] + assert report["warm_vs_cold"]["validation_errors"] == [] + assert report["load_ladder"]["validation_errors"] == [] + assert report["soak"]["validation_errors"] == [] + assert report["command_logs"]["missing_command_log_ids"] == sorted(REQUIRED_COMMAND_LOG_IDS) + assert report["target_run_identity"]["run_ids_match_command_logs"] is False + assert "swe_run_summary" in report["target_run_identity"]["missing_artifact_target_run_ids"] + assert report["swe_probe"]["row_count"] == 10 + assert report["swe_probe"]["row_status_counts"] == { + "accepted": 7, + "rejected": 1, + "quarantined": 2, + "other": 0, + } + assert validate_m12_final_report(report) == [] + + +def test_m12_final_report_write_persists_json(tmp_path) -> None: + output_path = tmp_path / "m12_final_report.json" + + report = write_m12_final_report( + output_path=output_path, + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + ) + + assert output_path.exists() + assert json.loads(output_path.read_text(encoding="utf-8"))["report_id"] == report["report_id"] + + +def test_pending_target_templates_make_final_report_noneligible_without_crashing(tmp_path) -> None: + load_path = tmp_path / "load_ladder_report.json" + soak_path = tmp_path / "soak_report.json" + command_log_path = tmp_path / "command_log_manifest.json" + load_path.write_text(json.dumps(LOAD_LADDER_REPORT_TEMPLATE), encoding="utf-8") + soak_path.write_text(json.dumps(SOAK_REPORT_TEMPLATE), encoding="utf-8") + command_log_path.write_text(json.dumps(COMMAND_LOG_MANIFEST_TEMPLATE), encoding="utf-8") + + report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + load_ladder_report_path=load_path, + soak_report_path=soak_path, + command_log_manifest_path=command_log_path, + ) + + assert report["m12_score_eligible"] is False + assert report["load_ladder"]["present"] is True + assert report["soak"]["present"] is True + assert report["command_logs"]["present"] is True + assert "load_ladder_required_levels_passed" in report["missing_gates"] + assert "load_ladder_report_valid" in report["missing_gates"] + assert "load_ladder_distributed" in report["missing_gates"] + assert report["load_ladder"]["validation_errors"] + assert "soak_status_passed" in report["missing_gates"] + assert "soak_report_valid" in report["missing_gates"] + assert "soak_duration_at_least_2h" in report["missing_gates"] + assert "soak_no_runtime_failures" in report["missing_gates"] + assert "soak_distributed" in report["missing_gates"] + assert report["soak"]["validation_errors"] + assert "command_log_manifest_complete" in report["missing_gates"] + assert "command_log_hashes_present" in report["missing_gates"] + assert "command_log_hashes_verified" in report["missing_gates"] + assert "command_log_commands_passed" in report["missing_gates"] + assert validate_m12_final_report(report) == [] + + +def test_malformed_optional_target_artifacts_make_final_report_noneligible_without_crashing(tmp_path) -> None: + load_path = tmp_path / "load_ladder_report.json" + soak_path = tmp_path / "soak_report.json" + command_log_path = tmp_path / "command_log_manifest.json" + load_path.write_text("{not valid load ladder json", encoding="utf-8") + soak_path.write_text("{not valid soak json", encoding="utf-8") + command_log_path.write_text("{not valid command log json", encoding="utf-8") + + report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + load_ladder_report_path=load_path, + soak_report_path=soak_path, + command_log_manifest_path=command_log_path, + ) + + assert report["m12_score_eligible"] is False + assert report["load_ladder"]["present"] is True + assert "JSONDecodeError" in report["load_ladder"]["read_error"] + assert "load_ladder_report_valid" in report["missing_gates"] + assert report["load_ladder"]["validation_errors"] == [ + f"load ladder report unreadable: {report['load_ladder']['read_error']}" + ] + assert report["soak"]["present"] is True + assert "JSONDecodeError" in report["soak"]["read_error"] + assert "soak_report_valid" in report["missing_gates"] + assert report["soak"]["validation_errors"] == [f"soak report unreadable: {report['soak']['read_error']}"] + assert report["command_logs"]["present"] is True + assert "JSONDecodeError" in report["command_logs"]["read_error"] + assert "command_log_manifest_valid" in report["missing_gates"] + assert report["command_logs"]["manifest_validation_errors"] == [ + f"command log manifest unreadable: {report['command_logs']['read_error']}" + ] + assert validate_m12_final_report(report) == [] + + +def test_malformed_required_preflight_artifact_make_final_report_noneligible_without_crashing( + tmp_path, +) -> None: + preflight_path = tmp_path / "m12_preflight_report.json" + preflight_path.write_text("{not valid preflight json", encoding="utf-8") + + report = build_m12_final_report( + preflight_report_path=preflight_path, + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + ) + + assert report["m12_score_eligible"] is False + assert report["preflight"]["present"] is True + assert "JSONDecodeError" in report["preflight"]["read_error"] + assert "preflight_report_readable" in report["missing_gates"] + assert "preflight_report_valid" in report["missing_gates"] + assert "preflight_passed" in report["missing_gates"] + assert report["preflight"]["validation_errors"] == [ + f"preflight report unreadable: {report['preflight']['read_error']}" + ] + assert validate_m12_final_report(report) == [] + + +def test_malformed_required_swe_verl_ray_and_warm_artifacts_make_final_report_noneligible( + tmp_path, +) -> None: + swe_path = tmp_path / "run_summary.json" + verl_path = tmp_path / "smoke_consumer_report.json" + ray_path = tmp_path / "ray_probe_report.json" + warm_path = tmp_path / "warm_vs_cold_report.json" + swe_path.write_text("{not valid swe json", encoding="utf-8") + verl_path.write_text("{not valid verl json", encoding="utf-8") + ray_path.write_text("{not valid ray json", encoding="utf-8") + warm_path.write_text("{not valid warm json", encoding="utf-8") + + report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=swe_path, + verl_smoke_report_path=verl_path, + ray_probe_report_path=ray_path, + warm_vs_cold_report_path=warm_path, + ) + + assert report["m12_score_eligible"] is False + assert "swe_run_summary_readable" in report["missing_gates"] + assert "swe_run_summary_valid" in report["missing_gates"] + assert "SWE run summary unreadable" in report["swe_probe"]["validation_errors"][0] + assert "verl_smoke_report_readable" in report["missing_gates"] + assert "verl_smoke_report_valid" in report["missing_gates"] + assert "VeRL smoke report unreadable" in report["verl_export"]["validation_errors"][0] + assert "ray_probe_report_readable" in report["missing_gates"] + assert "ray_probe_report_valid" in report["missing_gates"] + assert "Ray probe report unreadable" in report["ray_probe"]["validation_errors"][0] + assert "warm_vs_cold_report_readable" in report["missing_gates"] + assert "warm_vs_cold_report_valid" in report["missing_gates"] + assert "warm-vs-cold report unreadable" in report["warm_vs_cold"]["validation_errors"][0] + assert validate_m12_final_report(report) == [] + + +def test_forced_m12_eligibility_is_rejected_when_gates_are_missing() -> None: + report = _local_report() + report["m12_score_eligible"] = True + + errors = validate_m12_final_report(report) + + assert "m12_score_eligible must match final-report embedded evidence gates" in errors + assert "m12_score_eligible cannot be true while missing_gates is non-empty" in errors + assert "eligible report requires preflight.status=preflight_passed" not in errors + assert "eligible report requires preflight runtime fingerprint" not in errors + assert "eligible report requires MI300X product evidence" not in errors + assert "eligible report requires torch device_count >= 8" not in errors + assert "eligible report requires VeRL import availability" not in errors + assert "eligible report requires Ray distributed mode, not local_mode" in errors + assert "eligible report requires load ladder report" in errors + assert "eligible report requires soak report" in errors + assert "eligible report requires distributed soak, not local_mode" in errors + assert "eligible report requires command log manifest" in errors + + +def test_m12_final_report_rejects_unknown_or_unmapped_missing_gate() -> None: + report = _local_report() + report["missing_gates"].append("synthetic_unknown_gate") + report["missing_gate_remediations"].append( + { + "gate": "synthetic_unknown_gate", + "blocking_stage": "unknown", + "target_action_id": None, + "required_artifact_path": None, + "operator_action": "synthetic action", + } + ) + + errors = validate_m12_final_report(report) + + assert "unknown missing gate: synthetic_unknown_gate" in errors + assert "missing_gate_remediations row 31 has unknown gate: synthetic_unknown_gate" in errors + + +def test_m12_final_report_requires_remediation_rows_to_match_missing_gates() -> None: + report = _local_report() + report["missing_gate_remediations"] = report["missing_gate_remediations"][:-1] + + errors = validate_m12_final_report(report) + + assert "missing_gate_remediations gates must match missing_gates in order" in errors + + +def test_m12_final_report_rejects_stale_missing_gates_against_embedded_evidence() -> None: + report = _local_report() + removed_gate = "ray_probe_distributed" + report["missing_gates"] = [gate for gate in report["missing_gates"] if gate != removed_gate] + report["missing_gate_remediations"] = [ + item for item in report["missing_gate_remediations"] if item["gate"] != removed_gate + ] + + errors = validate_m12_final_report(report) + + assert "missing_gates must match final-report embedded evidence gates" in errors + + +def test_m12_final_report_rejects_stale_artifact_path_policy_summary() -> None: + report = _local_report() + report["artifact_path_policy"]["paths_match_target_defaults"] = True + + errors = validate_m12_final_report(report) + + assert "artifact_path_policy.paths_match_target_defaults is stale" in errors + + +def test_m12_final_report_requires_canonical_artifact_path_policy_metadata() -> None: + report = _local_report() + report["artifact_path_policy"]["policy_id"] = "stale-policy" + report["artifact_path_policy"]["required_target_paths"]["swe_run_summary"] = "wrong/path.json" + report["artifact_path_policy"]["reason"] = "" + + errors = validate_m12_final_report(report) + + assert "artifact_path_policy.policy_id must be m12_target_artifact_paths_v1" in errors + assert "artifact_path_policy.required_target_paths must match canonical M12 target artifact paths" in errors + assert "artifact_path_policy.reason must be non-empty" in errors + + +def test_m12_final_report_rejects_artifact_path_section_path_drift() -> None: + report = _local_report() + report["swe_probe"]["path"] = "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/stale_swe/run_summary.json" + + errors = validate_m12_final_report(report) + + assert "artifact_paths.swe_run_summary must match embedded section path" in errors + + +def test_m12_final_report_rejects_stale_empty_command_log_summary() -> None: + report = _local_report() + report["command_logs"]["archived_command_ids"] = ["target_preflight"] + report["command_logs"]["missing_command_log_ids"] = [] + report["command_logs"]["target_run_ids"] = ["stale-target-run"] + report["command_logs"]["single_target_run_id"] = "stale-target-run" + report["command_logs"]["command_text_mismatches"] = [ + { + "command_id": "target_preflight", + "expected": REQUIRED_COMMAND_LOG_COMMANDS["target_preflight"], + "observed": "python stale.py", + } + ] + report["command_logs"]["command_count"] = 1 + report["command_logs"]["hash_verified_command_ids"] = ["target_preflight"] + report["command_logs"]["all_required_logs_archived"] = True + report["command_logs"]["all_required_commands_passed"] = True + + errors = validate_m12_final_report(report) + + assert "command_logs.archived_command_ids is stale" in errors + assert "command_logs.missing_command_log_ids is stale" in errors + assert "command_logs.target_run_ids is stale" in errors + assert "command_logs.single_target_run_id is stale" in errors + assert "command_logs.command_text_mismatches is stale" in errors + assert "command_logs.command_count is stale" in errors + assert "command_logs.hash_verified_command_ids contains unknown command IDs" in errors + assert "command_logs.hash_verified_command_ids contains unarchived command IDs" in errors + assert "command_logs.all_required_logs_archived cannot be true without a readable manifest" in errors + assert "command_logs.all_required_commands_passed cannot be true without a readable manifest" in errors + + +def test_m12_final_report_rejects_stale_populated_command_log_summary(tmp_path) -> None: + manifest_path = _write_complete_command_log_manifest(tmp_path) + report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + command_log_manifest_path=manifest_path, + ) + assert validate_m12_final_report(report) == [] + + report["command_logs"]["archived_command_ids"] = report["command_logs"]["archived_command_ids"][:-1] + report["command_logs"]["missing_command_log_ids"] = ["target_preflight"] + report["command_logs"]["target_run_ids"] = [] + report["command_logs"]["single_target_run_id"] = None + report["command_logs"]["command_count"] = 0 + report["command_logs"]["all_required_logs_archived"] = False + report["command_logs"]["all_required_commands_passed"] = False + + errors = validate_m12_final_report(report) + + assert "command_logs.archived_command_ids is stale" in errors + assert "command_logs.missing_command_log_ids is stale" in errors + assert "command_logs.target_run_ids is stale" in errors + assert "command_logs.single_target_run_id is stale" in errors + assert "command_logs.command_count is stale" in errors + assert "command_logs.all_required_logs_archived is stale" in errors + assert "command_logs.all_required_commands_passed is stale" in errors + + +def test_m12_final_report_rejects_stale_target_run_identity_summary() -> None: + report = _local_report() + report["target_run_identity"]["artifact_target_run_ids"]["swe_run_summary"] = "stale-run" + report["target_run_identity"]["missing_artifact_target_run_ids"] = [] + report["target_run_identity"]["run_ids_match_command_logs"] = True + + errors = validate_m12_final_report(report) + + assert "target_run_identity artifact_target_run_ids must match embedded artifact sections" in errors + assert "target_run_identity missing_artifact_target_run_ids is stale" in errors + assert "target_run_identity run_ids_match_command_logs is stale" in errors + + +def test_m12_final_report_rejects_empty_or_non_target_remediation_fields() -> None: + report = _local_report() + report["missing_gate_remediations"][0]["operator_action"] = "" + report["missing_gate_remediations"][1]["required_artifact_path"] = "/tmp/not-a-target-artifact.json" + report["missing_gate_remediations"][2]["target_action_id"] = "" + + errors = validate_m12_final_report(report) + + assert "missing_gate_remediations row 1 requires operator_action" in errors + assert "missing_gate_remediations row 2 required_artifact_path must be an M12 target artifact path" in errors + assert "missing_gate_remediations row 3 target_action_id must be non-empty when set" in errors + + +def test_m12_final_report_remediation_summary_groups_by_target_action() -> None: + report = _local_report() + + summary = summarize_m12_final_report_remediations(report) + + assert summary["summary_id"] == "bb_zyphra_rl_phase1_m12_final_report_remediation_summary_v1" + assert summary["claim_boundary"] == "final_report_remediation_summary_not_scorecard_update" + assert summary["scorecard_update_allowed"] is False + assert summary["m12_points_awarded"] is False + assert summary["m12_score_eligible"] is False + assert summary["missing_gate_count"] == len(report["missing_gates"]) + assert summary["remediation_count"] == len(report["missing_gate_remediations"]) + assert summary["remediation_gates_match_missing_gates"] is True + actions = {item["target_action_id"]: item for item in summary["next_target_actions"]} + assert {"final_report", "target_transfer_archive_verify", "target_ray_warm_pool", "target_load_ladder", "target_soak", "m12_test_commands.sh"} <= set(actions) + assert "archive_verify_report_present" in actions["target_transfer_archive_verify"]["gates"] + assert TARGET_ARTIFACT_PATHS["archive_verify_report"] in actions["target_transfer_archive_verify"]["required_artifact_paths"] + assert "command_log_manifest_present" in actions["m12_test_commands.sh"]["gates"] + assert TARGET_ARTIFACT_PATHS["command_log_manifest"] in actions["m12_test_commands.sh"]["required_artifact_paths"] + assert validate_m12_final_report_remediation_summary(summary) == [] + + +def test_m12_final_report_remediation_summary_rejects_stale_counts() -> None: + summary = summarize_m12_final_report_remediations(_local_report()) + summary["missing_gate_count"] += 1 + summary["remediation_count"] -= 1 + + errors = validate_m12_final_report_remediation_summary(summary) + + assert "remediation_count must match next_target_actions gate count" in errors + assert "missing_gate_count must match next_target_actions gate count" in errors + assert "remediation_count must match missing_gate_count when gates match" in errors + + +def test_m12_final_report_remediation_summary_rejects_unsafe_action_fields() -> None: + summary = summarize_m12_final_report_remediations(_local_report()) + summary["next_target_actions"][0]["target_action_id"] = "" + summary["next_target_actions"][0]["gates"].append("synthetic_unknown_gate") + summary["next_target_actions"][0]["required_artifact_paths"].append("/tmp/not-target.json") + summary["next_target_actions"][0]["operator_actions"].append("") + + errors = validate_m12_final_report_remediation_summary(summary) + + assert "next_target_actions row 1 requires target_action_id" in errors + assert "next_target_actions row 1 has unknown gate: synthetic_unknown_gate" in errors + assert "next_target_actions row 1 has non-target artifact path: /tmp/not-target.json" in errors + assert "next_target_actions row 1 has empty operator_action" in errors + + +def test_m12_final_report_remediation_summary_cli_writes_json(tmp_path) -> None: + report_path = tmp_path / "m12_final_report.json" + output_path = tmp_path / "m12_remediation_summary.json" + report_path.write_text(json.dumps(_local_report(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/summarize_m12_final_report_remediations.py", + "--final-report", + str(report_path), + "--output", + str(output_path), + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0 + assert "missing_gates=30" in result.stdout + assert "remediations=30" in result.stdout + assert "target_transfer_archive_verify" in result.stdout + summary = json.loads(output_path.read_text(encoding="utf-8")) + assert summary["remediation_gates_match_missing_gates"] is True + assert summary["next_target_actions"] + + +def test_m12_final_report_require_eligible_cli_exits_nonzero_for_local_probe(tmp_path) -> None: + output_path = tmp_path / "m12_final_report.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/build_m12_final_report.py", + "--output", + str(output_path), + "--preflight-report", + str(PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json"), + "--swe-run-summary", + str(PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json"), + "--verl-smoke-report", + str(PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json"), + "--ray-probe-report", + str(PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json"), + "--warm-vs-cold-report", + str(PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json"), + "--require-eligible", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode in {1, 4} + if result.returncode == 4: + assert output_path.exists() + assert "score_eligible=False" in result.stdout + else: + assert "invalid_m12_final_report" in result.stderr + + +def test_synthetic_complete_m12_report_validates() -> None: + target_run_id = "m12-target-run-test" + report = _local_report() + report["m12_score_eligible"] = True + report["missing_gates"] = [] + report["missing_gate_remediations"] = [] + report["artifact_paths"] = dict(TARGET_ARTIFACT_PATHS) + report["artifact_path_policy"]["paths_match_target_defaults"] = True + report["archive_verify"] = { + "present": True, + "path": TARGET_ARTIFACT_PATHS["archive_verify_report"], + "read_error": None, + "report_id": "bb_zyphra_rl_phase1_m12_archive_verify_report_v1", + "claim_boundary": "transfer_archive_verification_not_m12_validation", + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "status": "passed", + "archive_manifest_id": "bb_zyphra_rl_phase1_m12_transfer_archive_manifest_v1", + "archive_claim_boundary": "transfer_archive_only_not_m12_validation", + "archive_sha256": "sha256:" + ("b" * 64), + "included_entry_count": 213, + "all_required_artifacts_present": True, + "all_transfer_requirements_covered": True, + "archive_contains_source_overlay": True, + "archive_deterministic": True, + "source_paths_portable": True, + "errors": [], + "validation_errors": [], + } + report["preflight"]["path"] = TARGET_ARTIFACT_PATHS["preflight_report"] + report["swe_probe"]["path"] = TARGET_ARTIFACT_PATHS["swe_run_summary"] + report["verl_export"]["path"] = TARGET_ARTIFACT_PATHS["verl_smoke_report"] + report["ray_probe"]["path"] = TARGET_ARTIFACT_PATHS["ray_probe_report"] + report["warm_vs_cold"]["path"] = TARGET_ARTIFACT_PATHS["warm_vs_cold_report"] + report["preflight"]["status"] = "preflight_passed" + report["preflight"]["target_run_id"] = target_run_id + report["preflight"]["blockers"] = [] + report["preflight"]["gpu"]["rocminfo_available"] = True + report["preflight"]["gpu"]["mi300x_product_evidence"] = True + report["preflight"]["gpu"]["torch_probe"]["device_count"] = 8 + report["preflight"]["gpu"]["torch_probe"]["device_names"] = ["AMD Instinct MI300X"] * 8 + report["preflight"]["python_modules"]["verl"]["available"] = True + report["preflight"]["python_modules"]["ray"]["available"] = True + report["preflight"]["filesystem_cas_smoke"]["status"] = "passed" + fingerprint = report["preflight"]["runtime_fingerprint"] + fingerprint["tool_presence"] = { + "rocm_smi": report["preflight"]["gpu"]["rocm_smi_available"], + "rocminfo": report["preflight"]["gpu"]["rocminfo_available"], + "docker": report["preflight"]["container_runtimes"]["docker"], + "gvisor_runsc": report["preflight"]["container_runtimes"]["gvisor_runsc"], + "firecracker": report["preflight"]["container_runtimes"]["firecracker"], + } + fingerprint["python_modules"] = report["preflight"]["python_modules"] + fingerprint["gpu_summary"] = { + "rocm_smi_output": report["preflight"]["gpu"]["rocm_smi_output"], + "torch_probe": report["preflight"]["gpu"]["torch_probe"], + } + fingerprint["ray_summary"] = {"status_output": report["preflight"]["ray_cluster"]["status_output"]} + fingerprint["sha256"] = _fingerprint_sha256(fingerprint) + report["ray_probe"]["ray_local_mode"] = False + report["swe_probe"]["target_run_id"] = target_run_id + report["verl_export"]["target_run_id"] = target_run_id + report["ray_probe"]["target_run_id"] = target_run_id + report["warm_vs_cold"]["target_run_id"] = target_run_id + report["load_ladder"] = { + "present": True, + "path": TARGET_ARTIFACT_PATHS["load_ladder_report"], + "target_run_id": target_run_id, + "validation_errors": [], + "required_levels": [5, 20, 50], + "optional_levels": [100], + "concurrency_levels": [ + {"target_sessions": 5, "status": "passed", "ray_local_mode": False}, + {"target_sessions": 20, "status": "passed", "ray_local_mode": False}, + {"target_sessions": 50, "status": "passed", "ray_local_mode": False}, + ], + "resource_skips": [{"target_sessions": 100, "reason": "insufficient target maintenance window"}], + "policy_version_integrity": True, + "queue_backpressure_integrity": True, + } + report["soak"] = { + "present": True, + "path": TARGET_ARTIFACT_PATHS["soak_report"], + "target_run_id": target_run_id, + "validation_errors": [], + "minimum_duration_seconds": 7200, + "duration_seconds": 7200, + "status": "passed", + "runtime_failure_count": 0, + "ray_local_mode": False, + } + command_entries = [ + { + "command_id": command_id, + "status": "passed", + "exit_code": 0, + "log_path": f"m12_command_logs/{command_id}.log", + "sha256": "sha256:" + ("a" * 64), + "started_at": "2026-06-17T00:00:00Z", + "completed_at": "2026-06-17T00:00:01Z", + "target_run_id": target_run_id, + "command": REQUIRED_COMMAND_LOG_COMMANDS[command_id], + } + for command_id in REQUIRED_COMMAND_LOG_IDS + ] + report["command_logs"] = { + "present": True, + "path": TARGET_ARTIFACT_PATHS["command_log_manifest"], + "manifest_id": "bb_zyphra_rl_phase1_m12_command_log_manifest_v1", + "manifest_validation_errors": [], + "required_command_ids": list(REQUIRED_COMMAND_LOG_IDS), + "manifest_required_command_ids": list(REQUIRED_COMMAND_LOG_IDS), + "archived_command_ids": list(REQUIRED_COMMAND_LOG_IDS), + "hash_verified_command_ids": list(REQUIRED_COMMAND_LOG_IDS), + "missing_command_log_ids": [], + "target_run_ids": [target_run_id], + "single_target_run_id": target_run_id, + "command_text_mismatches": [], + "all_required_logs_archived": True, + "all_required_commands_passed": True, + "command_count": len(command_entries), + "commands": command_entries, + } + report["target_run_identity"] = { + "single_command_log_target_run_id": target_run_id, + "artifact_target_run_ids": { + "preflight_report": target_run_id, + "swe_run_summary": target_run_id, + "verl_smoke_report": target_run_id, + "ray_probe_report": target_run_id, + "warm_vs_cold_report": target_run_id, + "load_ladder_report": target_run_id, + "soak_report": target_run_id, + }, + "missing_artifact_target_run_ids": [], + "mismatched_artifact_target_run_ids": {}, + "run_ids_match_command_logs": True, + } + + assert validate_m12_final_report(report) == [] + + stale_eligibility_report = json.loads(json.dumps(report)) + stale_eligibility_report["m12_score_eligible"] = False + assert "m12_score_eligible must match final-report embedded evidence gates" in validate_m12_final_report( + stale_eligibility_report + ) + + local_artifact_path_report = json.loads(json.dumps(report)) + local_artifact_path_report["artifact_paths"]["swe_run_summary"] = ( + "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/run_summary.json" + ) + local_artifact_path_report["artifact_path_policy"]["paths_match_target_defaults"] = False + artifact_path_errors = validate_m12_final_report(local_artifact_path_report) + assert "eligible report requires target-node default artifact paths" in artifact_path_errors + assert "eligible report requires artifact_path_policy.paths_match_target_defaults=true" in artifact_path_errors + + missing_fingerprint_report = json.loads(json.dumps(report)) + missing_fingerprint_report["preflight"].pop("runtime_fingerprint") + assert "eligible report requires preflight runtime fingerprint" in validate_m12_final_report( + missing_fingerprint_report + ) + + tampered_fingerprint_report = json.loads(json.dumps(report)) + tampered_fingerprint_report["preflight"]["runtime_fingerprint"]["platform"]["machine"] = "tampered-machine" + assert "eligible report requires preflight runtime fingerprint" in validate_m12_final_report( + tampered_fingerprint_report + ) + + unsafe_env_fingerprint_report = json.loads(json.dumps(report)) + unsafe_env_fingerprint_report["preflight"]["runtime_fingerprint"]["sanitized_environment"]["values"][ + "OPENAI_API_KEY" + ] = "should-not-be-here" + assert "eligible report requires preflight runtime fingerprint" in validate_m12_final_report( + unsafe_env_fingerprint_report + ) + + unredacted_path_report = json.loads(json.dumps(report)) + unredacted_fp = unredacted_path_report["preflight"]["runtime_fingerprint"] + unredacted_fp["sanitized_environment"]["values"]["CONDA_DEFAULT_ENV"] = "/tmp/private-conda-env" + unredacted_fp["sanitized_environment"]["redactions"].pop("CONDA_DEFAULT_ENV", None) + unredacted_fp["sha256"] = _fingerprint_sha256(unredacted_fp) + assert "eligible report requires preflight runtime fingerprint" in validate_m12_final_report( + unredacted_path_report + ) + + extra_top_level_report = json.loads(json.dumps(report)) + extra_top_level_fp = extra_top_level_report["preflight"]["runtime_fingerprint"] + extra_top_level_fp["unexpected_section"] = {"leak": "not allowed"} + extra_top_level_fp["sha256"] = _fingerprint_sha256(extra_top_level_fp) + assert "eligible report requires preflight runtime fingerprint" in validate_m12_final_report( + extra_top_level_report + ) + + extra_nested_report = json.loads(json.dumps(report)) + extra_nested_fp = extra_nested_report["preflight"]["runtime_fingerprint"] + extra_nested_fp["sanitized_environment"]["unexpected_field"] = "not allowed" + extra_nested_fp["sha256"] = _fingerprint_sha256(extra_nested_fp) + assert "eligible report requires preflight runtime fingerprint" in validate_m12_final_report( + extra_nested_report + ) + + preflight_validation_error_report = json.loads(json.dumps(report)) + preflight_validation_error_report["preflight"]["validation_errors"] = ["synthetic component error"] + assert "eligible report requires preflight validation errors to be empty" in validate_m12_final_report( + preflight_validation_error_report + ) + + preflight_top_level_drift_report = json.loads(json.dumps(report)) + preflight_top_level_drift_report["preflight"]["container_runtimes"]["docker"] = False + preflight_top_level_drift_report["preflight"]["container_runtimes"]["gvisor_runsc"] = False + preflight_top_level_drift_report["preflight"]["container_runtimes"]["firecracker"] = False + preflight_top_level_drift_errors = validate_m12_final_report(preflight_top_level_drift_report) + assert "eligible report requires preflight runtime fingerprint" in preflight_top_level_drift_errors + assert "eligible report requires at least one container runtime" in preflight_top_level_drift_errors + + stale_target_artifact_report = json.loads(json.dumps(report)) + stale_target_artifact_report["target_run_identity"]["artifact_target_run_ids"]["swe_run_summary"] = "stale-run" + stale_target_artifact_report["target_run_identity"]["mismatched_artifact_target_run_ids"] = { + "swe_run_summary": {"expected": target_run_id, "observed": "stale-run"} + } + stale_target_artifact_report["target_run_identity"]["run_ids_match_command_logs"] = False + target_identity_errors = validate_m12_final_report(stale_target_artifact_report) + assert "target_run_identity artifact_target_run_ids must match embedded artifact sections" in target_identity_errors + assert "target_run_identity mismatched_artifact_target_run_ids is stale" in target_identity_errors + assert "eligible report requires target artifacts to share command-log target_run_id" in target_identity_errors + + stale_section_target_run_report = json.loads(json.dumps(report)) + stale_section_target_run_report["swe_probe"]["target_run_id"] = "stale-run" + stale_section_errors = validate_m12_final_report(stale_section_target_run_report) + assert "target_run_identity artifact_target_run_ids must match embedded artifact sections" in stale_section_errors + assert "target_run_identity mismatched_artifact_target_run_ids is stale" in stale_section_errors + assert "eligible report requires swe_run_summary target_run_id to match command logs" in stale_section_errors + + no_inference_report = json.loads(json.dumps(report)) + no_inference_report["preflight"]["inference_engine_feasibility"] = { + "decision": "blocked_no_vllm_or_sglang", + "vllm_available": False, + "sglang_available": False, + } + assert "eligible report requires vLLM or SGLang inference-engine availability" in validate_m12_final_report( + no_inference_report + ) + + no_container_report = json.loads(json.dumps(report)) + no_container_report["preflight"]["container_runtimes"] = { + "docker": False, + "gvisor_runsc": False, + "firecracker": False, + } + assert "eligible report requires at least one container runtime" in validate_m12_final_report( + no_container_report + ) + + load_ladder_validation_error_report = json.loads(json.dumps(report)) + load_ladder_validation_error_report["load_ladder"]["validation_errors"] = ["synthetic component error"] + assert "eligible report requires load ladder validation errors to be empty" in validate_m12_final_report( + load_ladder_validation_error_report + ) + + soak_validation_error_report = json.loads(json.dumps(report)) + soak_validation_error_report["soak"]["validation_errors"] = ["synthetic component error"] + assert "eligible report requires soak validation errors to be empty" in validate_m12_final_report( + soak_validation_error_report + ) + + swe_validation_error_report = json.loads(json.dumps(report)) + swe_validation_error_report["swe_probe"]["validation_errors"] = ["synthetic component error"] + assert "eligible report requires SWE run summary validation errors to be empty" in validate_m12_final_report( + swe_validation_error_report + ) + + verl_validation_error_report = json.loads(json.dumps(report)) + verl_validation_error_report["verl_export"]["validation_errors"] = ["synthetic component error"] + assert "eligible report requires VeRL smoke validation errors to be empty" in validate_m12_final_report( + verl_validation_error_report + ) + + ray_validation_error_report = json.loads(json.dumps(report)) + ray_validation_error_report["ray_probe"]["validation_errors"] = ["synthetic component error"] + assert "eligible report requires Ray probe validation errors to be empty" in validate_m12_final_report( + ray_validation_error_report + ) + + warm_validation_error_report = json.loads(json.dumps(report)) + warm_validation_error_report["warm_vs_cold"]["validation_errors"] = ["synthetic component error"] + assert "eligible report requires warm-vs-cold validation errors to be empty" in validate_m12_final_report( + warm_validation_error_report + ) + + +def test_m12_final_report_verifies_command_log_hashes(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + log_dir.mkdir() + for command_id in REQUIRED_COMMAND_LOG_IDS: + log_path = log_dir / f"{command_id}.log" + _write_wrapper_style_log( + log_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + ) + record_command_log_result( + manifest_path=manifest_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + log_path=log_path, + exit_code=0, + started_at="2026-06-17T00:00:00Z", + completed_at="2026-06-17T00:00:01Z", + target_run_id="m12-target-run-test", + ) + + report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + command_log_manifest_path=manifest_path, + ) + + assert "command_log_manifest_present" not in report["missing_gates"] + assert "command_log_manifest_complete" not in report["missing_gates"] + assert "command_log_hashes_present" not in report["missing_gates"] + assert "command_log_hashes_verified" not in report["missing_gates"] + assert "command_log_commands_passed" not in report["missing_gates"] + assert "command_log_manifest_valid" not in report["missing_gates"] + assert "command_log_single_target_run_id" not in report["missing_gates"] + assert "command_log_required_ids_canonical" not in report["missing_gates"] + assert report["command_logs"]["manifest_validation_errors"] == [] + assert report["command_logs"]["manifest_required_command_ids"] == list(REQUIRED_COMMAND_LOG_IDS) + assert report["command_logs"]["hash_verified_command_ids"] == sorted(REQUIRED_COMMAND_LOG_IDS) + assert report["command_logs"]["single_target_run_id"] == "m12-target-run-test" + + first_log = log_dir / f"{REQUIRED_COMMAND_LOG_IDS[0]}.log" + first_log.write_text("tampered\n", encoding="utf-8") + tampered_report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + command_log_manifest_path=manifest_path, + ) + + assert "command_log_hashes_verified" in tampered_report["missing_gates"] + assert "command_log_manifest_valid" in tampered_report["missing_gates"] + assert tampered_report["command_logs"]["manifest_validation_errors"] == [ + f"attempt log sha256 mismatch: {REQUIRED_COMMAND_LOG_IDS[0]} attempt 1", + f"log sha256 mismatch: {REQUIRED_COMMAND_LOG_IDS[0]}", + ] + + +def test_m12_final_report_rejects_narrowed_command_log_required_ids(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + log_dir.mkdir() + for command_id in REQUIRED_COMMAND_LOG_IDS: + log_path = log_dir / f"{command_id}.log" + _write_wrapper_style_log( + log_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + ) + record_command_log_result( + manifest_path=manifest_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + log_path=log_path, + exit_code=0, + started_at="2026-06-17T00:00:00Z", + completed_at="2026-06-17T00:00:01Z", + target_run_id="m12-target-run-test", + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["required_command_ids"] = ["target_preflight"] + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + command_log_manifest_path=manifest_path, + ) + + assert "command_log_required_ids_canonical" in report["missing_gates"] + assert "command_log_manifest_valid" in report["missing_gates"] + assert report["command_logs"]["manifest_required_command_ids"] == ["target_preflight"] + assert report["command_logs"]["manifest_validation_errors"] == [ + "required_command_ids must equal canonical M12 required command IDs" + ] + + +def test_m12_final_report_rejects_stale_command_log_summary_flags(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + log_dir.mkdir() + for command_id in REQUIRED_COMMAND_LOG_IDS: + log_path = log_dir / f"{command_id}.log" + _write_wrapper_style_log( + log_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + ) + record_command_log_result( + manifest_path=manifest_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + log_path=log_path, + exit_code=0, + started_at="2026-06-17T00:00:00Z", + completed_at="2026-06-17T00:00:01Z", + target_run_id="m12-target-run-test", + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["all_required_logs_archived"] = False + manifest["all_required_commands_passed"] = False + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + command_log_manifest_path=manifest_path, + ) + + assert "command_log_manifest_complete" not in report["missing_gates"] + assert "command_log_manifest_valid" in report["missing_gates"] + assert "command_log_hashes_verified" not in report["missing_gates"] + assert "command_log_commands_passed" not in report["missing_gates"] + assert "command_log_required_logs_archived_summary" in report["missing_gates"] + assert "command_log_required_commands_passed_summary" in report["missing_gates"] + assert report["command_logs"]["manifest_validation_errors"] == [ + "all_required_logs_archived must match required command log rows", + "all_required_commands_passed must match required command statuses", + ] + + +def test_m12_final_report_rejects_missing_attempt_history(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + log_dir.mkdir() + for command_id in REQUIRED_COMMAND_LOG_IDS: + log_path = log_dir / f"{command_id}.log" + _write_wrapper_style_log( + log_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + ) + record_command_log_result( + manifest_path=manifest_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + log_path=log_path, + exit_code=0, + started_at="2026-06-17T00:00:00Z", + completed_at="2026-06-17T00:00:01Z", + target_run_id="m12-target-run-test", + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + target_entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + target_entry.pop("attempts") + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + command_log_manifest_path=manifest_path, + ) + + assert "command_log_manifest_valid" in report["missing_gates"] + assert report["command_logs"]["manifest_validation_errors"] == [ + "completed command entry must preserve attempts: target_preflight" + ] + + +def test_m12_final_report_rejects_mixed_target_run_ids(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + log_dir.mkdir() + for index, command_id in enumerate(REQUIRED_COMMAND_LOG_IDS): + log_path = log_dir / f"{command_id}.log" + target_run_id = "m12-target-run-a" if index == 0 else "m12-target-run-b" + _write_wrapper_style_log( + log_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + target_run_id=target_run_id, + ) + record_command_log_result( + manifest_path=manifest_path, + command_id=command_id, + command=REQUIRED_COMMAND_LOG_COMMANDS[command_id], + log_path=log_path, + exit_code=0, + started_at="2026-06-17T00:00:00Z", + completed_at="2026-06-17T00:00:01Z", + target_run_id=target_run_id, + ) + + report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + command_log_manifest_path=manifest_path, + ) + + assert "command_log_single_target_run_id" in report["missing_gates"] + assert report["command_logs"]["single_target_run_id"] is None + assert report["command_logs"]["target_run_ids"] == ["m12-target-run-a", "m12-target-run-b"] + + +def test_m12_final_report_rejects_required_command_text_mismatch(tmp_path) -> None: + manifest_path = tmp_path / "command_log_manifest.json" + log_dir = tmp_path / "logs" + log_dir.mkdir() + for command_id in REQUIRED_COMMAND_LOG_IDS: + log_path = log_dir / f"{command_id}.log" + command = ( + "python -c 'print(\"wrong command\")'" + if command_id == "target_preflight" + else REQUIRED_COMMAND_LOG_COMMANDS[command_id] + ) + _write_wrapper_style_log(log_path, command_id=command_id, command=command) + record_command_log_result( + manifest_path=manifest_path, + command_id=command_id, + command=command, + log_path=log_path, + exit_code=0, + started_at="2026-06-17T00:00:00Z", + completed_at="2026-06-17T00:00:01Z", + target_run_id="m12-target-run-test", + ) + + report = build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + command_log_manifest_path=manifest_path, + ) + + assert "command_log_expected_commands_match" in report["missing_gates"] + assert report["command_logs"]["command_text_mismatches"] == [ + { + "command_id": "target_preflight", + "expected": REQUIRED_COMMAND_LOG_COMMANDS["target_preflight"], + "observed": "python -c 'print(\"wrong command\")'", + } + ] diff --git a/tests/rl/m12/test_m12_load_soak.py b/tests/rl/m12/test_m12_load_soak.py new file mode 100644 index 00000000..f142d015 --- /dev/null +++ b/tests/rl/m12/test_m12_load_soak.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.m12 import ( + build_m12_load_ladder_report, + build_m12_soak_report, + validate_m12_load_ladder_report, + validate_m12_soak_report, +) + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_m12_load_ladder_report_builder_supports_pass_and_resource_skip() -> None: + package = load_env_package(PYTHON_TOY) + + report = build_m12_load_ladder_report( + package=package, + levels=[1, 2], + skip_levels={2: "unit-test resource skip"}, + min_rows_per_level=1, + local_mode=True, + ) + + assert report["report_id"] == "bb_zyphra_rl_phase1_m12_load_ladder_report_v1" + assert report["claim_boundary"] == "target_load_ladder_probe_not_scorecard_update" + assert report["policy_version_integrity"] is True + assert report["queue_backpressure_integrity"] is True + assert report["concurrency_levels"][0]["target_sessions"] == 1 + assert report["concurrency_levels"][0]["status"] == "passed" + assert report["concurrency_levels"][0]["row_count"] >= 1 + assert report["concurrency_levels"][1]["target_sessions"] == 2 + assert report["concurrency_levels"][1]["status"] == "resource_skipped" + assert report["resource_skips"] == [{"target_sessions": 2, "reason": "unit-test resource skip"}] + assert validate_m12_load_ladder_report( + report, + required_levels=[1], + optional_levels=[2], + require_distributed=False, + ) == [] + assert "load ladder requires distributed Ray for non-skipped levels" in validate_m12_load_ladder_report( + report, + required_levels=[1], + optional_levels=[2], + require_distributed=True, + ) + + +def test_m12_soak_report_builder_supports_tiny_local_smoke() -> None: + package = load_env_package(PYTHON_TOY) + + report = build_m12_soak_report( + package=package, + duration_seconds=0, + minimum_duration_seconds=0, + interval_seconds=0, + num_workers=1, + rows_per_iteration=1, + min_iterations=1, + local_mode=True, + ) + + assert report["report_id"] == "bb_zyphra_rl_phase1_m12_soak_report_v1" + assert report["claim_boundary"] == "target_soak_probe_not_scorecard_update" + assert report["status"] == "passed" + assert report["duration_seconds"] >= 0 + assert report["runtime_failure_count"] == 0 + assert report["row_count"] >= 1 + assert report["accepted_count"] >= 1 + assert validate_m12_soak_report( + report, + minimum_duration_seconds=0, + require_distributed=False, + ) == [] + assert "soak requires distributed Ray, not local_mode" in validate_m12_soak_report( + report, + minimum_duration_seconds=0, + require_distributed=True, + ) + + +def test_m12_load_ladder_cli_writes_report(tmp_path) -> None: + output = tmp_path / "load_ladder_report.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_load_ladder.py", + "--package", + str(PYTHON_TOY), + "--output", + str(output), + "--levels", + "1", + "--min-rows-per-level", + "1", + "--local-mode", + "--smoke-mode", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["concurrency_levels"][0]["status"] == "passed" + assert "policy_version_integrity=True" in result.stdout + + +def test_m12_load_ladder_cli_fails_closed_without_smoke_mode(tmp_path) -> None: + output = tmp_path / "load_ladder_report.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_load_ladder.py", + "--package", + str(PYTHON_TOY), + "--output", + str(output), + "--levels", + "1", + "--min-rows-per-level", + "1", + "--local-mode", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode != 0 + assert output.exists() + assert "invalid_m12_load_ladder_report" in result.stderr + + +def test_m12_soak_cli_writes_report(tmp_path) -> None: + output = tmp_path / "soak_report.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_soak.py", + "--package", + str(PYTHON_TOY), + "--output", + str(output), + "--duration-seconds", + "0", + "--minimum-duration-seconds", + "0", + "--interval-seconds", + "0", + "--num-workers", + "1", + "--rows-per-iteration", + "1", + "--min-iterations", + "1", + "--local-mode", + "--smoke-mode", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["status"] == "passed" + assert payload["runtime_failure_count"] == 0 + assert "status=passed" in result.stdout + + +def test_m12_soak_cli_fails_closed_without_smoke_mode(tmp_path) -> None: + output = tmp_path / "soak_report.json" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_soak.py", + "--package", + str(PYTHON_TOY), + "--output", + str(output), + "--duration-seconds", + "0", + "--minimum-duration-seconds", + "0", + "--interval-seconds", + "0", + "--num-workers", + "1", + "--rows-per-iteration", + "1", + "--min-iterations", + "1", + "--local-mode", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode != 0 + assert output.exists() + assert "invalid_m12_soak_report" in result.stderr diff --git a/tests/rl/m12/test_m12_preflight.py b/tests/rl/m12/test_m12_preflight.py new file mode 100644 index 00000000..dfdd7677 --- /dev/null +++ b/tests/rl/m12/test_m12_preflight.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import json +import hashlib +import subprocess +import sys +from pathlib import Path + +from breadboard.rl.m12 import run_m12_preflight, validate_m12_preflight_report, write_m12_preflight_report +from breadboard.rl.m12.preflight import ENV_VALUE_REDACTED_ABSOLUTE_PATH, _safe_environment_snapshot + + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def _fingerprint_sha256(fingerprint: dict) -> str: + payload = dict(fingerprint) + payload.pop("sha256", None) + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def test_m12_preflight_is_non_scoring_and_has_completion_gate(tmp_path) -> None: + report = run_m12_preflight() + + assert report["report_id"] == "bb_zyphra_rl_phase1_m12_preflight_v1" + assert report["claim_boundary"] == "target_preflight_only_not_m12_validation" + assert report["status"] in {"blocked", "preflight_passed"} + assert report["gpu"]["required_accelerator_count"] == 8 + assert report["gpu"]["required_accelerator_family"] == "MI300X" + assert report["filesystem_cas_smoke"]["status"] == "passed" + assert report["filesystem_cas_smoke"]["sha256"].startswith("sha256:") + assert report["ray_cluster"]["module_available"] in {True, False} + assert report["inference_engine_feasibility"]["decision"] in {"available", "blocked_no_vllm_or_sglang"} + fingerprint = report["runtime_fingerprint"] + assert fingerprint["fingerprint_id"] == "bb_zyphra_rl_phase1_m12_runtime_fingerprint_v1" + assert fingerprint["sha256"].startswith("sha256:") + assert fingerprint["sha256"] == _fingerprint_sha256(fingerprint) + assert "/" not in fingerprint["platform"]["python_executable_name"] + assert ( + fingerprint["sanitized_environment"]["policy"] + == "allowlist_only_redact_path_values_no_secret_keys_no_absolute_python_paths" + ) + assert all("KEY" not in key and "TOKEN" not in key and "SECRET" not in key for key in fingerprint["sanitized_environment"]["keys"]) + assert isinstance(fingerprint["sanitized_environment"]["redactions"], dict) + assert "m12_completion_gate" in report + assert "final M12 report" in "\n".join(report["m12_completion_gate"]) + assert report["required_next_step"] == "Run on target 8xMI300X node before awarding M12 points." + + written = write_m12_preflight_report(tmp_path) + persisted = json.loads((tmp_path / "m12_preflight_report.json").read_text(encoding="utf-8")) + assert persisted["report_id"] == written["report_id"] + assert persisted["claim_boundary"] == "target_preflight_only_not_m12_validation" + assert validate_m12_preflight_report(persisted) == [] + + +def test_m12_preflight_env_snapshot_redacts_path_like_values(monkeypatch) -> None: + monkeypatch.setenv("CONDA_DEFAULT_ENV", "/tmp/private-conda-env") + monkeypatch.setenv("RAY_ADDRESS", "ray://127.0.0.1:10001") + monkeypatch.setenv("OPENAI_API_KEY", "must-not-appear") + + snapshot = _safe_environment_snapshot() + + assert snapshot["values"]["CONDA_DEFAULT_ENV"] == ENV_VALUE_REDACTED_ABSOLUTE_PATH + assert snapshot["redactions"]["CONDA_DEFAULT_ENV"] == "absolute_or_home_path" + assert snapshot["values"]["RAY_ADDRESS"] == "ray://127.0.0.1:10001" + assert "OPENAI_API_KEY" not in snapshot["values"] + assert "OPENAI_API_KEY" not in snapshot["redactions"] + + +def test_m12_preflight_report_validator_rejects_invalid_status_and_blockers() -> None: + report = run_m12_preflight() + report["status"] = "preflight_passed" + report["blockers"] = ["torch_device_count_below_8"] + + assert "preflight_passed requires blockers=[]" in validate_m12_preflight_report(report) + + report["status"] = "blocked" + report["blockers"] = [] + assert "blocked preflight requires at least one blocker" in validate_m12_preflight_report(report) + + +def test_m12_preflight_report_validator_rejects_tampered_fingerprint() -> None: + report = run_m12_preflight() + report["runtime_fingerprint"]["unexpected_section"] = {"not": "allowed"} + + assert ( + "runtime_fingerprint must be exact-schema, self-hash-verified, path-redacted, and allowlisted" + in validate_m12_preflight_report(report) + ) + + +def test_m12_preflight_report_validator_rejects_fingerprint_top_level_drift() -> None: + report = run_m12_preflight() + report["container_runtimes"]["docker"] = not report["container_runtimes"]["docker"] + + assert "runtime_fingerprint must match top-level preflight evidence" in validate_m12_preflight_report(report) + + +def test_m12_preflight_report_validator_rejects_contradictory_passed_report() -> None: + report = run_m12_preflight() + report["status"] = "preflight_passed" + report["blockers"] = [] + report["gpu"]["rocminfo_available"] = False + report["gpu"]["rocm_smi_available"] = False + report["gpu"]["mi300x_product_evidence"] = False + report["gpu"]["torch_probe"]["parse_error"] = True + report["gpu"]["torch_probe"]["device_count"] = 0 + report["python_modules"]["torch"]["available"] = False + report["python_modules"]["ray"]["available"] = False + report["python_modules"]["verl"]["available"] = False + report["ray_cluster"]["module_available"] = False + report["inference_engine_feasibility"] = { + "decision": "blocked_no_vllm_or_sglang", + "vllm_available": False, + "sglang_available": False, + } + report["container_runtimes"] = {"docker": False, "gvisor_runsc": False, "firecracker": False} + report["filesystem_cas_smoke"]["status"] = "failed" + + errors = validate_m12_preflight_report(report) + + assert "preflight_passed requires ROCm tooling availability" in errors + assert "preflight_passed requires torch availability" in errors + assert "preflight_passed requires parseable torch probe" in errors + assert "preflight_passed requires torch device_count >= 8" in errors + assert "preflight_passed requires MI300X product evidence" in errors + assert "preflight_passed requires Ray availability" in errors + assert "preflight_passed requires VeRL availability" in errors + assert "preflight_passed requires vLLM or SGLang availability" in errors + assert "preflight_passed requires at least one container runtime" in errors + assert "preflight_passed requires filesystem/CAS smoke passed" in errors + + +def test_m12_preflight_blocks_when_target_stack_is_absent_here() -> None: + report = run_m12_preflight() + + if report["status"] == "blocked": + assert report["blockers"] + else: + assert not report["blockers"] + assert report["gpu"]["torch_probe"]["device_count"] >= 8 + assert report["gpu"]["mi300x_product_evidence"] is True + + +def test_m12_preflight_require_pass_exits_nonzero_when_blocked(tmp_path) -> None: + output_dir = tmp_path / "preflight" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/run_m12_preflight.py", + "--output-dir", + str(output_dir), + "--require-pass", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + report = json.loads((output_dir / "m12_preflight_report.json").read_text(encoding="utf-8")) + + if report["status"] == "preflight_passed": + assert result.returncode == 0 + else: + assert result.returncode == 3 + assert "status=blocked" in result.stdout diff --git a/tests/rl/m12/test_m12_promotion_audit.py b/tests/rl/m12/test_m12_promotion_audit.py new file mode 100644 index 00000000..fe5a8c69 --- /dev/null +++ b/tests/rl/m12/test_m12_promotion_audit.py @@ -0,0 +1,712 @@ +from __future__ import annotations + +import json +import hashlib +import shlex +import subprocess +import sys +from pathlib import Path + +from breadboard.rl.m12 import ( + build_m12_final_report, + build_m12_promotion_audit, + record_command_log_result, + validate_m12_promotion_audit, + write_m12_promotion_audit, +) +from breadboard.rl.m12.final_report import ( + OPTIONAL_COMMAND_LOG_COMMANDS, + REQUIRED_COMMAND_LOG_COMMANDS, + REQUIRED_COMMAND_LOG_IDS, + TARGET_ARTIFACT_PATHS, + TARGET_COMMAND_LOG_MANIFEST_PATH, + TARGET_FINAL_REPORT_PATH, + TARGET_PROMOTION_AUDIT_PATH, +) + + +REPO_ROOT = Path(__file__).resolve().parents[3] +WORKSPACE_ROOT = REPO_ROOT.parent +PHASE_DIR = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" +PHASE_RUNS = PHASE_DIR / "runs" +SCORECARD = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" +CLAIM_LEDGER = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" + + +def _target_path(tmp_path: Path, target_path: str) -> Path: + path = tmp_path / target_path.removeprefix("../") + path.parent.mkdir(parents=True, exist_ok=True) + return path + + +def _target_final_report_path(tmp_path: Path) -> Path: + return _target_path(tmp_path, TARGET_FINAL_REPORT_PATH) + + +def _target_command_manifest_path(tmp_path: Path) -> Path: + return _target_path(tmp_path, TARGET_COMMAND_LOG_MANIFEST_PATH) + + +def _target_promotion_audit_path(tmp_path: Path) -> Path: + return _target_path(tmp_path, TARGET_PROMOTION_AUDIT_PATH) + + +def _fingerprint_sha256(fingerprint: dict) -> str: + payload = dict(fingerprint) + payload.pop("sha256", None) + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def _write_wrapper_style_log( + log_path: Path, + *, + command_id: str, + command: str, + exit_code: int = 0, + started_at: str = "2026-06-17T00:00:00Z", + completed_at: str = "2026-06-17T00:00:01Z", + target_run_id: str = "m12-target-run-test", +) -> None: + log_path.write_text( + "\n".join( + [ + f"# command_id: {command_id}", + f"# target_run_id: {target_run_id}", + f"# command: {command}", + f"# argv_json: {json.dumps(shlex.split(command), ensure_ascii=True)}", + f"# started_at: {started_at}", + f"{command_id} passed", + f"# completed_at: {completed_at}", + f"# exit_code: {exit_code}", + "", + ] + ), + encoding="utf-8", + ) + + +def _local_report() -> dict: + return build_m12_final_report( + preflight_report_path=PHASE_RUNS / "m12_target_preflight" / "m12_preflight_report.json", + swe_run_summary_path=PHASE_RUNS / "m6_controlled_swe_toy" / "run_summary.json", + verl_smoke_report_path=PHASE_RUNS / "m7_verl_probe" / "smoke_consumer_report.json", + ray_probe_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "ray_probe_report.json", + warm_vs_cold_report_path=PHASE_RUNS / "m8_ray_warm_pool_probe" / "warm_vs_cold_report.json", + ) + + +def _write_complete_command_manifest(tmp_path: Path) -> Path: + manifest_path = _target_command_manifest_path(tmp_path) + log_dir = manifest_path.parent / "logs" + log_dir.mkdir() + for command_id in [*REQUIRED_COMMAND_LOG_IDS, "final_report"]: + log_path = log_dir / f"{command_id}.log" + command = REQUIRED_COMMAND_LOG_COMMANDS.get( + command_id, + OPTIONAL_COMMAND_LOG_COMMANDS.get(command_id, f"echo {command_id}"), + ) + _write_wrapper_style_log(log_path, command_id=command_id, command=command) + record_command_log_result( + manifest_path=manifest_path, + command_id=command_id, + command=command, + log_path=log_path, + exit_code=0, + started_at="2026-06-17T00:00:00Z", + completed_at="2026-06-17T00:00:01Z", + target_run_id="m12-target-run-test", + ) + return manifest_path + + +def _eligible_report(command_manifest_path: Path) -> dict: + target_run_id = "m12-target-run-test" + report = _local_report() + manifest = json.loads(command_manifest_path.read_text(encoding="utf-8")) + command_entries = [item for item in manifest["commands"] if item["command_id"] in REQUIRED_COMMAND_LOG_IDS] + report["m12_score_eligible"] = True + report["missing_gates"] = [] + report["missing_gate_remediations"] = [] + report["artifact_paths"] = dict(TARGET_ARTIFACT_PATHS) + report["artifact_path_policy"]["paths_match_target_defaults"] = True + report["archive_verify"] = { + "present": True, + "path": TARGET_ARTIFACT_PATHS["archive_verify_report"], + "read_error": None, + "report_id": "bb_zyphra_rl_phase1_m12_archive_verify_report_v1", + "claim_boundary": "transfer_archive_verification_not_m12_validation", + "scorecard_update_allowed": False, + "m12_points_awarded": False, + "status": "passed", + "archive_manifest_id": "bb_zyphra_rl_phase1_m12_transfer_archive_manifest_v1", + "archive_claim_boundary": "transfer_archive_only_not_m12_validation", + "archive_sha256": "sha256:" + ("1" * 64), + "included_entry_count": 213, + "all_required_artifacts_present": True, + "all_transfer_requirements_covered": True, + "archive_contains_source_overlay": True, + "archive_deterministic": True, + "source_paths_portable": True, + "errors": [], + "validation_errors": [], + } + report["preflight"]["path"] = TARGET_ARTIFACT_PATHS["preflight_report"] + report["swe_probe"]["path"] = TARGET_ARTIFACT_PATHS["swe_run_summary"] + report["verl_export"]["path"] = TARGET_ARTIFACT_PATHS["verl_smoke_report"] + report["ray_probe"]["path"] = TARGET_ARTIFACT_PATHS["ray_probe_report"] + report["warm_vs_cold"]["path"] = TARGET_ARTIFACT_PATHS["warm_vs_cold_report"] + report["preflight"]["status"] = "preflight_passed" + report["preflight"]["target_run_id"] = target_run_id + report["preflight"]["blockers"] = [] + report["preflight"]["gpu"]["rocminfo_available"] = True + report["preflight"]["gpu"]["mi300x_product_evidence"] = True + report["preflight"]["gpu"]["torch_probe"]["device_count"] = 8 + report["preflight"]["gpu"]["torch_probe"]["device_names"] = ["AMD Instinct MI300X"] * 8 + report["preflight"]["python_modules"]["verl"]["available"] = True + report["preflight"]["python_modules"]["ray"]["available"] = True + report["preflight"]["filesystem_cas_smoke"]["status"] = "passed" + fingerprint = report["preflight"]["runtime_fingerprint"] + fingerprint["tool_presence"] = { + "rocm_smi": report["preflight"]["gpu"]["rocm_smi_available"], + "rocminfo": report["preflight"]["gpu"]["rocminfo_available"], + "docker": report["preflight"]["container_runtimes"]["docker"], + "gvisor_runsc": report["preflight"]["container_runtimes"]["gvisor_runsc"], + "firecracker": report["preflight"]["container_runtimes"]["firecracker"], + } + fingerprint["python_modules"] = report["preflight"]["python_modules"] + fingerprint["gpu_summary"] = { + "rocm_smi_output": report["preflight"]["gpu"]["rocm_smi_output"], + "torch_probe": report["preflight"]["gpu"]["torch_probe"], + } + fingerprint["ray_summary"] = {"status_output": report["preflight"]["ray_cluster"]["status_output"]} + fingerprint["sha256"] = _fingerprint_sha256(fingerprint) + report["ray_probe"]["ray_local_mode"] = False + report["swe_probe"]["target_run_id"] = target_run_id + report["verl_export"]["target_run_id"] = target_run_id + report["ray_probe"]["target_run_id"] = target_run_id + report["warm_vs_cold"]["target_run_id"] = target_run_id + report["load_ladder"] = { + "present": True, + "path": TARGET_ARTIFACT_PATHS["load_ladder_report"], + "target_run_id": target_run_id, + "validation_errors": [], + "required_levels": [5, 20, 50], + "optional_levels": [100], + "concurrency_levels": [ + {"target_sessions": 5, "status": "passed", "ray_local_mode": False}, + {"target_sessions": 20, "status": "passed", "ray_local_mode": False}, + {"target_sessions": 50, "status": "passed", "ray_local_mode": False}, + ], + "resource_skips": [{"target_sessions": 100, "reason": "resource constrained target window"}], + "policy_version_integrity": True, + "queue_backpressure_integrity": True, + } + report["soak"] = { + "present": True, + "path": TARGET_ARTIFACT_PATHS["soak_report"], + "target_run_id": target_run_id, + "validation_errors": [], + "minimum_duration_seconds": 7200, + "duration_seconds": 7200, + "status": "passed", + "runtime_failure_count": 0, + "ray_local_mode": False, + } + report["command_logs"] = { + "present": True, + "path": TARGET_ARTIFACT_PATHS["command_log_manifest"], + "manifest_id": manifest["manifest_id"], + "manifest_validation_errors": [], + "required_command_ids": list(REQUIRED_COMMAND_LOG_IDS), + "manifest_required_command_ids": list(REQUIRED_COMMAND_LOG_IDS), + "archived_command_ids": list(REQUIRED_COMMAND_LOG_IDS), + "hash_verified_command_ids": list(REQUIRED_COMMAND_LOG_IDS), + "missing_command_log_ids": [], + "target_run_ids": [target_run_id], + "single_target_run_id": target_run_id, + "command_text_mismatches": [], + "all_required_logs_archived": True, + "all_required_commands_passed": True, + "command_count": len(command_entries), + "commands": command_entries, + } + report["target_run_identity"] = { + "single_command_log_target_run_id": target_run_id, + "artifact_target_run_ids": { + "preflight_report": target_run_id, + "swe_run_summary": target_run_id, + "verl_smoke_report": target_run_id, + "ray_probe_report": target_run_id, + "warm_vs_cold_report": target_run_id, + "load_ladder_report": target_run_id, + "soak_report": target_run_id, + }, + "missing_artifact_target_run_ids": [], + "mismatched_artifact_target_run_ids": {}, + "run_ids_match_command_logs": True, + } + return report + + +def test_m12_promotion_audit_rejects_local_noneligible_report(tmp_path) -> None: + final_report_path = tmp_path / "m12_final_report.json" + final_report_path.write_text(json.dumps(_local_report(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=tmp_path / "missing_command_log_manifest.json", + ) + + assert audit["audit_id"] == "bb_zyphra_rl_phase1_m12_promotion_audit_v1" + assert audit["claim_boundary"] == "promotion_review_only_not_scorecard_update" + assert audit["scorecard_update_allowed"] is False + assert audit["promotion_review_ready"] is False + assert "final_report.score_eligible" in audit["missing_requirements"] + assert "command_log_manifest.path_exists" in audit["missing_requirements"] + assert "scorecard.m12_state_valid_for_review" not in audit["missing_requirements"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_stale_missing_requirements(tmp_path) -> None: + final_report_path = tmp_path / "m12_final_report.json" + final_report_path.write_text(json.dumps(_local_report(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=tmp_path / "missing_command_log_manifest.json", + ) + + audit["missing_requirements"] = [] + audit["promotion_review_ready"] = True + audit["checks"]["final_report"]["score_eligible"] = "yes" + + errors = validate_m12_promotion_audit(audit) + + assert "checks.final_report.score_eligible must be boolean" in errors + assert "missing_requirements must match promotion-audit embedded checks" in errors + assert "promotion_review_ready must match promotion-audit embedded checks" in errors + + +def test_m12_promotion_audit_rejects_malformed_command_log_manifest_without_crashing(tmp_path) -> None: + final_report_path = tmp_path / "m12_final_report.json" + command_manifest_path = tmp_path / "command_log_manifest.json" + final_report_path.write_text(json.dumps(_local_report(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + command_manifest_path.write_text("{not valid command log json", encoding="utf-8") + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["command_log_manifest_read_error"] + assert "JSONDecodeError" in audit["command_log_manifest_read_error"] + assert audit["command_log_manifest_errors"] == [ + f"command log manifest unreadable: {audit['command_log_manifest_read_error']}" + ] + assert "command_log_manifest.required_manifest_valid" in audit["missing_requirements"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_malformed_final_report_without_crashing(tmp_path) -> None: + final_report_path = tmp_path / "m12_final_report.json" + command_manifest_path = tmp_path / "command_log_manifest.json" + final_report_path.write_text("{not valid final report json", encoding="utf-8") + command_manifest_path.write_text(json.dumps({"manifest_id": "bad"}), encoding="utf-8") + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["final_report_read_error"] + assert "JSONDecodeError" in audit["final_report_read_error"] + assert audit["final_report_validation_errors"] == [ + f"final report unreadable: {audit['final_report_read_error']}" + ] + assert "final_report.schema_valid" in audit["missing_requirements"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_malformed_scorecard_without_crashing(tmp_path) -> None: + final_report_path = tmp_path / "m12_final_report.json" + scorecard_path = tmp_path / "scorecard.yaml" + command_manifest_path = tmp_path / "command_log_manifest.json" + final_report_path.write_text(json.dumps(_local_report(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + scorecard_path.write_text("current_verified_points: [unterminated\n", encoding="utf-8") + command_manifest_path.write_text(json.dumps({"manifest_id": "bad"}), encoding="utf-8") + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=scorecard_path, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["scorecard_read_error"] + assert "ParserError" in audit["scorecard_read_error"] + assert audit["checks"]["scorecard"]["readable"] is False + assert "scorecard.readable" in audit["missing_requirements"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_missing_claim_ledger_without_crashing(tmp_path) -> None: + final_report_path = tmp_path / "m12_final_report.json" + command_manifest_path = tmp_path / "command_log_manifest.json" + missing_claim_ledger_path = tmp_path / "missing_claim_ledger.md" + final_report_path.write_text(json.dumps(_local_report(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + command_manifest_path.write_text(json.dumps({"manifest_id": "bad"}), encoding="utf-8") + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=missing_claim_ledger_path, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["claim_ledger_read_error"] == "FileNotFoundError: text artifact is missing" + assert audit["checks"]["claim_ledger"]["readable"] is False + assert "claim_ledger.path_exists" in audit["missing_requirements"] + assert "claim_ledger.readable" in audit["missing_requirements"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_accepts_synthetic_complete_target_evidence(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + final_report_path = _target_final_report_path(tmp_path) + final_report_path.write_text( + json.dumps(_eligible_report(command_manifest_path), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + audit = write_m12_promotion_audit( + output_path=_target_promotion_audit_path(tmp_path), + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is True + assert audit["missing_requirements"] == [] + assert audit["scorecard_update_allowed"] is False + assert audit["checks"]["command_log_manifest"]["required_command_text_matches"] is True + assert audit["checks"]["command_log_manifest"]["final_report_required_rows_match_manifest"] is True + assert audit["checks"]["command_log_manifest"]["final_report_command_text_matches"] is True + assert audit["checks"]["command_log_manifest"]["final_report_command_hash_verified"] is True + assert audit["checks"]["promotion_audit_output"]["path_matches_target_default"] is True + assert audit["scorecard_state"]["m12_verified_points"] == 80 + assert audit["scorecard_state"]["current_verified_points"] == 1000 + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_arbitrary_output_path_for_complete_evidence(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + final_report_path = _target_final_report_path(tmp_path) + final_report_path.write_text( + json.dumps(_eligible_report(command_manifest_path), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + audit = write_m12_promotion_audit( + output_path=tmp_path / "m12_promotion_audit.json", + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["checks"]["promotion_audit_output"]["path_matches_target_default"] is False + assert "promotion_audit_output.path_matches_target_default" in audit["missing_requirements"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_eligible_report_from_arbitrary_local_path(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + final_report_path = tmp_path / "m12_final_report.json" + final_report_path.write_text( + json.dumps(_eligible_report(command_manifest_path), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["checks"]["final_report"]["path_matches_target_default"] is False + assert "final_report.path_matches_target_default" in audit["missing_requirements"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_copied_command_manifest_path(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + copied_manifest_path = tmp_path / "command_log_manifest.json" + copied_manifest_path.write_text(command_manifest_path.read_text(encoding="utf-8"), encoding="utf-8") + final_report_path = _target_final_report_path(tmp_path) + final_report_path.write_text( + json.dumps(_eligible_report(command_manifest_path), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=copied_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["checks"]["command_log_manifest"]["path_matches_target_default"] is False + assert "command_log_manifest.path_matches_target_default" in audit["missing_requirements"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_copied_scorecard_and_claim_ledger_paths(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + final_report_path = _target_final_report_path(tmp_path) + final_report_path.write_text( + json.dumps(_eligible_report(command_manifest_path), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + copied_scorecard_path = tmp_path / "scorecard.yaml" + copied_claim_ledger_path = tmp_path / "claim_ledger.md" + copied_scorecard_path.write_text(SCORECARD.read_text(encoding="utf-8"), encoding="utf-8") + copied_claim_ledger_path.write_text(CLAIM_LEDGER.read_text(encoding="utf-8"), encoding="utf-8") + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=copied_scorecard_path, + claim_ledger_path=copied_claim_ledger_path, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["checks"]["scorecard"]["path_matches_target_default"] is False + assert audit["checks"]["claim_ledger"]["path_matches_target_default"] is False + assert "scorecard.path_matches_target_default" in audit["missing_requirements"] + assert "claim_ledger.path_matches_target_default" in audit["missing_requirements"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_passed_final_report_command_with_nonzero_exit_code(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + manifest = json.loads(command_manifest_path.read_text(encoding="utf-8")) + final_report_entry = next(item for item in manifest["commands"] if item["command_id"] == "final_report") + final_report_log_path = command_manifest_path.parent / final_report_entry["log_path"] + _write_wrapper_style_log( + final_report_log_path, + command_id="final_report", + command=final_report_entry["command"], + exit_code=4, + ) + final_report_log_sha = "sha256:" + hashlib.sha256(final_report_log_path.read_bytes()).hexdigest() + final_report_entry["status"] = "passed" + final_report_entry["exit_code"] = 4 + final_report_entry["sha256"] = final_report_log_sha + final_report_entry["attempts"][0]["status"] = "passed" + final_report_entry["attempts"][0]["exit_code"] = 4 + final_report_entry["attempts"][0]["sha256"] = final_report_log_sha + command_manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + final_report_path = _target_final_report_path(tmp_path) + final_report_path.write_text( + json.dumps(_eligible_report(command_manifest_path), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert "command_log_manifest.required_manifest_valid" in audit["missing_requirements"] + assert any( + "invalid status/exit_code for final_report: passed commands must have exit_code 0" in error + for error in audit["command_log_manifest_errors"] + ) + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_command_log_boundary_drift(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + manifest = json.loads(command_manifest_path.read_text(encoding="utf-8")) + manifest["claim_boundary"] = "scorecard_update_allowed" + manifest["scorecard_update_allowed"] = True + manifest["m12_points_awarded"] = True + command_manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + final_report_path = _target_final_report_path(tmp_path) + final_report_path.write_text( + json.dumps(_eligible_report(command_manifest_path), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert "command_log_manifest.required_manifest_valid" in audit["missing_requirements"] + assert "claim_boundary must remain target_command_logs_not_scorecard_update" in audit["command_log_manifest_errors"] + assert "scorecard_update_allowed must be false" in audit["command_log_manifest_errors"] + assert "m12_points_awarded must be false" in audit["command_log_manifest_errors"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rechecks_required_command_text_from_raw_manifest(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + manifest = json.loads(command_manifest_path.read_text(encoding="utf-8")) + preflight_entry = next(item for item in manifest["commands"] if item["command_id"] == "target_preflight") + preflight_entry["command"] = "python -c 'print(\"not target preflight\")'" + command_manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + final_report_path = _target_final_report_path(tmp_path) + final_report = _eligible_report(command_manifest_path) + final_report["command_logs"]["command_text_mismatches"] = [] + final_report_path.write_text(json.dumps(final_report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["checks"]["command_log_manifest"]["required_command_text_matches"] is False + assert "command_log_manifest.required_command_text_matches" in audit["missing_requirements"] + assert audit["required_command_text_mismatches"] == [ + { + "command_id": "target_preflight", + "expected": REQUIRED_COMMAND_LOG_COMMANDS["target_preflight"], + "observed": "python -c 'print(\"not target preflight\")'", + } + ] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_final_report_without_canonical_required_ids(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + final_report = _eligible_report(command_manifest_path) + final_report["command_logs"]["manifest_required_command_ids"] = ["target_preflight"] + final_report_path = _target_final_report_path(tmp_path) + final_report_path.write_text(json.dumps(final_report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert "final_report.schema_valid" in audit["missing_requirements"] + assert "eligible report requires canonical command log required_command_ids" in audit["final_report_validation_errors"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_final_report_required_row_drift(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + final_report = _eligible_report(command_manifest_path) + target_entry = next( + item for item in final_report["command_logs"]["commands"] if item["command_id"] == "target_preflight" + ) + target_entry["sha256"] = "sha256:" + ("0" * 64) + final_report_path = _target_final_report_path(tmp_path) + final_report_path.write_text(json.dumps(final_report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["checks"]["command_log_manifest"]["final_report_required_rows_match_manifest"] is False + assert "command_log_manifest.final_report_required_rows_match_manifest" in audit["missing_requirements"] + assert audit["final_report_required_command_row_mismatches"] == [ + {"command_id": "target_preflight", "reason": "row_differs_from_manifest"} + ] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_rejects_wrong_final_report_command_text(tmp_path) -> None: + command_manifest_path = _write_complete_command_manifest(tmp_path) + manifest = json.loads(command_manifest_path.read_text(encoding="utf-8")) + final_report_entry = next(item for item in manifest["commands"] if item["command_id"] == "final_report") + final_report_entry["command"] = "python -c 'print(\"not the final report builder\")'" + command_manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + final_report_path = _target_final_report_path(tmp_path) + final_report_path.write_text( + json.dumps(_eligible_report(command_manifest_path), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + audit = build_m12_promotion_audit( + final_report_path=final_report_path, + scorecard_path=SCORECARD, + claim_ledger_path=CLAIM_LEDGER, + command_log_manifest_path=command_manifest_path, + ) + + assert audit["promotion_review_ready"] is False + assert audit["checks"]["command_log_manifest"]["final_report_command_text_matches"] is False + assert "command_log_manifest.final_report_command_text_matches" in audit["missing_requirements"] + assert validate_m12_promotion_audit(audit) == [] + + +def test_m12_promotion_audit_cli_require_ready_fails_closed_for_local_report(tmp_path) -> None: + final_report_path = tmp_path / "m12_final_report.json" + output_path = tmp_path / "m12_promotion_audit.json" + final_report_path.write_text(json.dumps(_local_report(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/audit_m12_score_promotion.py", + "--output", + str(output_path), + "--final-report", + str(final_report_path), + "--scorecard", + str(SCORECARD), + "--claim-ledger", + str(CLAIM_LEDGER), + "--command-log-manifest", + str(tmp_path / "missing_command_log_manifest.json"), + "--require-ready", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 4 + assert output_path.exists() + assert "promotion_review_ready=False" in result.stdout diff --git a/tests/rl/m12/test_m12_score_boundary.py b/tests/rl/m12/test_m12_score_boundary.py new file mode 100644 index 00000000..2c6ce655 --- /dev/null +++ b/tests/rl/m12/test_m12_score_boundary.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PHASE_DIR = REPO_ROOT.parent / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" +SCORECARD = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" +CLAIM_LEDGER = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" +M12_REPORT = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_M12_VALIDATION_REPORT.md" + + +def test_m12_points_are_awarded_after_target_evidence_review() -> None: + scorecard = yaml.safe_load(SCORECARD.read_text(encoding="utf-8")) + m12 = next(milestone for milestone in scorecard["milestones"] if milestone["id"] == "M12") + + assert scorecard["current_verified_points"] == 1000 + assert m12["verified_points"] == 80 + assert m12["status"] == "completed" + assert scorecard["current_verified_points"] == sum( + milestone["verified_points"] for milestone in scorecard["milestones"] + ) + + +def test_claim_ledger_forbids_overbroad_production_claims() -> None: + text = CLAIM_LEDGER.read_text(encoding="utf-8") + + assert "BreadBoard passed 8xMI300X final validation" in text + assert "Target-node final report is `m12_score_eligible=true`" in text + assert "M12 scorecard edit is separately reviewed" in text + assert "BreadBoard supports production RL rollouts." in text + assert "No production, external benchmark, trainer, or scale support claim has been earned yet." in text + assert "transfer/preflight/final-report preparation only" in text + + +def test_m12_validation_report_records_target_promotion_boundary() -> None: + text = M12_REPORT.read_text(encoding="utf-8") + + assert "Status: passed, target validation executed on 8xMI300X" in text + assert "Score impact: 80 / 80 M12 points awarded" in text + assert "Program score after this report: 1000 / 1000" in text + assert "M12 is complete." in text + assert "separate reviewed scorecard/claim-ledger update" in text + assert "BreadBoard passed M12 8xMI300X final validation" in text + assert "archive_sha256_recorded_in=m12_transfer_archive_manifest.json" in text + assert "m12_score_eligible=true" in text + assert "promotion_review_ready=true" in text + assert "scorecard_update_allowed=false" in text + assert "Human Review" in text diff --git a/tests/rl/m12/test_m12_transfer_pack.py b/tests/rl/m12/test_m12_transfer_pack.py new file mode 100644 index 00000000..58d29e55 --- /dev/null +++ b/tests/rl/m12/test_m12_transfer_pack.py @@ -0,0 +1,1991 @@ +from __future__ import annotations + +import gzip +import hashlib +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tarfile + +import pytest + +from breadboard.rl.m12 import ( + apply_m12_transfer_overlay, + build_m12_readiness_summary, + build_m12_test_commands_script, + build_m12_transfer_manifest, + build_m12_transfer_summary, + validate_m12_readiness_summary, + validate_m12_test_commands_script, + validate_m12_transfer_archive_manifest, + validate_m12_transfer_overlay_report, + validate_m12_transfer_summary, + write_m12_transfer_archive, + write_m12_transfer_pack, +) +from breadboard.rl.m12.transfer import ( + COMMAND_LOG_MANIFEST_TEMPLATE, + EXPECTED_OUTPUTS, + LOAD_LADDER_REPORT_TEMPLATE, + M12_TEST_COMMAND_ROWS, + M12_TEST_COMMANDS, + REQUIRED_TRANSFER_ARTIFACTS, + SOAK_REPORT_TEMPLATE, + TRANSFER_PREP_FILES, +) + + +REPO_ROOT = Path(__file__).resolve().parents[3] +REPO_ARCHIVE_ROOT = Path("workspace") / REPO_ROOT.name + + +def _overlay_repo_root(workspace_root: Path) -> Path: + return workspace_root / REPO_ROOT.name + +FINAL_REPORT_TARGET_ARGUMENTS = ( + "--output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json", + "--archive-verify-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_archive_verify/m12_archive_verify_report.json", + "--preflight-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_target_preflight/m12_preflight_report.json", + "--swe-run-summary ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_swe_probe/run_summary.json", + "--verl-smoke-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_verl_probe/smoke_consumer_report.json", + "--ray-probe-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe/ray_probe_report.json", + "--warm-vs-cold-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_ray_probe/warm_vs_cold_report.json", + "--load-ladder-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_load_ladder/load_ladder_report.json", + "--soak-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_soak/soak_report.json", +) +FINAL_REPORT_MANIFEST_ARGUMENTS = ( + *FINAL_REPORT_TARGET_ARGUMENTS, + "--command-log-manifest ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json", +) +FINAL_REPORT_SCRIPT_ARGUMENTS = ( + *FINAL_REPORT_TARGET_ARGUMENTS, + '--command-log-manifest "$COMMAND_LOG_MANIFEST"', +) + + +def _write_archive_sidecar_and_manifest( + *, + manifest_path: Path, + manifest: dict, + mutated_archive_path: Path, +) -> Path: + archive_sha = "sha256:" + hashlib.sha256(mutated_archive_path.read_bytes()).hexdigest() + manifest["archive_name"] = mutated_archive_path.name + manifest["archive_path"] = mutated_archive_path.name + manifest["archive_sha256"] = archive_sha + manifest["archive_size_bytes"] = mutated_archive_path.stat().st_size + manifest["archive_sha256_file"] = mutated_archive_path.with_suffix(mutated_archive_path.suffix + ".sha256").name + mutated_archive_path.with_suffix(mutated_archive_path.suffix + ".sha256").write_text( + f"{archive_sha} {mutated_archive_path.name}\n", + encoding="utf-8", + ) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return manifest_path + + +def _write_archive_with_test_command_mutation(tmp_path: Path, mutate) -> Path: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + original_archive_path = tmp_path / "m12_transfer_evidence_pack.tar.gz" + mutated_archive_path = tmp_path / "m12_transfer_evidence_pack_mutated.tar.gz" + + with tarfile.open(original_archive_path, "r:gz") as source_archive: + members = [member for member in source_archive.getmembers() if member.isfile()] + payloads = {} + for member in members: + extracted = source_archive.extractfile(member) + assert extracted is not None + payload = extracted.read() + if member.name.endswith("/m12_test_commands.sh"): + payload = mutate(payload) + for entry in manifest["included_entries"]: + if entry["archive_path"] == member.name: + entry["size_bytes"] = len(payload) + entry["sha256"] = "sha256:" + hashlib.sha256(payload).hexdigest() + break + payloads[member.name] = payload + with mutated_archive_path.open("wb") as raw_archive: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw_archive, mtime=0) as gzip_archive: + with tarfile.open(fileobj=gzip_archive, mode="w", format=tarfile.PAX_FORMAT) as mutated_archive: + for member in members: + payload = payloads[member.name] + info = tarfile.TarInfo(member.name) + info.size = len(payload) + info.mode = member.mode + info.mtime = 0 + info.uid = 0 + info.gid = 0 + info.uname = "" + info.gname = "" + mutated_archive.addfile(info, io.BytesIO(payload)) + + return _write_archive_sidecar_and_manifest( + manifest_path=manifest_path, + manifest=manifest, + mutated_archive_path=mutated_archive_path, + ) + + +def _write_archive_with_test_command_target_run_removed(tmp_path: Path) -> Path: + return _write_archive_with_test_command_mutation( + tmp_path, + lambda payload: payload.replace(b'--target-run-id "$M12_TARGET_RUN_ID" ', b"", 1), + ) + + +def _write_archive_with_test_command_closeout_guard_removed(tmp_path: Path) -> Path: + return _write_archive_with_test_command_mutation( + tmp_path, + lambda payload: payload.replace( + b"Existing M12 close-out artifact would make target evidence ambiguous", + b"Existing M12 close-out artifact guard removed", + 1, + ), + ) + + +def _write_archive_with_json_member_mutation(tmp_path: Path, *, suffix: str, mutate) -> Path: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + original_archive_path = tmp_path / "m12_transfer_evidence_pack.tar.gz" + mutated_archive_path = tmp_path / "m12_transfer_evidence_pack_json_mutated.tar.gz" + + with tarfile.open(original_archive_path, "r:gz") as source_archive: + members = [member for member in source_archive.getmembers() if member.isfile()] + payloads = {} + found = False + for member in members: + extracted = source_archive.extractfile(member) + assert extracted is not None + payload = extracted.read() + if member.name.endswith(suffix): + document = json.loads(payload.decode("utf-8")) + mutate(document) + payload = json.dumps(document, indent=2, sort_keys=True).encode("utf-8") + b"\n" + found = True + for entry in manifest["included_entries"]: + if entry["archive_path"] == member.name: + entry["size_bytes"] = len(payload) + entry["sha256"] = "sha256:" + hashlib.sha256(payload).hexdigest() + break + payloads[member.name] = payload + assert found, suffix + with mutated_archive_path.open("wb") as raw_archive: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw_archive, mtime=0) as gzip_archive: + with tarfile.open(fileobj=gzip_archive, mode="w", format=tarfile.PAX_FORMAT) as mutated_archive: + for member in members: + payload = payloads[member.name] + info = tarfile.TarInfo(member.name) + info.size = len(payload) + info.mode = member.mode + info.mtime = 0 + info.uid = 0 + info.gid = 0 + info.uname = "" + info.gname = "" + mutated_archive.addfile(info, io.BytesIO(payload)) + + return _write_archive_sidecar_and_manifest( + manifest_path=manifest_path, + manifest=manifest, + mutated_archive_path=mutated_archive_path, + ) + + +def _write_archive_with_nonzero_gzip_mtime(tmp_path: Path) -> Path: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + original_archive_path = tmp_path / "m12_transfer_evidence_pack.tar.gz" + mutated_archive_path = tmp_path / "m12_transfer_evidence_pack_nonzero_mtime.tar.gz" + payload = bytearray(original_archive_path.read_bytes()) + payload[4:8] = (1).to_bytes(4, "little") + mutated_archive_path.write_bytes(bytes(payload)) + return _write_archive_sidecar_and_manifest( + manifest_path=manifest_path, + manifest=manifest, + mutated_archive_path=mutated_archive_path, + ) + + +def _write_archive_with_reversed_member_order(tmp_path: Path) -> Path: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + original_archive_path = tmp_path / "m12_transfer_evidence_pack.tar.gz" + mutated_archive_path = tmp_path / "m12_transfer_evidence_pack_reversed.tar.gz" + + with tarfile.open(original_archive_path, "r:gz") as source_archive: + members = [member for member in source_archive.getmembers() if member.isfile()] + payloads = {} + for member in members: + extracted = source_archive.extractfile(member) + assert extracted is not None + payloads[member.name] = extracted.read() + with mutated_archive_path.open("wb") as raw_archive: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw_archive, mtime=0) as gzip_archive: + with tarfile.open(fileobj=gzip_archive, mode="w", format=tarfile.PAX_FORMAT) as mutated_archive: + for member in reversed(members): + payload = payloads[member.name] + info = tarfile.TarInfo(member.name) + info.size = len(payload) + info.mode = member.mode + info.mtime = 0 + info.uid = 0 + info.gid = 0 + info.uname = "" + info.gname = "" + mutated_archive.addfile(info, io.BytesIO(payload)) + + return _write_archive_sidecar_and_manifest( + manifest_path=manifest_path, + manifest=manifest, + mutated_archive_path=mutated_archive_path, + ) + + +def _write_archive_with_duplicate_manifest_entry(tmp_path: Path) -> Path: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["included_entries"].append(dict(manifest["included_entries"][0])) + manifest["included_entry_count"] = len(manifest["included_entries"]) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return manifest_path + + +def _write_archive_with_absolute_source_path(tmp_path: Path) -> Path: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["included_entries"][0]["source_path"] = str(REPO_ROOT / "leaked_local_path.py") + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return manifest_path + + +def _write_archive_with_private_source_key(tmp_path: Path) -> Path: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["included_entries"][0]["_local_source_path"] = str(REPO_ROOT / "leaked_local_path.py") + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return manifest_path + + +def _write_archive_with_absolute_top_level_archive_path(tmp_path: Path) -> Path: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["archive_path"] = str((tmp_path / "m12_transfer_evidence_pack.tar.gz").resolve()) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return manifest_path + + +def _write_archive_with_duplicate_tar_member(tmp_path: Path) -> Path: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + original_archive_path = tmp_path / "m12_transfer_evidence_pack.tar.gz" + mutated_archive_path = tmp_path / "m12_transfer_evidence_pack_duplicate_member.tar.gz" + + with tarfile.open(original_archive_path, "r:gz") as source_archive: + members = [member for member in source_archive.getmembers() if member.isfile()] + payloads = {} + for member in members: + extracted = source_archive.extractfile(member) + assert extracted is not None + payloads[member.name] = extracted.read() + with mutated_archive_path.open("wb") as raw_archive: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw_archive, mtime=0) as gzip_archive: + with tarfile.open(fileobj=gzip_archive, mode="w", format=tarfile.PAX_FORMAT) as mutated_archive: + for member in members: + payload = payloads[member.name] + info = tarfile.TarInfo(member.name) + info.size = len(payload) + info.mode = member.mode + info.mtime = 0 + info.uid = 0 + info.gid = 0 + info.uname = "" + info.gname = "" + mutated_archive.addfile(info, io.BytesIO(payload)) + first = members[0] + payload = payloads[first.name] + duplicate_info = tarfile.TarInfo(first.name) + duplicate_info.size = len(payload) + duplicate_info.mode = first.mode + duplicate_info.mtime = 0 + duplicate_info.uid = 0 + duplicate_info.gid = 0 + duplicate_info.uname = "" + duplicate_info.gname = "" + mutated_archive.addfile(duplicate_info, io.BytesIO(payload)) + + return _write_archive_sidecar_and_manifest( + manifest_path=manifest_path, + manifest=manifest, + mutated_archive_path=mutated_archive_path, + ) + + +def _run_generated_overlay_script( + *, + script_path: Path, + manifest_path: Path, + report_path: Path, + workspace_root: Path, + cwd: Path, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(script_path), + "--manifest", + str(manifest_path), + "--workspace-root", + str(workspace_root), + "--output", + str(report_path), + ], + cwd=cwd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + +def test_m12_transfer_manifest_is_complete_and_non_scoring() -> None: + manifest = build_m12_transfer_manifest(REPO_ROOT) + + assert manifest["manifest_id"] == "bb_zyphra_rl_phase1_m12_transfer_manifest_v1" + assert manifest["claim_boundary"] == "transfer_preparation_only_not_m12_validation" + assert manifest["repo"]["root"] == REPO_ROOT.name + assert manifest["repo"]["root_path_portable"] is True + assert not Path(manifest["repo"]["root"]).is_absolute() + assert ".." not in Path(manifest["repo"]["root"]).parts + assert manifest["all_required_artifacts_present"] is True + assert manifest["all_transfer_requirements_covered"] is True + assert [artifact["path"] for artifact in manifest["artifacts"]] == REQUIRED_TRANSFER_ARTIFACTS + assert manifest["test_commands"] == M12_TEST_COMMANDS + assert len(manifest["test_commands"]) == len(M12_TEST_COMMAND_ROWS) + assert validate_m12_test_commands_script(build_m12_test_commands_script(), manifest) == [] + assert "requirements.txt" in REQUIRED_TRANSFER_ARTIFACTS + assert "breadboard/rl" in REQUIRED_TRANSFER_ARTIFACTS + assert "scripts/rl_phase1" in REQUIRED_TRANSFER_ARTIFACTS + assert "tests/rl" in REQUIRED_TRANSFER_ARTIFACTS + assert "tests/test_rl_phase1_scorecard_schema.py" in REQUIRED_TRANSFER_ARTIFACTS + assert "tests/test_rl_phase1_claim_ledger.py" in REQUIRED_TRANSFER_ARTIFACTS + assert "docs/rl_phase1" in REQUIRED_TRANSFER_ARTIFACTS + assert "examples/rl_env_packages" in REQUIRED_TRANSFER_ARTIFACTS + assert "../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_M12_VALIDATION_REPORT.md" in REQUIRED_TRANSFER_ARTIFACTS + assert "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/row_evidence" in REQUIRED_TRANSFER_ARTIFACTS + assert "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/smoke_consumer_report.json" in REQUIRED_TRANSFER_ARTIFACTS + assert "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m8_ray_warm_pool_probe/warm_vs_cold_report.json" in REQUIRED_TRANSFER_ARTIFACTS + assert "verify_m12_transfer_archive.py" in manifest["test_commands"][0] + preflight_command = next(command for command in manifest["test_commands"] if "run_m12_preflight.py" in command) + assert "--require-pass" in preflight_command + assert "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/verl_probe_rows.parquet" in REQUIRED_TRANSFER_ARTIFACTS + assert "../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m7_verl_probe/projection_manifest.json" in REQUIRED_TRANSFER_ARTIFACTS + assert "m12_node_verl_probe/projection_manifest.json" in manifest["expected_outputs"] + assert "m12_archive_verify/m12_archive_verify_report.json" in manifest["expected_outputs"] + assert "m12_node_load_ladder/load_ladder_report.json" in manifest["expected_outputs"] + assert "m12_node_soak/soak_report.json" in manifest["expected_outputs"] + assert "m12_command_logs/command_log_manifest.json" in manifest["expected_outputs"] + assert "m12_final_report/m12_final_report.json" in manifest["expected_outputs"] + assert "m12_promotion_audit/m12_promotion_audit.json" in manifest["expected_outputs"] + assert manifest["expected_outputs"] == EXPECTED_OUTPUTS + final_command = next(command for command in manifest["test_commands"] if "build_m12_final_report.py" in command) + promotion_command = next(command for command in manifest["test_commands"] if "audit_m12_score_promotion.py" in command) + assert "--command-log-manifest" in final_command + assert all(argument in final_command for argument in FINAL_REPORT_MANIFEST_ARGUMENTS) + assert "--require-eligible" in final_command + assert "--command-log-manifest" in promotion_command + assert "--require-ready" in promotion_command + assert "--output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json" in promotion_command + assert "--final-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json" in promotion_command + assert "--scorecard ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" in promotion_command + assert "--claim-ledger ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" in promotion_command + assert ( + "--command-log-manifest ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_command_logs/command_log_manifest.json" + in promotion_command + ) + covered = {item["requirement"] for item in manifest["transfer_requirement_coverage"]} + assert { + "Repo snapshot or commit SHA", + "Python environment lock", + "RL Phase 1 source/test/doc overlay", + "ROCm/PyTorch/VeRL/Ray versions", + "EnvPackage set", + "Run manifests", + "Test command list", + "Expected outputs", + "Rollback plan", + "Final target report contract", + "Score promotion audit", + "Load ladder and soak evidence", + "Raw command log archive", + }.issubset(covered) + overlay_coverage = next(item for item in manifest["transfer_requirement_coverage"] if item["requirement"] == "RL Phase 1 source/test/doc overlay") + assert "breadboard/rl" in overlay_coverage["covered_by"] + assert "tests/rl" in overlay_coverage["covered_by"] + final_report_coverage = next(item for item in manifest["transfer_requirement_coverage"] if item["requirement"] == "Final target report contract") + assert "breadboard/rl/m12/final_report.py" in final_report_coverage["covered_by"] + assert "scripts/rl_phase1/build_m12_final_report.py" in final_report_coverage["covered_by"] + assert "scripts/rl_phase1/summarize_m12_final_report_remediations.py" in final_report_coverage["covered_by"] + promotion_coverage = next(item for item in manifest["transfer_requirement_coverage"] if item["requirement"] == "Score promotion audit") + assert "breadboard/rl/m12/promotion_audit.py" in promotion_coverage["covered_by"] + assert "scripts/rl_phase1/audit_m12_score_promotion.py" in promotion_coverage["covered_by"] + load_soak_coverage = next(item for item in manifest["transfer_requirement_coverage"] if item["requirement"] == "Load ladder and soak evidence") + assert "breadboard/rl/m12/load_soak.py" in load_soak_coverage["covered_by"] + assert "scripts/rl_phase1/run_m12_load_ladder.py" in load_soak_coverage["covered_by"] + assert "scripts/rl_phase1/run_m12_soak.py" in load_soak_coverage["covered_by"] + assert "Archive preflight, run reports, and raw command logs with sha256 hashes." in manifest["rollback_plan"] + assert "If load/soak artifacts are missing or non-eligible, preserve the final report as a blocked target outcome." in manifest["rollback_plan"] + assert "Build and validate m12_final_report.json before any scorecard edit." in manifest["rollback_plan"] + assert "Do not update scorecard unless M12 target evidence satisfies the gate." in manifest["rollback_plan"] + assert all(artifact["exists"] for artifact in manifest["artifacts"]) + assert all(artifact.get("sha256", "").startswith("sha256:") for artifact in manifest["artifacts"] if artifact.get("kind") == "file") + + +def test_m12_transfer_pack_writes_portable_execution_files(tmp_path) -> None: + manifest = write_m12_transfer_pack(repo_root=REPO_ROOT, output_dir=tmp_path) + + manifest_path = tmp_path / "m12_transfer_manifest.json" + commands_path = tmp_path / "m12_test_commands.sh" + rollback_path = tmp_path / "m12_rollback_plan.md" + readiness_path = tmp_path / "m12_readiness_summary.json" + load_template_path = tmp_path / "m12_load_ladder_report_template.json" + soak_template_path = tmp_path / "m12_soak_report_template.json" + command_log_template_path = tmp_path / "m12_command_log_manifest_template.json" + overlay_script_path = tmp_path / "m12_apply_overlay.py" + bootstrap_script_path = tmp_path / "m12_target_bootstrap.sh" + + assert manifest_path.exists() + assert commands_path.exists() + assert overlay_script_path.exists() + assert bootstrap_script_path.exists() + assert rollback_path.exists() + assert readiness_path.exists() + assert load_template_path.exists() + assert soak_template_path.exists() + assert command_log_template_path.exists() + assert json.loads(manifest_path.read_text(encoding="utf-8"))["manifest_id"] == manifest["manifest_id"] + + commands = commands_path.read_text(encoding="utf-8") + assert 'REPO_ROOT="${REPO_ROOT:-$(pwd)}"' in commands + assert "Set REPO_ROOT to the BreadBoard repository root" in commands + assert "run_m12_preflight.py" in commands + assert "run_m12_logged_command.py" in commands + assert 'COMMAND_LOG_MANIFEST="$COMMAND_LOG_DIR/command_log_manifest.json"' in commands + assert 'M12_FINAL_REPORT_PATH="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json"' in commands + assert 'M12_REMEDIATION_SUMMARY_PATH="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_remediation_summary.json"' in commands + assert "m12_on_error()" in commands + assert "trap m12_on_error ERR" in commands + assert "summarize_m12_final_report_remediations.py" in commands + assert 'M12_TARGET_RUN_ID="${M12_TARGET_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)-$$}"' in commands + assert 'echo "m12_target_run_id=$M12_TARGET_RUN_ID"' in commands + assert "Existing M12 command log manifest belongs to different target run id(s)" in commands + assert '--target-run-id "$M12_TARGET_RUN_ID"' in commands + assert "--command-id target_transfer_archive_verify" in commands + assert "--output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_archive_verify/m12_archive_verify_report.json" in commands + assert "--command-id phase1_validation_suite" in commands + assert "--command-id target_preflight" in commands + assert "--command-id target_swe_probe" in commands + assert "--command-id target_verl_export" in commands + assert "--command-id target_ray_warm_pool" in commands + assert "--distributed" in commands + assert "--num-workers 20" in commands + assert "--command-id target_load_ladder" in commands + assert "--command-id target_soak" in commands + assert "--command-id final_report" in commands + assert "--command-id promotion_audit" in commands + assert "run_m12_load_ladder.py" in commands + assert "run_m12_soak.py" in commands + assert "--require-pass" in commands + assert "build_m12_final_report.py" in commands + assert "--command-log-manifest" in commands + assert all(argument in commands for argument in FINAL_REPORT_SCRIPT_ARGUMENTS) + assert "--require-eligible" in commands + assert "audit_m12_score_promotion.py" in commands + assert "--output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json" in commands + assert "--final-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json" in commands + assert "--scorecard ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" in commands + assert "--claim-ledger ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" in commands + assert "--require-ready" in commands + assert validate_m12_test_commands_script(commands, manifest) == [] + assert [ + line.split("--command-id ", 1)[1].split(" ", 1)[0] + for line in commands.splitlines() + if "run_m12_logged_command.py" in line + ] == [command_id for command_id, _ in M12_TEST_COMMAND_ROWS] + + rollback = rollback_path.read_text(encoding="utf-8") + assert "Archive preflight, run reports, and raw command logs with sha256 hashes." in rollback + assert "If load/soak artifacts are missing or non-eligible, preserve the final report as a blocked target outcome." in rollback + assert "Build and validate m12_final_report.json before any scorecard edit." in rollback + assert "Do not update scorecard unless M12 target evidence satisfies the gate." in rollback + + readiness = json.loads(readiness_path.read_text(encoding="utf-8")) + assert readiness["summary_id"] == "bb_zyphra_rl_phase1_m12_readiness_summary_v1" + assert readiness["scorecard_update_allowed"] is False + assert readiness["m12_points_awarded"] is False + assert readiness["artifact_count"] == len(manifest["artifacts"]) + assert readiness["command_count"] == len(manifest["test_commands"]) + assert readiness["expected_output_count"] == len(manifest["expected_outputs"]) + assert readiness["target_script_fail_closed"]["archive_verifier_runs_first"] is True + assert readiness["target_script_fail_closed"]["preflight_requires_pass"] is True + assert readiness["target_script_fail_closed"]["final_report_requires_eligible"] is True + assert readiness["target_script_fail_closed"]["promotion_audit_requires_ready"] is True + assert readiness["target_script_fail_closed"]["promotion_audit_uses_explicit_score_inputs"] is True + assert readiness["target_script_fail_closed"]["promotion_audit_uses_explicit_target_paths"] is True + assert readiness["target_script_fail_closed"]["generated_script_uses_logged_command_wrapper"] is True + assert readiness["target_script_fail_closed"]["generated_script_runs_concrete_load_soak"] is True + assert readiness["target_script_fail_closed"]["bootstrap_rejects_dirty_checkout_by_default"] is True + assert readiness["target_script_fail_closed"]["bootstrap_runs_overlaid_test_commands"] is True + assert readiness["target_script_fail_closed"]["bootstrap_cds_to_repo_root_before_handoff"] is True + assert readiness["target_script_fail_closed"]["generated_script_sets_target_run_id"] is True + assert readiness["target_script_fail_closed"]["generated_script_rejects_mixed_target_run_logs"] is True + assert readiness["target_script_fail_closed"]["generated_script_rejects_stale_closeout_artifacts"] is True + assert readiness["target_script_fail_closed"]["generated_script_summarizes_final_report_remediations_on_error"] is True + assert readiness["target_script_fail_closed"]["generated_script_manifest_consistent"] is True + assert readiness["generated_script_validation_errors"] == [] + assert "m12_node_load_ladder/load_ladder_report.json" in readiness["target_only_required_outputs"] + assert "m12_command_logs/command_log_manifest.json" in readiness["target_only_required_outputs"] + assert "m12_promotion_audit/m12_promotion_audit.json" in readiness["target_only_required_outputs"] + assert "m12_final_report.json has m12_score_eligible=true" in readiness["score_promotion_rule"] + assert "sha256 hashes" in readiness["score_promotion_rule"] + transfer_summary = json.loads((tmp_path / "m12_transfer_summary.json").read_text(encoding="utf-8")) + assert transfer_summary["bootstrap_overlaid_test_commands_handoff"] is True + assert transfer_summary["bootstrap_repo_root_cwd_handoff"] is True + assert transfer_summary["target_run_log_reuse_guard"] is True + assert transfer_summary["target_closeout_artifact_reuse_guard"] is True + assert validate_m12_readiness_summary(readiness, manifest) == [] + assert validate_m12_transfer_summary(transfer_summary, manifest) == [] + + load_template = json.loads(load_template_path.read_text(encoding="utf-8")) + assert load_template == LOAD_LADDER_REPORT_TEMPLATE + assert [item["target_sessions"] for item in load_template["concurrency_levels"]] == [5, 20, 50, 100] + assert load_template["policy_version_integrity"] is None + assert load_template["queue_backpressure_integrity"] is None + + soak_template = json.loads(soak_template_path.read_text(encoding="utf-8")) + assert soak_template == SOAK_REPORT_TEMPLATE + assert soak_template["minimum_duration_seconds"] == 7200 + assert soak_template["runtime_failure_count"] is None + + command_log_template = json.loads(command_log_template_path.read_text(encoding="utf-8")) + assert command_log_template == COMMAND_LOG_MANIFEST_TEMPLATE + assert command_log_template["manifest_id"] == "bb_zyphra_rl_phase1_m12_command_log_manifest_v1" + assert command_log_template["all_required_logs_archived"] is False + assert command_log_template["all_required_commands_passed"] is False + assert [item["command_id"] for item in command_log_template["commands"]] == [ + "target_transfer_archive_verify", + "phase1_validation_suite", + "target_preflight", + "target_swe_probe", + "target_verl_export", + "target_ray_warm_pool", + "target_load_ladder", + "target_soak", + "final_report", + "promotion_audit", + ] + assert command_log_template["required_command_ids"][0] == "target_transfer_archive_verify" + final_report_entry = next(item for item in command_log_template["commands"] if item["command_id"] == "final_report") + assert final_report_entry["required"] is False + promotion_audit_entry = next(item for item in command_log_template["commands"] if item["command_id"] == "promotion_audit") + assert promotion_audit_entry["required"] is False + + +def test_m12_readiness_and_transfer_summary_validators_reject_stale_counts() -> None: + manifest = build_m12_transfer_manifest(REPO_ROOT) + readiness = build_m12_readiness_summary(manifest) + readiness["artifact_count"] += 1 + readiness["target_script_fail_closed"]["preflight_requires_pass"] = False + + readiness_errors = validate_m12_readiness_summary(readiness, manifest) + + assert "artifact_count must match transfer manifest" in readiness_errors + assert "target_script_fail_closed.preflight_requires_pass must match transfer manifest" in readiness_errors + + transfer_summary = build_m12_transfer_summary(manifest) + transfer_summary["command_count"] -= 1 + transfer_summary["final_command_explicit_target_artifact_args"] = False + transfer_summary["promotion_audit_explicit_target_paths"] = False + transfer_summary["target_run_log_reuse_guard"] = False + transfer_summary["target_closeout_artifact_reuse_guard"] = False + transfer_summary["generated_script_manifest_consistent"] = False + + transfer_errors = validate_m12_transfer_summary(transfer_summary, manifest) + + assert "command_count must match transfer manifest" in transfer_errors + assert "final_command_explicit_target_artifact_args must match transfer manifest" in transfer_errors + assert "promotion_audit_explicit_target_paths must match transfer manifest" in transfer_errors + assert "target_run_log_reuse_guard must match transfer manifest" in transfer_errors + assert "target_closeout_artifact_reuse_guard must match transfer manifest" in transfer_errors + assert "generated_script_manifest_consistent must match transfer manifest" in transfer_errors + + +def test_m12_test_command_validator_rejects_script_manifest_drift() -> None: + manifest = build_m12_transfer_manifest(REPO_ROOT) + script = build_m12_test_commands_script() + + missing_target_run = script.replace('--target-run-id "$M12_TARGET_RUN_ID" ', "", 1) + errors = validate_m12_test_commands_script(missing_target_run, manifest) + assert "logged command line mismatch at position 1: target_transfer_archive_verify" in errors + assert "logged command line missing target run binding: target_transfer_archive_verify" in errors + + missing_log_reuse_guard = script.replace( + "Existing M12 command log manifest belongs to different target run id(s)", + "Existing M12 command log manifest guard removed", + ) + errors = validate_m12_test_commands_script(missing_log_reuse_guard, manifest) + assert "m12_test_commands.sh must reject mixed target-run command logs" in errors + + assert "M12_PROMOTION_AUDIT_PATH" in script + assert "Existing M12 close-out artifact would make target evidence ambiguous" in script + missing_closeout_guard = script.replace( + "Existing M12 close-out artifact would make target evidence ambiguous", + "Existing M12 close-out artifact guard removed", + ) + errors = validate_m12_test_commands_script(missing_closeout_guard, manifest) + assert "m12_test_commands.sh must reject stale close-out artifacts" in errors + + missing_scorecard = script.replace( + " --scorecard ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml", + "", + ) + errors = validate_m12_test_commands_script(missing_scorecard, manifest) + assert "promotion_audit command must pass explicit --scorecard path" in errors + + missing_claim_ledger = script.replace( + " --claim-ledger ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md", + "", + ) + errors = validate_m12_test_commands_script(missing_claim_ledger, manifest) + assert "promotion_audit command must pass explicit --claim-ledger path" in errors + + missing_promotion_output = script.replace( + " --output ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json", + "", + 1, + ) + errors = validate_m12_test_commands_script(missing_promotion_output, manifest) + assert "promotion_audit command must pass explicit --output path" in errors + + missing_promotion_final_report = script.replace( + " --final-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_final_report.json", + "", + ) + errors = validate_m12_test_commands_script(missing_promotion_final_report, manifest) + assert "promotion_audit command must pass explicit --final-report path" in errors + + missing_promotion_manifest = script.replace( + ' --claim-ledger ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md --command-log-manifest "$COMMAND_LOG_MANIFEST"', + " --claim-ledger ../docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md", + 1, + ) + errors = validate_m12_test_commands_script(missing_promotion_manifest, manifest) + assert "promotion_audit command must pass explicit --command-log-manifest path" in errors + + missing_promotion_require_ready = script.replace(" --require-ready", "", 1) + errors = validate_m12_test_commands_script(missing_promotion_require_ready, manifest) + assert "promotion_audit command must require ready evidence" in errors + + missing_final_report_input = script.replace( + " --load-ladder-report ../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_node_load_ladder/load_ladder_report.json", + "", + ) + errors = validate_m12_test_commands_script(missing_final_report_input, manifest) + assert "final_report command must pass explicit --load-ladder-report path" in errors + + wrong_manifest = dict(manifest) + wrong_manifest["test_commands"] = list(manifest["test_commands"][:-1]) + errors = validate_m12_test_commands_script(script, wrong_manifest) + assert "transfer manifest test_commands must match M12_TEST_COMMANDS" in errors + assert "transfer manifest test_commands count must match M12_TEST_COMMAND_ROWS" in errors + + +def test_m12_test_command_validator_requires_failure_remediation_trap() -> None: + manifest = build_m12_transfer_manifest(REPO_ROOT) + script = build_m12_test_commands_script() + assert "trap m12_on_error ERR" in script + assert "summarize_m12_final_report_remediations.py" in script + assert "M12_REMEDIATION_SUMMARY_PATH" in script + + missing_trap = script.replace("trap m12_on_error ERR\n\n", "") + errors = validate_m12_test_commands_script(missing_trap, manifest) + assert "m12_test_commands.sh must install ERR trap for final-report remediation summary" in errors + + missing_summary = script.replace("scripts/rl_phase1/summarize_m12_final_report_remediations.py", "") + errors = validate_m12_test_commands_script(missing_summary, manifest) + assert "m12_test_commands.sh must summarize final-report remediations on failure" in errors + + missing_summary_path = script.replace( + 'M12_REMEDIATION_SUMMARY_PATH="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_final_report/m12_remediation_summary.json"\n', + "", + ) + errors = validate_m12_test_commands_script(missing_summary_path, manifest) + assert "m12_test_commands.sh must define M12_REMEDIATION_SUMMARY_PATH" in errors + + missing_promotion_path = script.replace( + 'M12_PROMOTION_AUDIT_PATH="../docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_promotion_audit/m12_promotion_audit.json"\n', + "", + ) + errors = validate_m12_test_commands_script(missing_promotion_path, manifest) + assert "m12_test_commands.sh must define M12_PROMOTION_AUDIT_PATH" in errors + + +def test_m12_transfer_archive_is_portable_non_scoring_evidence_pack(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + + archive_path = tmp_path / archive_manifest["archive_path"] + sha_path = tmp_path / archive_manifest["archive_sha256_file"] + archive_manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + + assert archive_path.exists() + assert sha_path.exists() + assert archive_manifest_path.exists() + assert archive_manifest["archive_path"] == archive_path.name + assert archive_manifest["archive_sha256_file"] == sha_path.name + assert not Path(archive_manifest["archive_path"]).is_absolute() + assert not Path(archive_manifest["archive_sha256_file"]).is_absolute() + assert archive_manifest["archive_manifest_id"] == "bb_zyphra_rl_phase1_m12_transfer_archive_manifest_v1" + assert archive_manifest["claim_boundary"] == "transfer_archive_only_not_m12_validation" + assert archive_manifest["scorecard_update_allowed"] is False + assert archive_manifest["m12_points_awarded"] is False + assert archive_manifest["archive_is_repo_replacement"] is False + assert archive_manifest["archive_contains_source_overlay"] is True + assert archive_manifest["archive_excludes_pycache"] is True + assert archive_manifest["archive_deterministic"] is True + assert archive_manifest["source_paths_portable"] is True + assert archive_manifest["deterministic_archive_metadata"] == { + "gzip_mtime": 0, + "member_gid": 0, + "member_gname": "", + "member_mtime": 0, + "member_order": "sorted_by_archive_path", + "member_uid": 0, + "member_uname": "", + } + assert "breadboard/rl" in archive_manifest["source_overlay_paths"] + assert "tests/rl" in archive_manifest["source_overlay_paths"] + assert "exact repo SHA" in archive_manifest["required_operator_repo_step"] + assert "overlay the archived RL Phase 1" in archive_manifest["required_operator_repo_step"] + assert archive_manifest["archive_sha256"].startswith("sha256:") + assert archive_manifest["all_required_artifacts_present"] is True + assert archive_manifest["all_transfer_requirements_covered"] is True + assert archive_manifest["included_entry_count"] == len(archive_manifest["included_entries"]) + assert archive_manifest["included_entry_count"] > len(REQUIRED_TRANSFER_ARTIFACTS) + assert archive_manifest["generated_transfer_files"] == TRANSFER_PREP_FILES + assert sha_path.read_text(encoding="utf-8").startswith(archive_manifest["archive_sha256"]) + assert all(entry["mode"] in {0o644, 0o755} for entry in archive_manifest["included_entries"]) + assert all(entry["source_path"] == entry["archive_path"] for entry in archive_manifest["included_entries"]) + assert all(not Path(entry["source_path"]).is_absolute() for entry in archive_manifest["included_entries"]) + assert all(".." not in Path(entry["source_path"]).parts for entry in archive_manifest["included_entries"]) + assert all(not any(str(key).startswith("_") for key in entry) for entry in archive_manifest["included_entries"]) + assert int.from_bytes(archive_path.read_bytes()[4:8], "little") == 0 + + archived_paths = {entry["archive_path"] for entry in archive_manifest["included_entries"]} + assert (REPO_ARCHIVE_ROOT / "scripts/rl_phase1/build_m12_transfer_archive.py").as_posix() in archived_paths + assert (REPO_ARCHIVE_ROOT / "scripts/rl_phase1/audit_m12_score_promotion.py").as_posix() in archived_paths + assert (REPO_ARCHIVE_ROOT / "scripts/rl_phase1/check_m12_evidence_consistency.py").as_posix() in archived_paths + assert (REPO_ARCHIVE_ROOT / "breadboard/rl/m12/promotion_audit.py").as_posix() in archived_paths + assert (REPO_ARCHIVE_ROOT / "breadboard/rl/m12/evidence_consistency.py").as_posix() in archived_paths + assert (REPO_ARCHIVE_ROOT / "tests/rl/m12/test_m12_transfer_pack.py").as_posix() in archived_paths + assert (REPO_ARCHIVE_ROOT / "tests/test_rl_phase1_scorecard_schema.py").as_posix() in archived_paths + assert (REPO_ARCHIVE_ROOT / "docs/rl_phase1/m12_transfer_pack.md").as_posix() in archived_paths + assert not any("__pycache__" in path or path.endswith(".pyc") for path in archived_paths) + assert "workspace/docs_tmp/ZYPHRA/RL_PHASE_1/BB_ZYPHRA_RL_PHASE_1_M12_VALIDATION_REPORT.md" in archived_paths + assert "workspace/docs_tmp/ZYPHRA/RL_PHASE_1/runs/m6_controlled_swe_toy/run_summary.json" in archived_paths + assert any(path.endswith("/m12_test_commands.sh") for path in archived_paths) + assert any(path.endswith("/m12_apply_overlay.py") for path in archived_paths) + assert any(path.endswith("/m12_target_bootstrap.sh") for path in archived_paths) + assert any(path.endswith("/m12_transfer_summary.json") for path in archived_paths) + + with tarfile.open(archive_path, "r:gz") as archive: + members = [member for member in archive.getmembers() if member.isfile()] + assert [member.name for member in members] == sorted(entry["archive_path"] for entry in archive_manifest["included_entries"]) + archive_names = {member.name for member in members} + by_name = {member.name: member for member in members} + assert archived_paths.issubset(archive_names) + for entry in archive_manifest["included_entries"]: + member = by_name[entry["archive_path"]] + assert member.mtime == 0 + assert member.uid == 0 + assert member.gid == 0 + assert member.uname == "" + assert member.gname == "" + assert member.mode == entry["mode"] + assert validate_m12_transfer_archive_manifest(archive_manifest_path) == [] + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/verify_m12_transfer_archive.py", + "--manifest", + str(archive_manifest_path), + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert result.returncode == 0 + assert "status=passed" in result.stdout + + +def test_m12_transfer_archive_is_deterministic_for_same_inputs(tmp_path) -> None: + first = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + second = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + + assert second["archive_sha256"] == first["archive_sha256"] + assert second["archive_size_bytes"] == first["archive_size_bytes"] + assert second["included_entries"] == first["included_entries"] + assert validate_m12_transfer_archive_manifest(tmp_path / "m12_transfer_archive_manifest.json") == [] + + +def test_m12_transfer_overlay_dry_run_is_non_scoring_and_safe(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + workspace_root = tmp_path / "target_workspace" + + report = apply_m12_transfer_overlay( + manifest_path=manifest_path, + workspace_root=workspace_root, + dry_run=True, + allow_overwrite=False, + ) + + assert report["report_id"] == "bb_zyphra_rl_phase1_m12_overlay_apply_report_v1" + assert report["claim_boundary"] == "transfer_overlay_application_not_m12_validation" + assert report["status"] == "passed" + assert report["dry_run"] is True + assert report["allow_overwrite"] is False + assert report["scorecard_update_allowed"] is False + assert report["m12_points_awarded"] is False + assert report["would_write_count"] == archive_manifest["included_entry_count"] + assert report["written_count"] == 0 + assert report["existing_destination_count"] == 0 + assert report["errors"] == [] + assert validate_m12_transfer_overlay_report(report) == [] + assert not (_overlay_repo_root(workspace_root) / "breadboard" / "rl" / "m12" / "transfer.py").exists() + + +def test_m12_transfer_overlay_report_validator_rejects_stale_or_ambiguous_summaries(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + workspace_root = tmp_path / "target_workspace" + report = apply_m12_transfer_overlay( + manifest_path=manifest_path, + workspace_root=workspace_root, + dry_run=True, + allow_overwrite=False, + ) + + failed_without_errors = dict(report) + failed_without_errors["status"] = "failed" + assert "failed report must include at least one error" in validate_m12_transfer_overlay_report(failed_without_errors) + + stale_existing_count = dict(report) + stale_existing_count["existing_destination_count"] = 1 + assert "existing_destination_count must equal entries with exists=true" in validate_m12_transfer_overlay_report( + stale_existing_count + ) + + stale_write_count = dict(report) + stale_write_count["written_count"] = len(report["entries"]) + 1 + assert "written_count must be between 0 and would_write_count" in validate_m12_transfer_overlay_report( + stale_write_count + ) + + malformed_entry = dict(report) + malformed_entry["entries"] = [dict(report["entries"][0], exists="yes")] + malformed_entry["would_write_count"] = 1 + assert any( + error.startswith("entry exists must be boolean:") + for error in validate_m12_transfer_overlay_report(malformed_entry) + ) + + +def test_m12_transfer_overlay_apply_writes_verified_workspace_members(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + workspace_root = tmp_path / "target_workspace" + + report = apply_m12_transfer_overlay( + manifest_path=manifest_path, + workspace_root=workspace_root, + dry_run=False, + allow_overwrite=False, + ) + + assert report["status"] == "passed" + assert report["dry_run"] is False + assert report["scorecard_update_allowed"] is False + assert report["m12_points_awarded"] is False + assert report["would_write_count"] == archive_manifest["included_entry_count"] + assert report["written_count"] == archive_manifest["included_entry_count"] + assert report["errors"] == [] + assert validate_m12_transfer_overlay_report(report) == [] + assert (_overlay_repo_root(workspace_root) / "scripts" / "rl_phase1" / "apply_m12_transfer_overlay.py").exists() + assert (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml").exists() + + +def test_m12_transfer_overlay_rejects_directory_destination_before_writes(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + workspace_root = tmp_path / "target_workspace" + conflicting_destination = ( + workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh" + ) + conflicting_destination.mkdir(parents=True) + + report = apply_m12_transfer_overlay( + manifest_path=manifest_path, + workspace_root=workspace_root, + dry_run=False, + allow_overwrite=True, + ) + + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert any("destination exists and is directory:" in error for error in report["errors"]) + assert validate_m12_transfer_overlay_report(report) == [] + assert conflicting_destination.is_dir() + + +def test_m12_transfer_overlay_reports_apply_time_write_failure(monkeypatch, tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + workspace_root = tmp_path / "target_workspace" + + def fail_write_bytes(self: Path, data: bytes) -> int: + raise OSError("simulated write race") + + monkeypatch.setattr(Path, "write_bytes", fail_write_bytes) + + report = apply_m12_transfer_overlay( + manifest_path=manifest_path, + workspace_root=workspace_root, + dry_run=False, + allow_overwrite=True, + ) + + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert any("overlay write failed:" in error and "simulated write race" in error for error in report["errors"]) + assert validate_m12_transfer_overlay_report(report) == [] + + +def test_generated_m12_overlay_script_runs_without_repo_imports(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + dry_run = subprocess.run( + [ + sys.executable, + str(script_path), + "--manifest", + str(manifest_path), + "--workspace-root", + str(workspace_root), + "--output", + str(report_path), + ], + cwd=tmp_path, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert dry_run.returncode == 0 + assert "status=passed" in dry_run.stdout + written = json.loads(report_path.read_text(encoding="utf-8")) + assert written["dry_run"] is True + assert written["written_count"] == 0 + assert written["would_write_count"] == archive_manifest["included_entry_count"] + + apply_run = subprocess.run( + [ + sys.executable, + str(script_path), + "--manifest", + str(manifest_path), + "--workspace-root", + str(workspace_root), + "--output", + str(report_path), + "--apply", + ], + cwd=tmp_path, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert apply_run.returncode == 0 + assert "status=passed" in apply_run.stdout + applied = json.loads(report_path.read_text(encoding="utf-8")) + assert applied["dry_run"] is False + assert applied["written_count"] == archive_manifest["included_entry_count"] + assert (_overlay_repo_root(workspace_root) / "scripts" / "rl_phase1" / "apply_m12_transfer_overlay.py").exists() + assert (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_target_bootstrap_dry_run_checks_sha_and_overlay(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + bootstrap_path = tmp_path / "prep" / "m12_target_bootstrap.sh" + bootstrap_text = bootstrap_path.read_text(encoding="utf-8") + assert "Repo checkout is dirty before M12 overlay" in bootstrap_text + assert "ALLOW_M12_DIRTY_CHECKOUT=1" in bootstrap_text + assert "repo_dirty_check=clean" in bootstrap_text + assert "repo_dirty_check=override" in bootstrap_text + assert 'TARGET_TEST_COMMANDS="$TARGET_PREP_DIR/m12_test_commands.sh"' in bootstrap_text + assert "Missing overlaid M12 test command script after overlay apply" in bootstrap_text + assert 'cd "$REPO_ROOT"' in bootstrap_text + assert 'bash "$TARGET_TEST_COMMANDS"' in bootstrap_text + + result = subprocess.run( + ["bash", str(bootstrap_path)], + cwd=REPO_ROOT, + env={ + "PATH": str(Path(sys.executable).parent) + ":" + os.environ.get("PATH", ""), + "REPO_ROOT": str(REPO_ROOT), + "WORKSPACE_ROOT": str(REPO_ROOT.parent), + "BOOTSTRAP_DRY_RUN_ONLY": "1", + "ALLOW_M12_DIRTY_CHECKOUT": "1", + }, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "bootstrap_dry_run_only=true" in result.stdout + assert "repo_dirty_check=" in result.stdout + report_path = tmp_path / "prep" / "m12_overlay_apply_dry_run_report.json" + assert report_path.exists() + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "passed" + assert report["dry_run"] is True + assert report["would_write_count"] == archive_manifest["included_entry_count"] + assert report["written_count"] == 0 + assert report["existing_destination_count"] > 0 + + +def test_generated_m12_target_bootstrap_hands_off_to_overlay_from_repo_root(tmp_path) -> None: + repo_root = tmp_path / "breadboard_repo_integration_main_20260326" + repo_root.mkdir() + (repo_root / "README.md").write_text("target repo\n", encoding="utf-8") + subprocess.run(["git", "init"], cwd=repo_root, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.run(["git", "add", "README.md"], cwd=repo_root, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.run( + [ + "git", + "-c", + "user.email=m12-bootstrap-test@example.invalid", + "-c", + "user.name=M12 Bootstrap Test", + "commit", + "-m", + "init", + ], + cwd=repo_root, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + head = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo_root, text=True).strip() + + prep_dir = tmp_path / "prep" + write_m12_transfer_pack(repo_root=REPO_ROOT, output_dir=prep_dir) + transfer_manifest_path = prep_dir / "m12_transfer_manifest.json" + transfer_manifest = json.loads(transfer_manifest_path.read_text(encoding="utf-8")) + transfer_manifest["repo"]["head"] = head + transfer_manifest_path.write_text(json.dumps(transfer_manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + (prep_dir / "m12_transfer_archive_manifest.json").write_text("{}\n", encoding="utf-8") + (prep_dir / "m12_transfer_evidence_pack.tar.gz").write_bytes(b"fake archive for bootstrap handoff test\n") + (prep_dir / "m12_transfer_evidence_pack.tar.gz.sha256").write_text( + "sha256:" + ("0" * 64) + " m12_transfer_evidence_pack.tar.gz\n", + encoding="utf-8", + ) + + (prep_dir / "m12_test_commands.sh").write_text("exit 42\n", encoding="utf-8") + (prep_dir / "m12_apply_overlay.py").write_text( + """#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument("--manifest") +parser.add_argument("--workspace-root", required=True) +parser.add_argument("--output", required=True) +parser.add_argument("--apply", action="store_true") +parser.add_argument("--allow-overwrite", action="store_true") +args = parser.parse_args() +if args.apply: + target = Path(args.workspace_root) / "docs_tmp/ZYPHRA/RL_PHASE_1/runs/m12_transfer_prep" + target.mkdir(parents=True, exist_ok=True) + (target / "m12_test_commands.sh").write_text( + 'printf "overlaid\\\\n" > "$M12_HANDOFF_MARKER"\\n' + 'pwd > "$M12_HANDOFF_CWD"\\n', + encoding="utf-8", + ) +report = { + "status": "passed", + "dry_run": not args.apply, + "written_count": 1 if args.apply else 0, + "would_write_count": 1, + "errors": [], +} +Path(args.output).write_text(json.dumps(report, indent=2, sort_keys=True) + "\\n", encoding="utf-8") +print("status=passed") +""", + encoding="utf-8", + ) + + handoff_marker = tmp_path / "handoff_marker.txt" + handoff_cwd = tmp_path / "handoff_cwd.txt" + result = subprocess.run( + ["bash", str(prep_dir / "m12_target_bootstrap.sh")], + cwd=tmp_path, + env={ + "PATH": str(Path(sys.executable).parent) + ":" + os.environ.get("PATH", ""), + "REPO_ROOT": str(repo_root), + "WORKSPACE_ROOT": str(tmp_path), + "M12_HANDOFF_MARKER": str(handoff_marker), + "M12_HANDOFF_CWD": str(handoff_cwd), + }, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert handoff_marker.read_text(encoding="utf-8") == "overlaid\n" + assert handoff_cwd.read_text(encoding="utf-8").strip() == str(repo_root) + + +def test_m12_transfer_archive_validator_detects_sidecar_mismatch(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + sha_path = tmp_path / archive_manifest["archive_sha256_file"] + sha_path.write_text("sha256:" + ("0" * 64) + " m12_transfer_evidence_pack.tar.gz\n", encoding="utf-8") + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert "archive sha256 sidecar does not match archive manifest" in errors + + +def test_m12_transfer_archive_verifier_cli_writes_non_scoring_report(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + report_path = tmp_path / "m12_archive_verify_report.json" + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/verify_m12_transfer_archive.py", + "--manifest", + str(manifest_path), + "--output", + str(report_path), + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 0 + assert "status=passed archive_manifest_verified=true" in result.stdout + assert str(report_path) in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["report_id"] == "bb_zyphra_rl_phase1_m12_archive_verify_report_v1" + assert report["claim_boundary"] == "transfer_archive_verification_not_m12_validation" + assert report["scorecard_update_allowed"] is False + assert report["m12_points_awarded"] is False + assert report["status"] == "passed" + assert report["archive_sha256"] == archive_manifest["archive_sha256"] + assert report["included_entry_count"] == archive_manifest["included_entry_count"] + assert report["errors"] == [] + + +def test_m12_transfer_archive_verifier_cli_writes_failed_report(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + report_path = tmp_path / "m12_archive_verify_report.json" + sha_path = tmp_path / archive_manifest["archive_sha256_file"] + sha_path.write_text("sha256:" + ("0" * 64) + " m12_transfer_evidence_pack.tar.gz\n", encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase1/verify_m12_transfer_archive.py", + "--manifest", + str(manifest_path), + "--output", + str(report_path), + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 5 + assert "status=failed archive_manifest_verified=false" in result.stdout + assert "archive sha256 sidecar does not match archive manifest" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["scorecard_update_allowed"] is False + assert report["m12_points_awarded"] is False + assert "archive sha256 sidecar does not match archive manifest" in report["errors"] + + +def test_m12_transfer_archive_validator_detects_member_hash_mismatch(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path) + manifest_path = tmp_path / "m12_transfer_archive_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + target_entry = manifest["included_entries"][0] + target_entry["sha256"] = "sha256:" + ("0" * 64) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert f"archive member sha256 mismatch: {target_entry['archive_path']}" in errors + + +def test_m12_transfer_archive_validator_detects_nonzero_gzip_mtime(tmp_path) -> None: + manifest_path = _write_archive_with_nonzero_gzip_mtime(tmp_path) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert "archive gzip mtime must be zero" in errors + + +def test_m12_transfer_archive_validator_detects_unsorted_members(tmp_path) -> None: + manifest_path = _write_archive_with_reversed_member_order(tmp_path) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert "archive member order must match sorted included_entries" in errors + + +def test_m12_transfer_archive_validator_rejects_semantically_stale_readiness_summary(tmp_path) -> None: + manifest_path = _write_archive_with_json_member_mutation( + tmp_path, + suffix="/m12_readiness_summary.json", + mutate=lambda document: ( + document.__setitem__("artifact_count", int(document["artifact_count"]) + 1), + document["target_script_fail_closed"].__setitem__("preflight_requires_pass", False), + ), + ) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert "archived m12_readiness_summary.json invalid: artifact_count must match transfer manifest" in errors + assert ( + "archived m12_readiness_summary.json invalid: target_script_fail_closed.preflight_requires_pass must match transfer manifest" + in errors + ) + + +def test_m12_transfer_archive_validator_rejects_semantically_stale_transfer_summary(tmp_path) -> None: + manifest_path = _write_archive_with_json_member_mutation( + tmp_path, + suffix="/m12_transfer_summary.json", + mutate=lambda document: ( + document.__setitem__("command_count", int(document["command_count"]) - 1), + document.__setitem__("generated_script_manifest_consistent", False), + ), + ) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert "archived m12_transfer_summary.json invalid: command_count must match transfer manifest" in errors + assert ( + "archived m12_transfer_summary.json invalid: generated_script_manifest_consistent must match transfer manifest" + in errors + ) + + +def test_m12_transfer_archive_validator_rejects_inner_transfer_manifest_boundary_drift(tmp_path) -> None: + manifest_path = _write_archive_with_json_member_mutation( + tmp_path, + suffix="/m12_transfer_manifest.json", + mutate=lambda document: ( + document.__setitem__("manifest_id", "stale_transfer_manifest"), + document.__setitem__("claim_boundary", "scorecard_update_allowed"), + document["repo"].__setitem__("root_path_portable", False), + ), + ) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert ( + "archived m12_transfer_manifest.json invalid: " + "manifest_id must be bb_zyphra_rl_phase1_m12_transfer_manifest_v1" + in errors + ) + assert ( + "archived m12_transfer_manifest.json invalid: " + "claim_boundary must remain transfer_preparation_only_not_m12_validation" + in errors + ) + assert "archived m12_transfer_manifest.json invalid: repo.root_path_portable must be true" in errors + + +def test_m12_transfer_archive_validator_detects_duplicate_manifest_entries(tmp_path) -> None: + manifest_path = _write_archive_with_duplicate_manifest_entry(tmp_path) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert "included_entries archive_path values must be unique" in errors + + +def test_m12_transfer_archive_validator_rejects_absolute_source_path(tmp_path) -> None: + manifest_path = _write_archive_with_absolute_source_path(tmp_path) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert any(error.startswith("unsafe included source path: ") for error in errors) + + +def test_m12_transfer_archive_validator_rejects_private_source_key(tmp_path) -> None: + manifest_path = _write_archive_with_private_source_key(tmp_path) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert "included_entry private keys are not allowed: _local_source_path" in errors + + +def test_m12_transfer_archive_validator_rejects_absolute_top_level_archive_path(tmp_path) -> None: + manifest_path = _write_archive_with_absolute_top_level_archive_path(tmp_path) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert "archive_path must be portable colocated file name: m12_transfer_evidence_pack.tar.gz" in errors + + +def test_m12_transfer_archive_validator_detects_duplicate_tar_members(tmp_path) -> None: + manifest_path = _write_archive_with_duplicate_tar_member(tmp_path) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert "archive file member paths must be unique" in errors + + +def test_m12_transfer_archive_validator_detects_archived_script_manifest_drift(tmp_path) -> None: + manifest_path = _write_archive_with_test_command_target_run_removed(tmp_path) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert ( + "archived m12_test_commands.sh invalid: logged command line mismatch at position 1: target_transfer_archive_verify" + in errors + ) + assert ( + "archived m12_test_commands.sh invalid: logged command line missing target run binding: target_transfer_archive_verify" + in errors + ) + + +def test_m12_transfer_archive_validator_rejects_archived_script_without_closeout_guard(tmp_path) -> None: + manifest_path = _write_archive_with_test_command_closeout_guard_removed(tmp_path) + + errors = validate_m12_transfer_archive_manifest(manifest_path) + + assert "archived m12_test_commands.sh invalid: m12_test_commands.sh must reject stale close-out artifacts" in errors + + +def test_generated_m12_overlay_script_rejects_nonzero_gzip_mtime_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_nonzero_gzip_mtime(tmp_path / "prep") + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert "archive gzip mtime must be zero" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_unsorted_members_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_reversed_member_order(tmp_path / "prep") + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert "archive member order must match sorted included_entries" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_duplicate_manifest_entries_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_duplicate_manifest_entry(tmp_path / "prep") + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert "included_entries archive_path values must be unique" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_absolute_source_path_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_absolute_source_path(tmp_path / "prep") + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert "unsafe included source path: " in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_absolute_top_level_archive_path_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_absolute_top_level_archive_path(tmp_path / "prep") + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert "archive_path must be portable colocated file name: m12_transfer_evidence_pack.tar.gz" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_duplicate_tar_members_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_duplicate_tar_member(tmp_path / "prep") + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert "archive file member paths must be unique" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_archived_script_manifest_drift_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_test_command_target_run_removed(tmp_path / "prep") + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = subprocess.run( + [ + sys.executable, + str(script_path), + "--manifest", + str(manifest_path), + "--workspace-root", + str(workspace_root), + "--output", + str(report_path), + ], + cwd=tmp_path, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 6 + assert "archived logged command line mismatch at position 1: target_transfer_archive_verify" in result.stdout + assert "archived logged command line missing target run binding: target_transfer_archive_verify" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_inner_transfer_manifest_boundary_drift_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_json_member_mutation( + tmp_path / "prep", + suffix="/m12_transfer_manifest.json", + mutate=lambda document: ( + document.__setitem__("manifest_id", "stale_transfer_manifest"), + document.__setitem__("claim_boundary", "scorecard_update_allowed"), + document["repo"].__setitem__("root_path_portable", False), + ), + ) + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert ( + "archived m12_transfer_manifest.json invalid: " + "manifest_id must be bb_zyphra_rl_phase1_m12_transfer_manifest_v1" + in result.stdout + ) + assert ( + "archived m12_transfer_manifest.json invalid: " + "claim_boundary must remain transfer_preparation_only_not_m12_validation" + in result.stdout + ) + assert "archived m12_transfer_manifest.json invalid: repo.root_path_portable must be true" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_stale_readiness_summary_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_json_member_mutation( + tmp_path / "prep", + suffix="/m12_readiness_summary.json", + mutate=lambda document: ( + document.__setitem__("artifact_count", int(document["artifact_count"]) + 1), + document["target_script_fail_closed"].__setitem__("preflight_requires_pass", False), + ), + ) + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert "archived m12_readiness_summary.json invalid: artifact_count must match transfer manifest" in result.stdout + assert ( + "archived m12_readiness_summary.json invalid: target_script_fail_closed.preflight_requires_pass must match transfer manifest" + in result.stdout + ) + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_readiness_boundary_drift_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_json_member_mutation( + tmp_path / "prep", + suffix="/m12_readiness_summary.json", + mutate=lambda document: ( + document.__setitem__("summary_id", "stale_readiness_summary"), + document.__setitem__("scorecard_update_allowed", True), + document.__setitem__("target_only_required_outputs", []), + ), + ) + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert ( + "archived m12_readiness_summary.json invalid: " + "summary_id must be bb_zyphra_rl_phase1_m12_readiness_summary_v1" + in result.stdout + ) + assert "archived m12_readiness_summary.json invalid: scorecard_update_allowed must be false" in result.stdout + assert ( + "archived m12_readiness_summary.json invalid: target_only_required_outputs must match expected target outputs" + in result.stdout + ) + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_readiness_fail_closed_key_drift_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_json_member_mutation( + tmp_path / "prep", + suffix="/m12_readiness_summary.json", + mutate=lambda document: document["target_script_fail_closed"].__setitem__("extra_fail_open_gate", True), + ) + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert ( + "archived m12_readiness_summary.json invalid: " + "target_script_fail_closed keys must match expected fail-closed checks" + in result.stdout + ) + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_stale_transfer_summary_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_json_member_mutation( + tmp_path / "prep", + suffix="/m12_transfer_summary.json", + mutate=lambda document: ( + document.__setitem__("command_count", int(document["command_count"]) - 1), + document.__setitem__("generated_script_manifest_consistent", False), + ), + ) + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert "archived m12_transfer_summary.json invalid: command_count must match transfer manifest" in result.stdout + assert ( + "archived m12_transfer_summary.json invalid: generated_script_manifest_consistent must match transfer manifest" + in result.stdout + ) + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_transfer_summary_boundary_drift_before_writes(tmp_path) -> None: + manifest_path = _write_archive_with_json_member_mutation( + tmp_path / "prep", + suffix="/m12_transfer_summary.json", + mutate=lambda document: ( + document.__setitem__("manifest_id", "stale_transfer_manifest"), + document.__setitem__("claim_boundary", "scorecard_update_allowed"), + document.__setitem__("concrete_load_soak_scripts", False), + ), + ) + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + + result = _run_generated_overlay_script( + script_path=script_path, + manifest_path=manifest_path, + report_path=report_path, + workspace_root=workspace_root, + cwd=tmp_path, + ) + + assert result.returncode == 6 + assert ( + "archived m12_transfer_summary.json invalid: " + "manifest_id must be bb_zyphra_rl_phase1_m12_transfer_manifest_v1" + in result.stdout + ) + assert ( + "archived m12_transfer_summary.json invalid: " + "claim_boundary must remain transfer_preparation_only_not_m12_validation" + in result.stdout + ) + assert "archived m12_transfer_summary.json invalid: concrete_load_soak_scripts must match transfer manifest" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_sidecar_mismatch_before_writes(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + (tmp_path / "prep" / archive_manifest["archive_sha256_file"]).write_text( + "sha256:" + ("0" * 64) + " m12_transfer_evidence_pack.tar.gz\n", + encoding="utf-8", + ) + + result = subprocess.run( + [ + sys.executable, + str(script_path), + "--manifest", + str(manifest_path), + "--workspace-root", + str(workspace_root), + "--output", + str(report_path), + ], + cwd=tmp_path, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 6 + assert "archive sha256 sidecar does not match archive manifest" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_generated_file_coverage_drift_before_writes(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["generated_transfer_files"] = manifest["generated_transfer_files"][:-1] + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + str(script_path), + "--manifest", + str(manifest_path), + "--workspace-root", + str(workspace_root), + "--output", + str(report_path), + ], + cwd=tmp_path, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 6 + assert "generated_transfer_files must match GENERATED_TRANSFER_FILES" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_reports_corrupt_archive_before_writes(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + archive_path = tmp_path / "prep" / archive_manifest["archive_path"] + archive_path.write_bytes(b"not a tar.gz archive") + corrupt_sha = "sha256:" + hashlib.sha256(archive_path.read_bytes()).hexdigest() + sha_path = tmp_path / "prep" / archive_manifest["archive_sha256_file"] + sha_path.write_text(f"{corrupt_sha} {archive_path.name}\n", encoding="utf-8") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["archive_sha256"] = corrupt_sha + manifest["archive_size_bytes"] = archive_path.stat().st_size + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + str(script_path), + "--manifest", + str(manifest_path), + "--workspace-root", + str(workspace_root), + "--output", + str(report_path), + ], + cwd=tmp_path, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 6 + assert "archive file is not readable tar.gz:" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert not (workspace_root / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" / "runs" / "m12_transfer_prep" / "m12_test_commands.sh").exists() + + +def test_generated_m12_overlay_script_rejects_parent_file_collision_before_writes(tmp_path) -> None: + write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + workspace_root.mkdir() + blocking_parent = workspace_root / "docs_tmp" + blocking_parent.write_text("not a directory\n", encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + str(script_path), + "--manifest", + str(manifest_path), + "--workspace-root", + str(workspace_root), + "--output", + str(report_path), + "--apply", + "--allow-overwrite", + ], + cwd=tmp_path, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + assert result.returncode == 6 + assert "destination parent exists and is not directory:" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert report["written_count"] == 0 + assert blocking_parent.is_file() + + +def test_generated_m12_overlay_script_reports_apply_time_write_failure(tmp_path) -> None: + archive_manifest = write_m12_transfer_archive(repo_root=REPO_ROOT, output_dir=tmp_path / "prep") + manifest_path = tmp_path / "prep" / "m12_transfer_archive_manifest.json" + script_path = tmp_path / "prep" / "m12_apply_overlay.py" + report_path = tmp_path / "overlay_report.json" + workspace_root = tmp_path / "target_workspace" + unwritable_parent = _overlay_repo_root(workspace_root) / "breadboard" / "rl" + unwritable_parent.mkdir(parents=True) + unwritable_parent.chmod(0o500) + try: + if os.access(unwritable_parent, os.W_OK): + pytest.skip("filesystem permissions allow writes despite chmod; cannot force write failure safely") + result = subprocess.run( + [ + sys.executable, + str(script_path), + "--manifest", + str(manifest_path), + "--workspace-root", + str(workspace_root), + "--output", + str(report_path), + "--apply", + "--allow-overwrite", + ], + cwd=tmp_path, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + finally: + unwritable_parent.chmod(0o700) + + assert result.returncode == 6 + assert "overlay write failed:" in result.stdout + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["status"] == "failed" + assert 0 < report["written_count"] < archive_manifest["included_entry_count"] diff --git a/tests/rl/phase2/__init__.py b/tests/rl/phase2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/rl/phase2/test_loop_scale.py b/tests/rl/phase2/test_loop_scale.py new file mode 100644 index 00000000..4e9f5507 --- /dev/null +++ b/tests/rl/phase2/test_loop_scale.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +from breadboard.rl.phase2.benchmark import ( + build_benchmark_slice_report, + build_fixture_benchmark_source_pin, + source_sha256, +) +from breadboard.rl.phase2.benchflow import fixture_benchflow_import_report +from breadboard.rl.phase2.closed_loop import build_closed_loop_fixture_ledger +from breadboard.rl.phase2.env_family import build_lean_console_fixture_probe_report +from breadboard.rl.phase2.scale import build_scale_ladder_v2_report, fixture_scale_metrics +from breadboard.rl.phase2.verifier import ( + VerifierCallEvidence, + build_live_verifier_integration_report, +) + + +def test_closed_loop_fixture_closes_accepted_and_rejected_replays() -> None: + ledger = build_closed_loop_fixture_ledger(target_run_id="target-fixture-loop") + + assert ledger["claim_boundary"] == "p2_m3_closed_loop_prototype_not_production_rl_claim" + assert ledger["scorecard_update_allowed"] is False + assert ledger["passed"] is True + assert ledger["target_run_id"] == "target-fixture-loop" + assert ledger["policy_snapshot"]["policy_snapshot_id"].startswith("policy_snapshot:") + + closures = {closure["replay_status"]: closure for closure in ledger["replay_closures"]} + assert set(closures) == {"accepted_replay_closed", "rejected_replay_closed"} + assert closures["accepted_replay_closed"]["admission_accepted"] is True + assert closures["accepted_replay_closed"]["trainer_handoff_id"] == "trainer_handoff:accepted-only" + assert closures["accepted_replay_closed"]["errors"] == [] + assert closures["rejected_replay_closed"]["admission_accepted"] is False + assert closures["rejected_replay_closed"]["trainer_handoff_id"] is None + assert closures["rejected_replay_closed"]["errors"] == [] + + +def test_scale_ladder_v2_level_gates_and_failure_taxonomy() -> None: + metrics = fixture_scale_metrics() + metrics[250] = { + **metrics[250], + "scheduler_failures": 1, + } + + report = build_scale_ladder_v2_report(metrics, target_run_id="target-fixture-scale").to_dict() + + assert report["claim_boundary"] == "p2_m4_scale_ladder_not_arbitrary_production_scale" + assert report["scorecard_update_allowed"] is False + assert report["passed"] is False + assert [level["level"] for level in report["levels"]] == [100, 250, 500, 1000] + assert report["failure_taxonomy"]["scheduler"] == 1 + assert report["failure_taxonomy"]["verifier"] == 50 + scheduler_250 = [ + gate + for gate in report["gates"] + if gate["level"] == 250 and gate["gate"] == "scheduler_resilience" + ][0] + assert scheduler_250["passed"] is False + assert report["overall_passed"] is False + + +def test_benchmark_slice_rejects_source_hash_mismatch() -> None: + source_pin = build_fixture_benchmark_source_pin() + report = build_benchmark_slice_report( + source_pin, + observed_source_sha256=source_sha256("tampered fixture\n"), + contamination_controls=[ + "source_hash_pin", + "train_overlap_manifest", + "prompt_solution_leakage_scan", + ], + failure_replay_refs=["cas://benchmark/failure-replay/001"], + metrics={"attempted": 3, "completed": 2, "failed": 1}, + target_run_id="target-fixture-benchmark", + ).to_dict() + + assert report["status"] == "rejected_hash_mismatch" + assert report["claim_boundary"] == "p2_m5_named_benchmark_slice_not_general_benchmark_claim" + assert report["accepted_for_claim"] is False + assert report["passed"] is False + assert report["scorecard_update_allowed"] is False + assert report["source_pin"]["expected_source_sha256"] != report["observed_source_sha256"] + assert report["errors"] == ["source_hash_mismatch"] + + +def test_live_verifier_instability_quarantines_report() -> None: + calls = [ + VerifierCallEvidence( + call_id="ors-call-1", + provider="ors", + endpoint_id="ors.fixture.local", + verifier_version="openreward-fixture-v1", + request_hash="a" * 64, + response_hash="b" * 64, + reward_scalar=0.70, + latency_ms=40, + ), + VerifierCallEvidence( + call_id="openreward-call-1", + provider="openreward", + endpoint_id="openreward.fixture.local", + verifier_version="openreward-fixture-v2", + request_hash="a" * 64, + response_hash="c" * 64, + reward_scalar=0.95, + latency_ms=55, + ), + ] + + report = build_live_verifier_integration_report( + calls, + baseline_reward=0.70, + drift_tolerance=0.05, + target_run_id="target-fixture-verifier", + ).to_dict() + + assert report["status"] == "quarantined_verifier_instability" + assert report["claim_boundary"] == "p2_m6_live_verifier_probe_not_general_verifier_claim" + assert report["quarantined"] is True + assert report["passed"] is False + assert report["scorecard_update_allowed"] is False + assert report["max_abs_drift"] == 0.25 + assert "reward_drift_exceeded" in report["quarantine_reasons"] + assert "verifier_version_drift" in report["quarantine_reasons"] + + +def test_benchflow_hardening_import_preserves_and_loses_fields() -> None: + report = fixture_benchflow_import_report().to_dict() + + assert report["claim_boundary"] == "p2_m7_benchflow_probe_not_full_security_coverage_claim" + assert report["scorecard_update_allowed"] is False + assert report["preserved_fields"] == [ + "workspace_isolation", + "network_egress_block", + "path_traversal_probe", + ] + assert report["lost_fields"] == ["benchflow_harbor_attestation"] + assert report["field_mapping"]["path_traversal_probe"] == "RewardHackProbeSuite.path_escape" + assert report["imported_probe_catches_fixture"] is True + + +def test_second_environment_family_target_smoke_report_schema() -> None: + report = build_lean_console_fixture_probe_report().to_dict() + + assert report["report_id"] == "bb_zyphra_rl_phase2_second_env_family_v1" + assert report["family_id"] == "lean_console" + assert report["env_package_id"] == "lean_console_fixture_env" + assert report["target_run_id"] == "fixture-phase2-second-env-family" + assert report["claim_boundary"] == "p2_m8_second_environment_probe_not_general_env_support_claim" + assert report["scorecard_update_allowed"] is False + assert report["smoke_ready"] is True + assert set(report) >= { + "renderer_probe", + "replay_probe", + "export_probe", + "target_smoke", + "preserved_fields", + "lost_fields", + } + assert report["target_smoke"] == { + "name": "target_smoke", + "status": "passed", + "evidence_ref": "cas://env-family/target-smoke/lean_console", + } diff --git a/tests/rl/phase2/test_service_final.py b/tests/rl/phase2/test_service_final.py new file mode 100644 index 00000000..731998b9 --- /dev/null +++ b/tests/rl/phase2/test_service_final.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +from dataclasses import replace + +from breadboard.rl.phase2.final_report import ( + EXPECTED_MILESTONE_CLAIM_BOUNDARIES, + PHASE2_COMPONENT_MILESTONES, + PHASE2_FINAL_CLAIM_BOUNDARY, + PHASE2_FINAL_REPORT_ID, + build_phase2_component_report, + build_phase2_final_report, + validate_phase2_final_report, +) +from breadboard.rl.phase2.hardening import ( + ArtifactEgressRequest, + DestructiveActionRequest, + EgressPolicy, + build_hardening_report, + validate_hardening_report, +) +from breadboard.rl.phase2.observability import ( + ObservabilityCaps, + ObservabilitySample, + build_observability_report, + validate_observability_report, +) +from breadboard.rl.phase2.promotion_audit import ( + build_phase2_promotion_audit, + validate_phase2_promotion_audit, +) +from breadboard.rl.phase2.service import ( + ArtifactRecord, + RLRunServiceContract, + ResourceCaps, + RunSubmission, + validate_service_surface_report, +) + + +def _submission(run_id: str = "phase2-service-run") -> RunSubmission: + return RunSubmission( + run_id=run_id, + tenant_id="tenant-a", + workspace_id="workspace-a", + env_package_ref="sha256:env-package", + target_run_id="20260622T190029Z-slurm-231441", + requested_tasks=10, + requested_gpus=2, + requested_budget_usd=25.0, + requested_duration_seconds=600, + ) + + +def test_submit_status_cancel_collect_replay_audit_lifecycle() -> None: + service = RLRunServiceContract( + ResourceCaps( + max_tasks=100, + max_gpus=8, + max_budget_usd=100.0, + max_duration_seconds=3600, + max_artifact_bytes=4096, + ) + ) + + submitted = service.submit(_submission()) + assert submitted.state == "queued" + assert submitted.accepted is True + assert service.status("phase2-service-run").state == "queued" + rejected = service.submit(replace(_submission("over-cap-run"), requested_tasks=101)) + assert rejected.state == "rejected" + assert rejected.accepted is False + assert "requested_tasks exceeds max_tasks" in rejected.reason + + + assert service.start("phase2-service-run").state == "running" + cancel_requested = service.cancel("phase2-service-run", operator_id="operator-a", reason="operator stop") + assert cancel_requested.state == "cancel_requested" + assert cancel_requested.cancellation_state == "operator_requested" + assert service.acknowledge_cancelled("phase2-service-run").state == "cancelled" + + service.add_artifact( + ArtifactRecord( + run_id="phase2-service-run", + artifact_id="replay-1", + relative_path="workspace-a/artifacts/replay.jsonl", + sha256="sha256:abc", + bytes=128, + egress_allowed=True, + ) + ) + collected = service.collect("phase2-service-run") + assert collected["artifacts"][0]["artifact_id"] == "replay-1" + assert service.replay("phase2-service-run", artifact_id="replay-1")["replay_available"] is True + assert [event["event_type"] for event in service.stream("phase2-service-run")] == [ + "run_submitted", + "run_started", + "cancel_requested", + "run_cancelled", + "artifact_recorded", + ] + + audit = service.audit("phase2-service-run") + assert validate_service_surface_report(audit) == [] + assert audit["scorecard_update_allowed"] is False + assert audit["target_run_id"] == "20260622T190029Z-slurm-231441" + + +def test_observability_report_records_cap_rejection() -> None: + caps = ObservabilityCaps( + max_budget_usd=50.0, + max_gpu_hours=4.0, + max_queue_wait_seconds=120.0, + max_verifier_latency_ms=500.0, + max_failure_rate=0.10, + ) + report = build_observability_report( + run_id="phase2-observe-run", + target_run_id="target-run", + caps=caps, + requested_budget_usd=75.0, + projected_gpu_hours=8.0, + samples=[ + ObservabilitySample(30.0, 70.0, 10, 5.0, 100.0), + ObservabilitySample(240.0, 80.0, 5, 5.0, 900.0, failure_class="verifier_timeout"), + ], + ) + + assert validate_observability_report(report) == [] + assert report["cap_evaluation"]["accepted"] is False + assert report["failure_taxonomy"] == {"verifier_timeout": 1} + assert report["task_throughput"]["tasks_per_second"] == 1.5 + assert "requested_budget_usd exceeds max_budget_usd" in report["cap_evaluation"]["rejections"] + assert "projected_gpu_hours exceeds max_gpu_hours" in report["cap_evaluation"]["rejections"] + + +def test_hardening_redaction_egress_and_destructive_guards() -> None: + report = build_hardening_report( + run_id="phase2-hardening-run", + target_run_id="target-run", + tenant_id="tenant-a", + workspace_id="workspace-a", + egress_policy=EgressPolicy(allowed_prefixes=("workspace-a/artifacts",), max_artifact_bytes=1024), + egress_requests=[ + ArtifactEgressRequest("workspace-a/artifacts/public.json", 64, "public"), + ArtifactEgressRequest("../escape/private.txt", 64, "secret"), + ], + destructive_actions=[ + DestructiveActionRequest("safe", "python run.py", "workspace-a/jobs/run.py"), + DestructiveActionRequest("bad", "rm -rf /", "/"), + ], + environment={"API_TOKEN": "secret-token", "CACHE_DIR": "/Users/operator/cache", "SAFE_FLAG": "1"}, + adversarial_package_results=[{"package_id": "path_escape", "passed": False}], + ) + + assert validate_hardening_report(report) == [] + assert report["redacted_environment"]["API_TOKEN"] == "" + assert report["redacted_environment"]["CACHE_DIR"] == "" + assert report["artifact_egress_results"][0]["allowed"] is True + assert report["artifact_egress_results"][1]["allowed"] is False + assert report["destructive_action_guards"][0]["allowed"] is True + assert report["destructive_action_guards"][1]["allowed"] is False + assert report["hardening_passed"] is False + + +def _complete_milestone_reports(target_run_id: str) -> dict[str, dict]: + return { + milestone_id: build_phase2_component_report( + milestone_id=milestone_id, + report_id=f"report-{milestone_id}", + claim_boundary=EXPECTED_MILESTONE_CLAIM_BOUNDARIES[milestone_id], + target_run_id=target_run_id, + passed=True, + ) + for milestone_id in PHASE2_COMPONENT_MILESTONES + } + + +def test_final_report_fails_when_any_milestone_report_missing() -> None: + target_run_id = "target-run" + reports = _complete_milestone_reports(target_run_id) + del reports["P2-M10"] + + final_report = build_phase2_final_report( + target_run_id=target_run_id, + milestone_reports=reports, + command_log_manifest={"target_run_id": target_run_id, "commands": []}, + ) + + assert final_report["final_report_ready"] is False + assert final_report["missing_milestones"] == ["P2-M10"] + assert "missing_milestones must be empty" in validate_phase2_final_report(final_report) + + +def test_promotion_audit_ready_only_when_scorecard_claim_ledger_and_final_report_agree() -> None: + target_run_id = "target-run" + final_report = build_phase2_final_report( + target_run_id=target_run_id, + milestone_reports=_complete_milestone_reports(target_run_id), + command_log_manifest={"target_run_id": target_run_id, "commands": ["archived"]}, + ) + assert validate_phase2_final_report(final_report) == [] + + scorecard = { + "claim_boundary": PHASE2_FINAL_CLAIM_BOUNDARY, + "final_report_id": PHASE2_FINAL_REPORT_ID, + "scorecard_update_allowed": False, + "target_run_id": target_run_id, + "total_points": 1000, + } + claim_ledger = { + "allowed_claim_boundary": PHASE2_FINAL_CLAIM_BOUNDARY, + "final_report_id": PHASE2_FINAL_REPORT_ID, + "target_run_id": target_run_id, + } + + ready_audit = build_phase2_promotion_audit( + target_run_id=target_run_id, + scorecard=scorecard, + claim_ledger=claim_ledger, + final_report=final_report, + ) + assert ready_audit["promotion_review_ready"] is True + assert validate_phase2_promotion_audit(ready_audit) == [] + + bad_scorecard = dict(scorecard, target_run_id="other-run") + blocked_audit = build_phase2_promotion_audit( + target_run_id=target_run_id, + scorecard=bad_scorecard, + claim_ledger=claim_ledger, + final_report=final_report, + ) + assert blocked_audit["promotion_review_ready"] is False + assert "scorecard.target_run_id_matches" in blocked_audit["missing_requirements"] diff --git a/tests/rl/phase2/test_verl_bridge.py b/tests/rl/phase2/test_verl_bridge.py new file mode 100644 index 00000000..d0af2b16 --- /dev/null +++ b/tests/rl/phase2/test_verl_bridge.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from breadboard.rl.phase2.bridge import ( + VERL_BATCH_CLAIM_BOUNDARY, + build_verl_batch_from_projection_rows, + build_verl_dataproto_like_payload, + detect_verl_dataproto_api, +) +from breadboard.rl.phase2.trainer import build_trainer_dry_run_report, report_to_json + + +def _row(**overrides): + row = { + "rollout_id": "run.phase2", + "trajectory_id": "run.phase2.task-1.trajectory", + "episode_id": "run.phase2.task-1.episode", + "task_id": "task-1", + "split_id": "train", + "env_package_id": "env.pkg", + "env_package_hash": "sha256:env", + "group_id": "group-a", + "policy_snapshot_id": "policy-snapshot-001", + "policy": {"policy_id": "policy-a", "policy_snapshot_id": "policy-snapshot-001"}, + "prompt_ids": [11, 12], + "completion_ids": [21, 22], + "input_ids": [11, 12, 21, 22], + "attention_mask": [1, 1, 1, 1], + "loss_mask": [False, False, True, True], + "assistant_mask": [False, False, True, True], + "tool_action_mask": [False, False, False, True], + "reward_mask": [False, False, False, True], + "completion_logprobs": [-0.25, -0.5], + "completion_logprob_status": "native_available", + "renderer": {"renderer_id": "renderer-a"}, + "reward": {"scalar": 1.0, "reward_vector": {"unit": 1.0}}, + "runtime": {"runtime_backend": "local"}, + "admission": { + "row_status": "accepted", + "quarantine_status": "clear", + "trainable": True, + }, + "projection_manifest_id": "projection-001", + "trainable_candidate": True, + "metadata": {"source": "unit"}, + } + row.update(overrides) + return row + + +def test_valid_projection_rows_build_verl_batch() -> None: + batch = build_verl_batch_from_projection_rows([_row()], target_run_id="target-run-1") + payload = batch.to_dict() + + assert payload["claim_boundary"] == VERL_BATCH_CLAIM_BOUNDARY + assert payload["scorecard_update_allowed"] is False + assert payload["target_run_id"] == "target-run-1" + assert payload["policy_snapshot_id"] == "policy-snapshot-001" + assert payload["tensor_shape_metadata"]["input_ids"]["shape"] == [1, 4] + assert payload["tensor_shape_metadata"]["completion_logprobs"]["shape"] == [1, 2] + assert payload["masks"]["reward_mask"] == [[False, False, False, True]] + assert payload["logprobs"]["completion_logprobs"] == [[-0.25, -0.5]] + assert payload["rewards"]["sequence_rewards"] == [1.0] + assert payload["verl_dataproto_api"]["required"] is False + dataproto_like = build_verl_dataproto_like_payload(batch) + assert dataproto_like["batch"]["old_log_probs"] == [[-0.25, -0.5]] + assert dataproto_like["non_tensor_batch"]["target_run_id"] == "target-run-1" + assert dataproto_like["meta_info"]["scorecard_update_allowed"] is False + + +def test_bad_mask_lengths_are_rejected() -> None: + row = _row(attention_mask=[1, 1, 1]) + + with pytest.raises(ValueError, match="attention_mask length must equal input_ids length"): + build_verl_batch_from_projection_rows([row], target_run_id="target-run-1") + + +def test_missing_policy_snapshot_is_rejected() -> None: + row = _row(policy={"policy_id": "policy-a"}) + row.pop("policy_snapshot_id") + + with pytest.raises(ValueError, match="policy_snapshot_id must be present"): + build_verl_batch_from_projection_rows([row], target_run_id="target-run-1") + + +def test_quarantined_rows_are_rejected() -> None: + row = _row( + admission={"row_status": "quarantined", "quarantine_status": "quarantined", "trainable": False}, + trainable_candidate=False, + ) + + with pytest.raises(ValueError, match="quarantined"): + build_verl_batch_from_projection_rows([row], target_run_id="target-run-1") + + +def test_preserved_and_lost_field_ledger_is_recorded() -> None: + batch = build_verl_batch_from_projection_rows( + [_row(full_workspace_bytes="not materialized into trainer batch")], + target_run_id="target-run-1", + ) + ledger = batch.to_dict()["field_ledger"] + + assert "policy_snapshot_id" in ledger["preserved_fields"] + assert "task_id" in ledger["preserved_fields"] + assert "full_workspace_bytes" in ledger["lost_fields"] + assert ledger["row_ledgers"][0]["lost_fields"] == ["full_workspace_bytes"] + assert ledger["provenance_loss_detected"] is False + + +def test_optional_real_verl_dataproto_detection_does_not_require_dependency() -> None: + def importer(module_name: str): + if module_name == "verl.protocol": + return SimpleNamespace(DataProto=object) + raise ImportError(module_name) + + evidence = detect_verl_dataproto_api(importer) + + assert evidence["available"] is True + assert evidence["required"] is False + assert evidence["module"] == "verl.protocol" + + +def test_trainer_dry_run_report_records_modes_and_device_evidence() -> None: + batch = build_verl_batch_from_projection_rows([_row()], target_run_id="target-run-1") + + no_update_report = build_trainer_dry_run_report( + batch, + target_run_id="target-run-1", + mode="no_update", + device="cpu", + ) + one_step_report = build_trainer_dry_run_report( + batch, + target_run_id="target-run-1", + mode="one_step", + device="cpu", + ) + + assert no_update_report["scorecard_update_allowed"] is False + assert no_update_report["mode"] == "no_update" + assert no_update_report["dry_run_result"]["planned_step_count"] == 0 + assert one_step_report["mode"] == "one_step" + assert one_step_report["dry_run_result"]["planned_step_count"] == 1 + assert one_step_report["target_run_id"] == "target-run-1" + assert one_step_report["device_evidence"]["requested_device"] == "cpu" + assert one_step_report["dry_run_result"]["weight_update_performed"] is False + assert report_to_json(one_step_report).endswith("\n") + + +def test_trainer_dry_run_rejects_provenance_loss() -> None: + batch_payload = build_verl_batch_from_projection_rows([_row()], target_run_id="target-run-1").to_dict() + batch_payload["field_ledger"]["lost_fields"] = ["policy_snapshot_id"] + batch_payload["field_ledger"]["critical_lost_fields"] = [] + batch_payload["field_ledger"]["provenance_loss_detected"] = False + + with pytest.raises(ValueError, match="rejects provenance loss"): + build_trainer_dry_run_report(batch_payload, target_run_id="target-run-1") diff --git a/tests/rl/phase3/test_api_router.py b/tests/rl/phase3/test_api_router.py new file mode 100644 index 00000000..bb10a4eb --- /dev/null +++ b/tests/rl/phase3/test_api_router.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import pytest + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from breadboard.rl.phase3.api_router import create_phase3_rl_router +from breadboard.rl.phase3.service_live import LiveRLRunService +from breadboard.rl.phase3.store import SQLiteRLRunStore + + +def client(tmp_path): + app = FastAPI() + app.include_router(create_phase3_rl_router(LiveRLRunService(SQLiteRLRunStore(tmp_path / "runs.db"))), prefix="/rl") + return TestClient(app) + + +def payload() -> dict: + return {"run_id": "run-1", "tenant_id": "tenant-a", "workspace_id": "ws", "env_package_ref": "ws/env.tar", "target_run_id": "20260623T000000Z-slurm-234555", "requested_tasks": 1, "requested_gpus": 1, "requested_budget_usd": 1, "requested_duration_seconds": 60} + + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("requested_tasks", 0), + ("requested_tasks", -1), + ("requested_gpus", 0), + ("requested_gpus", -1), + ("requested_budget_usd", 0.0), + ("requested_budget_usd", -1.0), + ("requested_duration_seconds", 0), + ("requested_duration_seconds", -1), + ], +) +def test_submit_rejects_non_positive_resource_requests(tmp_path, field: str, value: int | float) -> None: + c = client(tmp_path) + body = payload() + body["run_id"] = f"run-{field}-{abs(int(value))}" + body[field] = value + + response = c.post("/rl/runs", json=body) + + assert response.status_code == 422 + assert field in response.text + +def test_submit_status_cancel_events(tmp_path) -> None: + c = client(tmp_path) + response = c.post("/rl/runs", json=payload()) + assert response.status_code == 200 + assert response.json()["state"] == "queued" + assert c.get("/rl/runs/run-1", params={"tenant_id": "tenant-a", "workspace_id": "ws"}).json()["run_id"] == "run-1" + assert c.post("/rl/runs/run-1/cancel", json={"tenant_id": "tenant-a", "workspace_id": "ws", "reason": "stop"}).json()["cancellation_state"] == "requested" + events = c.get("/rl/runs/run-1/events", params={"tenant_id": "tenant-a", "workspace_id": "ws"}).text.strip().splitlines() + assert len(events) >= 2 + + +def test_tenant_mismatch_returns_403(tmp_path) -> None: + c = client(tmp_path) + c.post("/rl/runs", json=payload()) + response = c.get("/rl/runs/run-1", params={"tenant_id": "tenant-b", "workspace_id": "ws"}) + assert response.status_code == 403 + + +def test_cli_bridge_default_run_store_uses_sqlite_memory_dsn(tmp_path, monkeypatch) -> None: + from agentic_coder_prototype.api.cli_bridge.app import create_app + + monkeypatch.delenv("BREADBOARD_RL_RUN_STORE", raising=False) + monkeypatch.chdir(tmp_path) + + bridge = TestClient(create_app()) + response = bridge.post("/v1/rl/runs", json=payload()) + + assert response.status_code == 200 + assert bridge.get("/v1/rl/runs/run-1", params={"tenant_id": "tenant-a", "workspace_id": "ws"}).status_code == 200 + assert bridge.get("/rl/runs/run-1", params={"tenant_id": "tenant-a", "workspace_id": "ws"}).status_code == 200 + assert not (tmp_path / ":memory:").exists() diff --git a/tests/rl/phase3/test_benchmark_campaign.py b/tests/rl/phase3/test_benchmark_campaign.py new file mode 100644 index 00000000..47451be5 --- /dev/null +++ b/tests/rl/phase3/test_benchmark_campaign.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +from breadboard.rl.phase2.benchmark import source_sha256 +from breadboard.rl.phase3.benchmark_campaign import BenchmarkCampaignSpec, build_benchmark_campaign_report +from breadboard.rl.phase3.evidence import PHASE3_COMMAND_LOG_MANIFEST_SCHEMA, sha256_file + +TARGET = "20260623T000000Z-slurm-234555" + + +def manifest(evidence: Path) -> dict: + raw = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "command_logs" / "cmd.log" + raw.parent.mkdir(parents=True) + raw.write_text("ok") + return {"schema_version": PHASE3_COMMAND_LOG_MANIFEST_SCHEMA, "target_run_id": TARGET, "commands": [{"command_id": "cmd", "argv": ["x"], "raw_log_path": "command_logs/cmd.log", "raw_log_sha256": sha256_file(raw), "slurm_job_id": "1", "target_run_id": TARGET, "node": "n", "started_at": "a", "completed_at": "b", "exit_code": 0, "status": "passed"}]} + + +def test_benchmark_hash_mismatch_fails(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "bench" + out.mkdir(parents=True) + summary = out / "summary.json"; summary.write_text(json.dumps({"source_payload": "actual", "metrics": {"attempted": 1}})) + contam = out / "contam.json"; contam.write_text(json.dumps({"controls": ["source_hash_pin", "train_overlap_manifest", "prompt_solution_leakage_scan"], "train_overlap_manifest": "none", "prompt_solution_leakage_scan": "none"})) + report = build_benchmark_campaign_report(BenchmarkCampaignSpec("bench", "v1", "fixture", "bad", "validation", 1, contam, out), run_summary_path=summary, replay_dir=out, command_log_manifest=manifest(evidence)) + assert report["passed"] is False + assert report["errors"] + +def test_benchmark_campaign_uses_docs_tmp_evidence_root(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "bench" + out.mkdir(parents=True) + source_payload = "external-benchmark-row\n" + summary = out / "summary.json" + summary.write_text(json.dumps({ + "source_payload": source_payload, + "failed_tasks": [], + "metrics": {"attempted": 1, "accepted": 1, "rejected": 0, "quarantined": 0}, + })) + contam = out / "contam.json" + contam.write_text(json.dumps({ + "controls": ["source_hash_pin", "train_overlap_manifest", "prompt_solution_leakage_scan"], + "train_overlap_manifest": "none", + "prompt_solution_leakage_scan": "none", + })) + report = build_benchmark_campaign_report( + BenchmarkCampaignSpec("bench", "v1", "fixture", source_sha256(source_payload), "validation", 1, contam, out), + run_summary_path=summary, + replay_dir=out, + command_log_manifest=manifest(evidence), + ) + assert report["passed"] is True + assert report["source_pin"]["benchmark_version"] == "v1" + assert report["slice_report"]["claim_boundary"] == "phase3_named_benchmark_campaign_scope" + assert report["slice_report"]["metadata"]["benchmark_input_kind"] == "external_jsonl" + +def test_external_benchmark_all_success_does_not_require_failure_replay(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "bench" + out.mkdir(parents=True) + source_payload = "external-benchmark-row\n" + summary = out / "summary.json" + summary.write_text(json.dumps({ + "source_payload": source_payload, + "failed_tasks": [], + "quarantined_tasks": [], + "metrics": {"attempted": 1, "accepted": 1, "rejected": 0, "quarantined": 0}, + })) + contam = out / "contam.json" + contam.write_text(json.dumps({ + "controls": ["source_hash_pin", "train_overlap_manifest", "prompt_solution_leakage_scan"], + "train_overlap_manifest": "none", + "prompt_solution_leakage_scan": "none", + })) + report = build_benchmark_campaign_report( + BenchmarkCampaignSpec("bench", "v1", "external", source_sha256(source_payload), "validation", 1, contam, out), + run_summary_path=summary, + replay_dir=out, + command_log_manifest=manifest(evidence), + ) + + assert report["passed"] is True + assert "missing_failure_replay" not in report["errors"] + assert "missing_failure_replay" not in report["slice_report"]["errors"] + assert report["slice_report"]["status"] == "external_benchmark_package_accepted" + + + +def test_benchmark_campaign_rejects_zero_accepted_external_tasks(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "bench" + out.mkdir(parents=True) + source_payload = "external-benchmark-row\n" + summary = out / "summary.json" + summary.write_text(json.dumps({ + "source_payload": source_payload, + "failed_tasks": ["task-1"], + "quarantined_tasks": [], + "metrics": {"attempted": 1, "accepted": 0, "rejected": 1, "quarantined": 0}, + })) + (out / "task-1.json").write_text("{}") + contam = out / "contam.json" + contam.write_text(json.dumps({ + "controls": ["source_hash_pin", "train_overlap_manifest", "prompt_solution_leakage_scan"], + "train_overlap_manifest": "none", + "prompt_solution_leakage_scan": {"candidate_source": "phase3_vllm_model_pipeline"}, + })) + report = build_benchmark_campaign_report( + BenchmarkCampaignSpec("bench", "v1", "external", source_sha256(source_payload), "validation", 1, contam, out), + run_summary_path=summary, + replay_dir=out, + command_log_manifest=manifest(evidence), + ) + + assert report["passed"] is False + assert "benchmark_no_accepted_tasks" in report["errors"] + assert "benchmark_no_accepted_tasks" in report["slice_report"]["errors"] + assert report["slice_report"]["status"] == "rejected_external_benchmark_controls" + + +def test_benchmark_campaign_rejects_unsafe_duplicate_and_mismatched_replay_ids(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "bench" + out.mkdir(parents=True) + source_payload = "external-benchmark-row\n" + summary = out / "summary.json" + summary.write_text(json.dumps({ + "source_payload": source_payload, + "failed_tasks": ["task-1", "task-1", "../secret", "task-2"], + "quarantined_tasks": [], + "metrics": {"attempted": 4, "accepted": 1, "rejected": 3, "quarantined": 0}, + })) + (out / "task-1.json").write_text(json.dumps({"task_id": "task-1"})) + (out / "task-2.json").write_text(json.dumps({"task_id": "other-task"})) + contam = out / "contam.json" + contam.write_text(json.dumps({ + "controls": ["source_hash_pin", "train_overlap_manifest", "prompt_solution_leakage_scan"], + "train_overlap_manifest": "none", + "prompt_solution_leakage_scan": {"candidate_source": "phase3_vllm_model_pipeline"}, + })) + + report = build_benchmark_campaign_report( + BenchmarkCampaignSpec("bench", "v1", "external", source_sha256(source_payload), "validation", 4, contam, out), + run_summary_path=summary, + replay_dir=out, + command_log_manifest=manifest(evidence), + ) + + assert report["passed"] is False + assert "benchmark_replay_task_id_duplicate:task-1" in report["errors"] + assert "unsafe replay task_id:../secret" in report["errors"] + assert "replay artifact task_id mismatch for task-2" in report["errors"] + + + +def test_benchmark_campaign_accepts_humaneval_slash_replay_ids(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "bench" + replay = out / "HumanEval" + replay.mkdir(parents=True) + source_payload = "external-benchmark-row\n" + summary = out / "summary.json" + summary.write_text(json.dumps({ + "source_payload": source_payload, + "failed_tasks": ["HumanEval/0"], + "quarantined_tasks": [], + "metrics": {"attempted": 1, "accepted": 1, "rejected": 1, "quarantined": 0}, + })) + (replay / "0.json").write_text(json.dumps({"task_id": "HumanEval/0"})) + contam = out / "contam.json" + contam.write_text(json.dumps({ + "controls": ["source_hash_pin", "train_overlap_manifest", "prompt_solution_leakage_scan"], + "train_overlap_manifest": "none", + "prompt_solution_leakage_scan": {"candidate_source": "phase3_vllm_model_pipeline"}, + })) + + report = build_benchmark_campaign_report( + BenchmarkCampaignSpec("bench", "v1", "external", source_sha256(source_payload), "validation", 1, contam, out), + run_summary_path=summary, + replay_dir=out, + command_log_manifest=manifest(evidence), + ) + + assert report["passed"] is True + assert str(out / "HumanEval" / "0.json") in report["slice_report"]["failure_replay_refs"] + + +def test_benchmark_campaign_rejects_duplicate_logical_replay_artifacts(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "bench" + (out / "HumanEval").mkdir(parents=True) + source_payload = "external-benchmark-row\n" + summary = out / "summary.json" + summary.write_text(json.dumps({ + "source_payload": source_payload, + "failed_tasks": [], + "quarantined_tasks": [], + "metrics": {"attempted": 1, "accepted": 1, "rejected": 0, "quarantined": 0}, + })) + (out / "HumanEval" / "0.json").write_text(json.dumps({"task_id": "HumanEval/0"})) + (out / "HumanEval_0.json").write_text(json.dumps({"task_id": "HumanEval/0"})) + contam = out / "contam.json" + contam.write_text(json.dumps({ + "controls": ["source_hash_pin", "train_overlap_manifest", "prompt_solution_leakage_scan"], + "train_overlap_manifest": "none", + "prompt_solution_leakage_scan": {"candidate_source": "phase3_vllm_model_pipeline"}, + })) + + report = build_benchmark_campaign_report( + BenchmarkCampaignSpec("bench", "v1", "external", source_sha256(source_payload), "validation", 1, contam, out), + run_summary_path=summary, + replay_dir=out, + command_log_manifest=manifest(evidence), + ) + + assert report["passed"] is False + assert any(error.startswith("duplicate replay artifact for HumanEval/0:") for error in report["errors"]) +def test_benchmark_campaign_rejects_hand_written_target_payload_candidates(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "bench" + out.mkdir(parents=True) + source_payload = "external-benchmark-row\n" + summary = out / "summary.json" + summary.write_text(json.dumps({ + "source_payload": source_payload, + "failed_tasks": [], + "quarantined_tasks": [], + "metrics": {"attempted": 1, "accepted": 1, "rejected": 0, "quarantined": 0}, + })) + contam = out / "contam.json" + contam.write_text(json.dumps({ + "controls": ["source_hash_pin", "train_overlap_manifest", "prompt_solution_leakage_scan"], + "train_overlap_manifest": "none", + "prompt_solution_leakage_scan": {"candidate_source": "hand_written_baseline_in_target_payload"}, + })) + report = build_benchmark_campaign_report( + BenchmarkCampaignSpec("bench", "v1", "external", source_sha256(source_payload), "validation", 1, contam, out), + run_summary_path=summary, + replay_dir=out, + command_log_manifest=manifest(evidence), + ) + + assert report["passed"] is False + assert "benchmark_candidate_source_not_phase3_model_pipeline" in report["errors"] + assert "benchmark_candidate_source_not_phase3_model_pipeline" in report["slice_report"]["errors"] + assert report["slice_report"]["status"] == "rejected_external_benchmark_controls" + assert report["slice_report"]["passed"] is False + assert report["slice_report"]["accepted_for_claim"] is False + +def test_benchmark_runner_uses_locked_fixture_without_external_jsonl(tmp_path: Path, monkeypatch) -> None: + evidence = tmp_path / "docs_tmp" + phase_dir = evidence / "ZYPHRA" / "RL_PHASE_3" + command_manifest = manifest(evidence) + (phase_dir / "runs").mkdir(parents=True, exist_ok=True) + (phase_dir / "runs" / "phase3_command_log_manifest.json").write_text(json.dumps(command_manifest)) + monkeypatch.delenv("PHASE3_BENCHMARK_JSONL", raising=False) + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/run_phase3_benchmark_campaign.py", + "--phase-dir", + str(phase_dir), + "--target-run-id", + TARGET, + ], + cwd=Path(__file__).resolve().parents[3], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert result.returncode == 0 + report = json.loads((phase_dir / "runs" / "benchmark_campaign" / "p3-m7_benchmark_campaign.json").read_text()) + assert report["passed"] is True + assert report["benchmark_report"]["slice_report"]["source_pin"]["benchmark_version"] == "fixture-v2" + assert report["required_artifact_keys"] == ["benchmark_report", "benchmark_source", "run_summary", "contamination", "replay_manifest"] diff --git a/tests/rl/phase3/test_closed_loop_runner.py b/tests/rl/phase3/test_closed_loop_runner.py new file mode 100644 index 00000000..d1c8886a --- /dev/null +++ b/tests/rl/phase3/test_closed_loop_runner.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from breadboard.rl.phase3.rollout_runner import Phase3ClosedLoopSpec, run_phase3_closed_loop, validate_phase3_closed_loop_report + +TARGET = "20260623T000000Z-slurm-234555" + + +def _spec(tmp_path: Path, rows: list[dict]) -> Phase3ClosedLoopSpec: + manifest = tmp_path / "tasks.json" + manifest.write_text(json.dumps({"rows": rows})) + env = tmp_path / "env.tar"; env.write_text("env") + return Phase3ClosedLoopSpec(TARGET, env, manifest, "policy-1", "verl_ppo", tmp_path / "out", 10) + + +def _row(task: str, status: str = "accepted", quarantine: str = "clear", replay: str | None = None, policy: str = "policy-1") -> dict: + return {"task_id": task, "policy_snapshot_id": policy, "admission": {"row_status": status, "quarantine_status": quarantine}, "accepted_replay_ref": replay, "input_ids": [1, 2], "prompt_ids": [1], "completion_ids": [2], "attention_mask": [1, 1], "loss_mask": [False, True], "assistant_mask": [False, True], "tool_action_mask": [False, False], "reward_mask": [False, True], "completion_logprobs": [-0.1], "completion_logprob_status": "native_available", "reward": {"scalar": 1}, "rollout_id": task, "trajectory_id": task, "episode_id": task, "projection_manifest_id": "pm", "trainable_candidate": True} + + +def test_accepted_rejected_replay_closure(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr("breadboard.rl.phase3.rollout_runner.build_phase3_dataproto", lambda *a, **k: object()) + report = run_phase3_closed_loop(_spec(tmp_path, [_row("a", replay="replay/a.json"), _row("b", status="rejected")])) + assert report["accepted_replay_refs"] == ["replay/a.json"] + assert report["rejected_count"] == 1 + assert validate_phase3_closed_loop_report(report) == [] + + +def test_quarantined_row_exclusion(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr("breadboard.rl.phase3.rollout_runner.build_phase3_dataproto", lambda batch, **k: (_ for _ in ()).throw(AssertionError("quarantined entered")) if any(r["task_id"] == "q" for r in batch["rows"]) else object()) + report = run_phase3_closed_loop(_spec(tmp_path, [_row("a", replay="replay/a.json"), _row("q", quarantine="quarantined")])) + assert report["quarantined_count"] == 1 + assert report["passed"] is True + + +def test_trainer_update_failure_propagates(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr("breadboard.rl.phase3.rollout_runner.build_phase3_dataproto", lambda *a, **k: (_ for _ in ()).throw(ValueError("bad"))) + report = run_phase3_closed_loop(_spec(tmp_path, [_row("a", replay="replay/a.json")])) + assert report["passed"] is False + assert any("trainer update failed" in error for error in report["errors"]) + + +def test_policy_snapshot_continuity(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr("breadboard.rl.phase3.rollout_runner.build_phase3_dataproto", lambda *a, **k: object()) + report = run_phase3_closed_loop(_spec(tmp_path, [_row("a", replay="replay/a.json", policy="wrong")])) + assert report["passed"] is False + assert any("policy snapshot" in error for error in report["errors"]) diff --git a/tests/rl/phase3/test_direct_node_preflight_runner.py b/tests/rl/phase3/test_direct_node_preflight_runner.py new file mode 100644 index 00000000..9ef58e5c --- /dev/null +++ b/tests/rl/phase3/test_direct_node_preflight_runner.py @@ -0,0 +1,565 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess + +import pytest + +import scripts.rl_phase3.run_phase3_direct_node_preflight as direct_node_preflight +from scripts.rl_phase3.run_phase3_direct_node_preflight import ENDPOINT_ENV_VARS, _presence_shell, _remote_precheck_command, _remote_run_command, main + + +def _payload_zip(tmp_path): # noqa: ANN001, ANN202 + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + return payload + + +def _load_assessment(output_dir, command_id: str): # noqa: ANN001, ANN202 + return json.loads((output_dir / f"{command_id}_direct_node_preflight.json").read_text()) + + +def _sha256_file(path): # noqa: ANN001, ANN202 + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_precheck_builder_records_presence_only_endpoint_checks_and_no_canonical_manifest() -> None: + remote = _remote_precheck_command( + direct_run_id="direct-20260706", + command_id="precheck_probe", + remote_root="/shared/bb-p3-root", + image="rocm-image:dev", + hip_visible_devices="0,1", + create_remote_root=False, + ) + + assert "PHASE4_DIRECT_MODE=precheck" in remote + assert "PHASE4_DIRECT_SCHEDULER=none_direct_ssh" in remote + assert "REMOTE_ROOT=present" in remote + assert "RUNTIME_VENV=present" in remote + assert "RUNTIME_IMAGE=present" in remote + assert "phase3_command_log_manifest.json" not in remote + for name in ENDPOINT_ENV_VARS: + assert f"ENV_{name}=present" in remote + assert f"ENV_{name}=absent" in remote + assert f"ENV_{name}=${{{name}" not in remote + + +def test_presence_shell_separates_endpoint_if_blocks_for_remote_precheck() -> None: + presence = _presence_shell(("BREADBOARD_ORS_TOKEN", "HF_HOME")) + remote = _remote_precheck_command( + direct_run_id="direct-20260706", + command_id="precheck_probe", + remote_root="/shared/bb-p3-root", + image="rocm-image:dev", + hip_visible_devices="0,1", + create_remote_root=False, + ) + + assert "fi if [ -n" not in presence + assert "fi if [ -n" not in remote + assert 'ENV_BREADBOARD_ORS_TOKEN=absent; fi; if [ -n "${HF_HOME:-}" ]' in presence + assert all(f"; fi; if [ -n \"${{{name}:-}}\" ]" in remote for name in ENDPOINT_ENV_VARS[1:]) + +def test_precheck_main_writes_non_promotional_assessment_hash_and_endpoint_booleans(tmp_path, monkeypatch) -> None: # noqa: ANN001 + payload = _payload_zip(tmp_path) + output_dir = tmp_path / "out" + monkeypatch.setenv("BREADBOARD_ORS_TOKEN", "super-secret-token") + calls: list[list[str]] = [] + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + calls.append(command) + assert command[0] == "ssh" + assert kwargs["env"]["BREADBOARD_ORS_TOKEN"] == "super-secret-token" + return subprocess.CompletedProcess( + command, + 0, + stdout=( + "PHASE4_DIRECT_NODE=perf-eng-2\n" + "PHASE4_DIRECT_MODE=precheck\n" + "PHASE4_DIRECT_SCHEDULER=none_direct_ssh\n" + "REMOTE_ROOT=present\n" + "RUNTIME_VENV=present\n" + "RUNTIME_IMAGE=present\n" + "CMD_docker=present:/usr/bin/docker\n" + "DEVICE_KFD=present\n" + "ENV_BREADBOARD_ORS_TOKEN=present\n" + "ENV_BREADBOARD_OPENREWARD_TOKEN=absent\n" + "ENV_HF_HOME=present\n" + "IMAGE_ID=sha256:image-id\n" + ), + stderr="", + ) + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--command-id", + "direct_precheck", + "--direct-run-id", + "direct-20260706T193804Z", + "--payload-zip", + str(payload), + "--output-dir", + str(output_dir), + "--mode", + "precheck", + "--image", + "rocm-image:dev", + "--hip-visible-devices", + "0,1,2,3", + ] + ) + + assert result == 0 + assert len(calls) == 1 + assert calls[0][0] == "ssh" + assert not (output_dir / "phase3_command_log_manifest.json").exists() + + assessment = _load_assessment(output_dir, "direct_precheck") + assert assessment["claim_boundary"] == "phase4_direct_node_dev_preflight_only_not_phase3_promotion" + assert assessment["promotional"] is False + assert assessment["scorecard_update_allowed"] is False + assert assessment["canonical_phase3_command_log_manifest_eligible"] is False + assert assessment["canonical_phase3_command_log_manifest_reason"] == "not_slurm_direct_ssh_preflight" + assert assessment["scheduler"] == "none_direct_ssh" + assert assessment["slurm_job_id_present"] is False + assert assessment["target_run_id"] is None + assert assessment["status"] == "passed" + assert assessment["passed"] is True + assert assessment["runtime"] == {"remote_root": "present", "venv": "present", "image": "present"} + assert assessment["endpoint_env_presence"]["BREADBOARD_ORS_TOKEN"] is True + assert assessment["endpoint_env_presence"]["BREADBOARD_OPENREWARD_TOKEN"] is False + assert assessment["endpoint_env_presence"]["HF_HOME"] is True + assert all(isinstance(value, bool) for value in assessment["endpoint_env_presence"].values()) + assert "super-secret-token" not in json.dumps(assessment, sort_keys=True) + + raw_log = output_dir / "command_logs" / "direct_precheck.log" + expected_raw_hash = _sha256_file(raw_log) + assert assessment["raw_log_path"] == "command_logs/direct_precheck.log" + assert assessment["raw_log_sha256"] == expected_raw_hash + assert assessment["input_hashes"]["raw_log"] == expected_raw_hash + + +@pytest.mark.parametrize( + ("missing_key", "runtime_field"), + [ + ("REMOTE_ROOT", "remote_root"), + ("RUNTIME_VENV", "venv"), + ("RUNTIME_IMAGE", "image"), + ], +) +def test_precheck_readiness_requires_runtime_root_venv_and_image(tmp_path, monkeypatch, missing_key: str, runtime_field: str) -> None: # noqa: ANN001 + payload = _payload_zip(tmp_path) + output_dir = tmp_path / f"out_{missing_key.lower()}" + values = {"REMOTE_ROOT": "present", "RUNTIME_VENV": "present", "RUNTIME_IMAGE": "present"} + values[missing_key] = "absent" + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + return subprocess.CompletedProcess( + command, + 0, + stdout=( + "PHASE4_DIRECT_NODE=perf-eng-2\n" + "PHASE4_DIRECT_MODE=precheck\n" + f"REMOTE_ROOT={values['REMOTE_ROOT']}\n" + f"RUNTIME_VENV={values['RUNTIME_VENV']}\n" + f"RUNTIME_IMAGE={values['RUNTIME_IMAGE']}\n" + ), + stderr="", + ) + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--command-id", + f"precheck_{missing_key.lower()}", + "--direct-run-id", + f"direct-{missing_key.lower()}", + "--payload-zip", + str(payload), + "--output-dir", + str(output_dir), + "--mode", + "precheck", + ] + ) + + assert result == 1 + assert not (output_dir / "phase3_command_log_manifest.json").exists() + assessment = _load_assessment(output_dir, f"precheck_{missing_key.lower()}") + assert assessment["passed"] is False + assert assessment["status"] == "not_ready" + assert assessment["runtime"][runtime_field] == "absent" + + +@pytest.mark.parametrize( + ("blocked_reason", "exit_code", "missing_line"), + [ + ("direct_runtime_image_missing", 93, "RUNTIME_IMAGE=absent"), + ("direct_runtime_venv_missing", 94, "RUNTIME_VENV=absent"), + ], +) +def test_run_mode_runtime_gate_blocks_before_payload_execution_and_never_writes_canonical_manifest( + tmp_path, monkeypatch, blocked_reason: str, exit_code: int, missing_line: str +) -> None: # noqa: ANN001 + payload = _payload_zip(tmp_path) + output_dir = tmp_path / f"out_{blocked_reason}" + events: list[str] = [] + + def fake_enforce(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + events.append("enforce") + assert command[0] == "ssh" + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + events.append(command[0]) + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + remote_payload = command[2] + assert remote_payload.index(blocked_reason) < remote_payload.index("WORK=$(mktemp") + assert remote_payload.index(blocked_reason) < remote_payload.index("python3 -m zipfile -e") + assert remote_payload.index(blocked_reason) < remote_payload.index("./run.sh") + return subprocess.CompletedProcess( + command, + exit_code, + stdout=( + "PHASE4_DIRECT_NODE=perf-eng-2\n" + "PHASE4_DIRECT_MODE=run\n" + f"PHASE4_BLOCKED_REASON={blocked_reason}\n" + f"{missing_line}\n" + ), + stderr="", + ) + + monkeypatch.setattr(direct_node_preflight, "enforce_command_request", fake_enforce) + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--command-id", + blocked_reason, + "--direct-run-id", + f"direct-{blocked_reason}", + "--payload-zip", + str(payload), + "--output-dir", + str(output_dir), + "--mode", + "run", + ] + ) + + assert result == exit_code + assert events == ["enforce", "scp", "ssh"] + assert not (output_dir / "phase3_command_log_manifest.json").exists() + assessment = _load_assessment(output_dir, blocked_reason) + assert assessment["mode"] == "run" + assert assessment["passed"] is False + assert assessment["status"] == "blocked" + assert assessment["blocked_reason"] == blocked_reason + assert assessment["canonical_phase3_command_log_manifest_eligible"] is False + assert assessment["component_reports"] == [] + + +def test_run_mode_nonzero_remote_without_blocked_reason_reports_remote_command_failed(tmp_path, monkeypatch) -> None: # noqa: ANN001 + payload = _payload_zip(tmp_path) + output_dir = tmp_path / "out_remote_command_failed" + events: list[str] = [] + + def fake_enforce(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + events.append("enforce") + assert command[0] == "ssh" + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + events.append(command[0]) + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + return subprocess.CompletedProcess( + command, + 17, + stdout=( + "PHASE4_DIRECT_NODE=perf-eng-2\n" + "PHASE4_DIRECT_MODE=run\n" + ), + stderr="payload exited 17\n", + ) + + monkeypatch.setattr(direct_node_preflight, "enforce_command_request", fake_enforce) + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--command-id", + "run_remote_command_failed", + "--direct-run-id", + "direct-remote-command-failed", + "--payload-zip", + str(payload), + "--output-dir", + str(output_dir), + "--mode", + "run", + ] + ) + + assert result == 17 + assert events == ["enforce", "scp", "ssh"] + assert not (output_dir / "phase3_command_log_manifest.json").exists() + assessment = _load_assessment(output_dir, "run_remote_command_failed") + assert assessment["mode"] == "run" + assert assessment["exit_code"] == 17 + assert assessment["passed"] is False + assert assessment["status"] == "blocked" + assert assessment["blocked_reason"] == "remote_command_failed" + assert assessment["component_reports"] == [] + + +def test_run_mode_nonzero_with_component_report_uses_component_blocker(tmp_path, monkeypatch) -> None: # noqa: ANN001 + payload = _payload_zip(tmp_path) + output_dir = tmp_path / "out_component_blocked" + + def fake_enforce(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + assert command[0] == "ssh" + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + component = {"component": "payload", "passed": False, "blocked_reason": "real_rollout_server_not_used"} + return subprocess.CompletedProcess( + command, + 2, + stdout=( + "PHASE4_DIRECT_NODE=perf-eng-2\n" + "PHASE4_DIRECT_MODE=run\n" + f"PHASE3_COMPONENT_REPORT_JSON={json.dumps(component)}\n" + ), + stderr="", + ) + + monkeypatch.setattr(direct_node_preflight, "enforce_command_request", fake_enforce) + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--command-id", + "run_component_blocked", + "--direct-run-id", + "direct-component-blocked", + "--payload-zip", + str(payload), + "--output-dir", + str(output_dir), + "--mode", + "run", + ] + ) + + assert result == 2 + assessment = _load_assessment(output_dir, "run_component_blocked") + assert assessment["passed"] is False + assert assessment["status"] == "blocked" + assert assessment["blocked_reason"] == "real_rollout_server_not_used" + assert assessment["component_blocked_reasons"] == ["real_rollout_server_not_used"] + assert assessment["component_failed_count"] == 1 + assert assessment["component_reports"][0]["blocked_reason"] == "real_rollout_server_not_used" + + +def test_run_mode_success_enforces_remote_command_before_payload_copy_and_ssh(tmp_path, monkeypatch) -> None: # noqa: ANN001 + payload = _payload_zip(tmp_path) + output_dir = tmp_path / "out_success" + events: list[str] = [] + + def fake_enforce(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + events.append("enforce") + assert command[0] == "ssh" + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + events.append(command[0]) + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + return subprocess.CompletedProcess( + command, + 0, + stdout=( + "PHASE4_DIRECT_NODE=perf-eng-2\n" + "PHASE4_DIRECT_MODE=run\n" + f"PHASE3_COMPONENT_REPORT_JSON={json.dumps({'component': 'payload', 'passed': True})}\n" + ), + stderr="", + ) + + monkeypatch.setattr(direct_node_preflight, "enforce_command_request", fake_enforce) + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--command-id", + "run_success", + "--direct-run-id", + "direct-run-success", + "--payload-zip", + str(payload), + "--output-dir", + str(output_dir), + "--mode", + "run", + ] + ) + + assert result == 0 + assert events == ["enforce", "scp", "ssh"] + assert not (output_dir / "phase3_command_log_manifest.json").exists() + assessment = _load_assessment(output_dir, "run_success") + assert assessment["mode"] == "run" + assert assessment["passed"] is True + assert assessment["status"] == "passed" + assert assessment["blocked_reason"] == "" + assert assessment["canonical_phase3_command_log_manifest_eligible"] is False + + + +def test_run_mode_exit_zero_without_inline_component_evidence_blocks_without_canonical_manifest( + tmp_path, monkeypatch +) -> None: # noqa: ANN001 + payload = _payload_zip(tmp_path) + output_dir = tmp_path / "out_missing_payload_evidence" + events: list[str] = [] + + def fake_enforce(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + events.append("enforce") + assert command[0] == "ssh" + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + events.append(command[0]) + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + return subprocess.CompletedProcess( + command, + 0, + stdout=( + "PHASE4_DIRECT_NODE=perf-eng-2\n" + "PHASE4_DIRECT_MODE=run\n" + ), + stderr="", + ) + + monkeypatch.setattr(direct_node_preflight, "enforce_command_request", fake_enforce) + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--command-id", + "run_missing_payload_evidence", + "--direct-run-id", + "direct-missing-payload-evidence", + "--payload-zip", + str(payload), + "--output-dir", + str(output_dir), + "--mode", + "run", + ] + ) + + assert result == 1 + assert events == ["enforce", "scp", "ssh"] + assert not (output_dir / "phase3_command_log_manifest.json").exists() + assessment = _load_assessment(output_dir, "run_missing_payload_evidence") + assert assessment["claim_boundary"] == "phase4_direct_node_dev_preflight_only_not_phase3_promotion" + assert assessment["promotional"] is False + assert assessment["scorecard_update_allowed"] is False + assert assessment["mode"] == "run" + assert assessment["exit_code"] == 0 + assert assessment["passed"] is False + assert assessment["status"] == "blocked" + assert assessment["blocked_reason"] == "payload_evidence_missing" + assert assessment["canonical_phase3_command_log_manifest_eligible"] is False + assert assessment["canonical_phase3_command_log_manifest_reason"] == "not_slurm_direct_ssh_preflight" + assert assessment["component_reports"] == [] + +def test_run_command_builder_orders_runtime_gates_before_payload_unpack() -> None: + remote = _remote_run_command( + direct_run_id="direct-run", + command_id="payload_probe", + remote_zip="/tmp/payload_probe.zip", + remote_root="/shared/bb-p3-root", + image="rocm-image:dev", + hip_visible_devices="0", + ) + phase3_target_fragments = [ + fragment.strip() + for fragment in remote.split(";") + if "PHASE3_TARGET_RUN_ID" in fragment + ] + + assert phase3_target_fragments == ["export PHASE3_TARGET_RUN_ID="] + assert "export PHASE3_TARGET_RUN_ID=direct" not in remote + assert not any("direct-run" in fragment for fragment in phase3_target_fragments) + assert not any("PHASE4_DIRECT_RUN_ID" in fragment for fragment in phase3_target_fragments) + + assert remote.index("direct_runtime_root_missing") < remote.index("WORK=$(mktemp") + assert remote.index("direct_runtime_venv_missing") < remote.index("WORK=$(mktemp") + assert remote.index("direct_runtime_image_missing") < remote.index("WORK=$(mktemp") + assert remote.index("WORK=$(mktemp") < remote.index("python3 -m zipfile -e") + assert remote.index("python3 -m zipfile -e") < remote.index("test -f ./run.sh") + assert remote.index("test -f ./run.sh") < remote.index("bash ./run.sh") + assert "phase3_command_log_manifest.json" not in remote + + +def test_run_mode_scp_failure_writes_payload_transfer_assessment_without_canonical_manifest(tmp_path, monkeypatch) -> None: # noqa: ANN001 + payload = _payload_zip(tmp_path) + output_dir = tmp_path / "out" + calls: list[list[str]] = [] + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202, ARG001 + calls.append(command) + assert command[0] == "scp" + return subprocess.CompletedProcess(command, 23, stdout="", stderr="scp: permission denied\n") + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--command-id", + "run_scp_failure", + "--direct-run-id", + "direct-scp-failure", + "--payload-zip", + str(payload), + "--output-dir", + str(output_dir), + "--mode", + "run", + ] + ) + + assert result == 23 + assert [call[0] for call in calls] == ["scp"] + assert not (output_dir / "phase3_command_log_manifest.json").exists() + assessment = _load_assessment(output_dir, "run_scp_failure") + assert assessment["passed"] is False + assert assessment["status"] == "blocked" + assert assessment["blocked_reason"] == "payload_transfer_failed" + assert assessment["canonical_phase3_command_log_manifest_eligible"] is False + raw_log = output_dir / "command_logs" / "run_scp_failure.log" + assert raw_log.read_text() == "scp: permission denied\n" + assert assessment["raw_log_sha256"] == _sha256_file(raw_log) diff --git a/tests/rl/phase3/test_env_families.py b/tests/rl/phase3/test_env_families.py new file mode 100644 index 00000000..b8f5bcac --- /dev/null +++ b/tests/rl/phase3/test_env_families.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from breadboard.rl.phase3.env_families import run_lean_console_env_probe + +TARGET = "20260623T000000Z-slurm-234555" + + +def test_lean_runtime_absence_fails_claim(tmp_path) -> None: + env = tmp_path / "env.tar"; env.write_text("env") + report = run_lean_console_env_probe(env, target_run_id=TARGET, output_dir=tmp_path / "out") + if report["passed"] is False: + assert report["blocked_reason"] in {"lean_runtime_unavailable", "env_package_missing"} + assert report["scorecard_update_allowed"] is False diff --git a/tests/rl/phase3/test_evidence_gates.py b/tests/rl/phase3/test_evidence_gates.py new file mode 100644 index 00000000..12f9aa56 --- /dev/null +++ b/tests/rl/phase3/test_evidence_gates.py @@ -0,0 +1,128 @@ +from __future__ import annotations +import json + +from pathlib import Path + +from breadboard.rl.phase3.evidence import PHASE3_COMMAND_LOG_MANIFEST_SCHEMA, PHASE3_COMPONENT_REPORT_SCHEMA, sha256_file, validate_phase3_command_log_manifest, validate_phase3_component_report + +TARGET = "20260623T000000Z-slurm-234555" + + +def _manifest(tmp_path: Path, text: str = "ok") -> dict: + raw = tmp_path / "ZYPHRA" / "RL_PHASE_3" / "runs" / "command_logs" / "cmd.log" + raw.parent.mkdir(parents=True) + raw.write_text(text) + return { + "schema_version": PHASE3_COMMAND_LOG_MANIFEST_SCHEMA, + "target_run_id": TARGET, + "commands": [{ + "command_id": "cmd", + "argv": ["python"], + "raw_log_path": "command_logs/cmd.log", + "raw_log_sha256": sha256_file(raw), + "slurm_job_id": "234555", + "target_run_id": TARGET, + "node": "mi300x-1", + "started_at": "t0", + "completed_at": "t1", + "exit_code": 0, + "status": "passed", + }], + } + + +def test_stale_raw_log_hash_rejected(tmp_path: Path) -> None: + manifest = _manifest(tmp_path) + raw = tmp_path / "ZYPHRA" / "RL_PHASE_3" / "runs" / "command_logs" / "cmd.log" + raw.write_text("changed") + errors = validate_phase3_command_log_manifest(manifest, target_run_id=TARGET, repo_root=tmp_path, evidence_root=tmp_path) + assert any("raw_log_sha256" in error for error in errors) + + +def test_missing_slurm_fields_rejected(tmp_path: Path) -> None: + manifest = _manifest(tmp_path) + manifest["commands"][0]["slurm_job_id"] = "" + errors = validate_phase3_command_log_manifest(manifest, target_run_id=TARGET, repo_root=tmp_path, evidence_root=tmp_path) + assert any("slurm_job_id" in error for error in errors) + + +def test_duplicate_command_id_rejected(tmp_path: Path) -> None: + manifest = _manifest(tmp_path) + manifest["commands"].append(dict(manifest["commands"][0])) + errors = validate_phase3_command_log_manifest(manifest, target_run_id=TARGET, repo_root=tmp_path, evidence_root=tmp_path) + assert any("unique" in error for error in errors) + + + +def test_command_manifest_rejects_inline_report_without_passed_true(tmp_path: Path) -> None: + payload = { + "schema_version": PHASE3_COMPONENT_REPORT_SCHEMA, + "component": "gate", + "claim_boundary": "boundary", + "target_run_id": TARGET, + "report_id": "r", + } + manifest = _manifest(tmp_path, f"PHASE3_COMPONENT_REPORT_JSON={json.dumps(payload)}\n") + + errors = validate_phase3_command_log_manifest(manifest, target_run_id=TARGET, repo_root=tmp_path, evidence_root=tmp_path) + + assert "commands[1].inline_reports[1].passed must be true" in errors + + +def test_command_manifest_accepts_legacy_passed_inline_without_component_flag(tmp_path: Path) -> None: + payload = { + "schema_version": PHASE3_COMPONENT_REPORT_SCHEMA, + "component": "gate", + "claim_boundary": "boundary", + "target_run_id": TARGET, + "report_id": "r", + "passed": True, + } + manifest = _manifest(tmp_path, f"PHASE3_COMPONENT_REPORT_JSON={json.dumps(payload)}\n") + + errors = validate_phase3_command_log_manifest(manifest, target_run_id=TARGET, repo_root=tmp_path, evidence_root=tmp_path) + + assert errors == [] + +def test_generic_passed_true_component_rejected(tmp_path: Path) -> None: + report = {"schema_version": PHASE3_COMPONENT_REPORT_SCHEMA, "claim_boundary": "boundary", "target_run_id": TARGET, "passed": True, "scorecard_update_allowed": False, "report_id": "r"} + errors = validate_phase3_component_report(report, expected_schema=PHASE3_COMPONENT_REPORT_SCHEMA, expected_claim_boundary="boundary", target_run_id=TARGET, required_artifact_keys=("artifact",), evidence_root=tmp_path) + assert any("component" in error or "input_hashes" in error or "artifact_paths" in error for error in errors) + + +def test_component_report_with_real_artifact_passes(tmp_path: Path) -> None: + artifact = tmp_path / "artifact.json" + artifact.write_text("{}") + report = { + "schema_version": PHASE3_COMPONENT_REPORT_SCHEMA, + "component": "gate", + "claim_boundary": "boundary", + "target_run_id": TARGET, + "passed": True, + "scorecard_update_allowed": False, + "report_id": "r", + "input_hashes": {"artifact": sha256_file(artifact)}, + "artifact_paths": {"artifact": "artifact.json"}, + } + assert validate_phase3_component_report(report, expected_schema=PHASE3_COMPONENT_REPORT_SCHEMA, expected_claim_boundary="boundary", target_run_id=TARGET, required_artifact_keys=("artifact",), evidence_root=tmp_path) == [] + + + +def test_component_report_rejects_stale_artifact_input_hash(tmp_path: Path) -> None: + artifact = tmp_path / "artifact.json" + artifact.write_text('{"changed": true}\n') + report = { + "schema_version": PHASE3_COMPONENT_REPORT_SCHEMA, + "component": "gate", + "claim_boundary": "boundary", + "target_run_id": TARGET, + "passed": True, + "scorecard_update_allowed": False, + "report_id": "r", + "input_hashes": {"artifact": "sha256:" + "0" * 64}, + "artifact_paths": {"artifact": "artifact.json"}, + } + + errors = validate_phase3_component_report(report, expected_schema=PHASE3_COMPONENT_REPORT_SCHEMA, expected_claim_boundary="boundary", target_run_id=TARGET, required_artifact_keys=("artifact",), evidence_root=tmp_path) + + assert "input_hashes.artifact must match artifact_paths.artifact content sha256" in errors \ No newline at end of file diff --git a/tests/rl/phase3/test_final_report_builder.py b/tests/rl/phase3/test_final_report_builder.py new file mode 100644 index 00000000..1347e7b9 --- /dev/null +++ b/tests/rl/phase3/test_final_report_builder.py @@ -0,0 +1,823 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +from breadboard.rl.phase3.final_report import PHASE3_CORE_CLAIM_BOUNDARY, PHASE3_CORE_READINESS_SCHEMA, PHASE3_FINAL_CLAIM_BOUNDARY, PHASE3_FINAL_REPORT_ID, PHASE3_MILESTONE_BLOCKED_CLAIM_BOUNDARIES, PHASE3_MILESTONE_CLAIM_BOUNDARIES, PHASE3_MILESTONE_POINTS, PHASE3_MILESTONES, build_phase3_core_readiness, build_phase3_final_report, validate_phase3_final_report +from scripts.rl_phase3.build_phase3_final_report import _collect_milestone_reports, _load_json, _scorecard + + +def write_report(path: Path, *, milestone_id: str, report_id: str, passed: bool) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"milestone_id": milestone_id, "report_id": report_id, "passed": passed}) + "\n") + + +def valid_final_report(tmp_path: Path) -> dict: + target = "20260624T040000Z-slurm-243958" + reports = {} + for milestone in PHASE3_MILESTONES: + report_path = tmp_path / f"{milestone}.json" + report_path.write_text("{}") + artifact_hash = "sha256:" + hashlib.sha256(report_path.read_bytes()).hexdigest() + reports[milestone] = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": f"report-{milestone}", + "milestone_id": milestone, + "component": "component", + "claim_boundary": PHASE3_MILESTONE_CLAIM_BOUNDARIES[milestone], + "target_run_id": target, + "points": 1, + "passed": True, + "input_hashes": {"artifact": artifact_hash}, + "artifact_paths": {"artifact": str(report_path)}, + "required_artifact_keys": ["artifact"], + "scorecard_update_allowed": False, + } + harbor_routes = [ + "GET /health", + "GET /metrics.json", + "GET /list_tasks", + "POST /score", + "POST /trial/create", + "POST /trial/{trial_id}/exec", + "GET /trial/{trial_id}", + "POST /trial/{trial_id}/finalize", + ] + reports["P3-M9"]["claim_boundary"] = PHASE3_MILESTONE_CLAIM_BOUNDARIES["P3-M9"] + reports["P3-M9"]["provider_kind"] = "harbor_facade" + reports["P3-M9"]["provider_report"] = { + "schema_version": "bb.rl.phase3.harbor_service_proof.v1", + "report_id": "phase3_harbor_service_proof", + "claim_boundary": "phase3_harbor_nemo_gym_named_endpoint_scope", + "target_run_id": target, + "attestation_backend": "harbor_facade", + "provider_kind": "harbor_facade", + "endpoint_identity": "https://harbor.example", + "backend_identity": "harbor-test", + "env_package_sha256": "sha256:env", + "task_name": "phase3-harbor-smoke", + "trial_id_sha256": "sha256:trial", + "harbor_routes": harbor_routes, + "harbor_calls": [ + { + "method": route.split(" ", 1)[0], + "route_template": route, + "path": route.split(" ", 1)[1], + "status_code": 200, + "request_sha256": "sha256:req", + "response_sha256": "sha256:resp", + "latency_seconds": 0.1, + "passed": True, + "blocked_reason": "", + } + for route in harbor_routes + ], + "passed": True, + "scorecard_update_allowed": False, + } + reports["P3-M11"]["observability_evidence"] = { + "passed": True, + "errors": [], + "verifier_latency": 0.1, + "metric_sections": { + "verifier_metrics": { + "source": "verifier_client", + "endpoint": "https://verifier.example/ready", + "env_presence": {"BREADBOARD_VERIFIER_TOKEN": True}, + }, + "object_store_metrics": { + "source": "object_store", + "object_store": "configured_http_object_store", + "object_store_writes": 1, + "artifact_bytes": 128, + "written_sha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "readback_sha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "endpoint": "https://object-store.example", + "put_endpoint": "https://object-store.example/objects/phase3-put", + "get_endpoint": "https://object-store.example/objects/phase3-get", + "delete_endpoint": "https://object-store.example/objects/phase3-delete", + "put_status": 201, + "get_status": 200, + "delete_status": 204, + "write_read_verified": True, + "delete_verified": True, + "env_presence": { + "BREADBOARD_OBJECT_STORE_BASE_URL": True, + "BREADBOARD_OBJECT_STORE_BUCKET": True, + "BREADBOARD_OBJECT_STORE_TOKEN": True, + "BREADBOARD_OBJECT_STORE_PUT_URL_TEMPLATE": True, + "BREADBOARD_OBJECT_STORE_GET_URL_TEMPLATE": True, + "BREADBOARD_OBJECT_STORE_DELETE_URL_TEMPLATE": True, + }, + }, + "scheduler_metrics": { + "source": "scheduler_control", + "scheduler_control": { + "endpoint_present": True, + "token_present": True, + }, + }, + }, + } + reports["P3-M8"]["provider_kind"] = "none" + reports["P3-M8"]["retirement_accepted"] = True + reports["P3-M8"]["rubric_decision"] = "phase3_p3m8_provider_milestone_retired_and_accepted_20260707" + reports["P3-M8"]["provider_report"] = { + "schema_version": "bb.rl.phase3.retired_provider_milestone.v1", + "report_id": "phase3_p3m8_retired_provider_milestone_accepted", + "claim_boundary": PHASE3_MILESTONE_CLAIM_BOUNDARIES["P3-M8"], + "target_run_id": target, + "provider_kind": "none", + "retirement_accepted": True, + "rubric_decision": "phase3_p3m8_provider_milestone_retired_and_accepted_20260707", + "passed": True, + "scorecard_update_allowed": False, + } + runs_root = tmp_path / "ZYPHRA" / "RL_PHASE_3" / "runs" + command_log = runs_root / "command_logs" / "cmd.log" + command_log.parent.mkdir(parents=True, exist_ok=True) + command_log.write_text("ok") + cmd_sha = "sha256:" + hashlib.sha256(command_log.read_bytes()).hexdigest() + parity_artifacts = {} + for name in ( + "reward_function", + "accepted_projection_rows", + "evidence_manifest", + "metrics", + "introspection_report", + "runtime_ppo_script", + "runtime_grpo_script", + "runtime_closed_loop_script", + "runtime_install_report", + ): + path = tmp_path / "parity" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("ok\n") + parity_artifacts[name] = str(path) + parity_path = tmp_path / "parity" / "phase3_parity_report.json" + parity_payload = { + "schema_version": "bb.rl.phase3.parity_report.v1", + "report_id": "phase3_parity_report", + "claim_boundary": "phase3_ppo_grpo_closed_loop_parity_named_scope", + "target_run_id": target, + "scorecard_update_allowed": False, + "passed": True, + "scorer": {}, + "rollout": {"rollout_name": "vllm"}, + "token_logprob": {}, + "checkpoint": { + key: { + "optimizer_step_count": 1, + "checkpoint_changed": True, + "checkpoint_before_sha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "checkpoint_after_sha256": "sha256:after", + "device_count": 8, + "n_gpus_per_node": 8, + } + for key in ("ppo", "grpo", "closed_loop") + }, + "model_merge": {}, + "infra": { + "runtime_evidence": {"container_image": "vllm/vllm-openai-rocm:nightly", "runtime_path": "/shared/bb-p3-root/phase3_vllm_verl_py312"}, + "introspection": { + "verl_version": "0.8.0", + "torch_version": "2.9.1+rocm6.4", + "cuda_available": True, + "device_count": 8, + "devices": ["AMD Instinct MI300X"] * 8, + }, + }, + "dataproto": {"dataproto_ok": True}, + "limitations": [], + "checklist": { + "C7_checkpoint_parity": {"status": "satisfied"}, + "C10_infrastructure_parity": {"status": "satisfied"}, + }, + "artifact_paths": parity_artifacts, + "errors": [], + } + parity_path.write_text(json.dumps(parity_payload) + "\n") + parity_sha = "sha256:" + hashlib.sha256(parity_path.read_bytes()).hexdigest() + for milestone in ("P3-M2", "P3-M3", "P3-M4"): + reports[milestone]["artifact_paths"]["parity_report"] = str(parity_path) + reports[milestone]["input_hashes"]["parity_report"] = parity_sha + reports[milestone]["parity_report_id"] = "phase3_parity_report" + return build_phase3_final_report( + target_run_id=target, + milestone_reports=reports, + command_log_manifest={ + "schema_version": "bb.rl.phase3.command_log_manifest.v1", + "target_run_id": target, + "commands": [ + { + "command_id": "cmd", + "argv": ["x"], + "raw_log_path": "command_logs/cmd.log", + "raw_log_sha256": cmd_sha, + "slurm_job_id": "1", + "target_run_id": target, + "node": "n", + "started_at": "a", + "completed_at": "b", + "exit_code": 0, + "status": "passed", + } + ], + }, + scorecard={"reviewed_final_report_id": PHASE3_FINAL_REPORT_ID}, + claim_ledger_text=f"{target}\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n", + ) + + +def _replace_summary_from_report(report: dict, milestone_id: str) -> None: + component = report["milestone_reports"][milestone_id] + for summary in report["milestone_summaries"]: + if summary["milestone_id"] == milestone_id: + summary.update( + { + "report_id": component.get("report_id"), + "schema_version": component.get("schema_version"), + "claim_boundary": component.get("claim_boundary"), + "passed": component.get("passed") is True, + "blocked_reason": component.get("blocked_reason", ""), + "claim_ready": component.get("passed") is True and not component.get("blocked_reason"), + "scorecard_update_allowed": component.get("scorecard_update_allowed"), + } + ) + return + raise AssertionError(f"{milestone_id} summary missing") + + +def _defer_milestone(report: dict, milestone_id: str, blocked_reason: str) -> None: + component = report["milestone_reports"][milestone_id] + component["passed"] = False + component["blocked_reason"] = blocked_reason + component["points"] = 0 + component["claim_boundary"] = PHASE3_MILESTONE_BLOCKED_CLAIM_BOUNDARIES.get(milestone_id, PHASE3_MILESTONE_CLAIM_BOUNDARIES[milestone_id]) + component["scorecard_update_allowed"] = False + _replace_summary_from_report(report, milestone_id) + + +def test_final_report_embeds_core_readiness_without_promoting_scorecard(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + + core = report["core_readiness"] + + assert core["schema_version"] == PHASE3_CORE_READINESS_SCHEMA + assert core["claim_boundary"] == PHASE3_CORE_CLAIM_BOUNDARY + assert core["ready"] is True + assert core["report_label"] == "active-artifact-audit-clean" + assert core["artifact_audit_clean"] is True + assert core["ready_meaning"] == "Existing artifacts satisfy the promoted exact-scope Phase 3 boundary; broader or successor claims require separate canonical promotion." + assert core["scorecard_update_allowed"] is False + assert core["retired_milestones"] == [] + assert "P3-M8" in core["active_milestones"] + assert "P3-M9" in core["active_milestones"] + assert "P3-M11" in core["active_milestones"] + assert "P3-M11" in {status["milestone_id"] for status in core["milestone_statuses"]} + assert core["blocked_active_milestones"] == [] + assert report["scorecard_update_allowed"] is False + + + +def test_final_report_active_scope_accounts_for_milestone_points(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + core = report["core_readiness"] + expected_total = sum(PHASE3_MILESTONE_POINTS[milestone_id] for milestone_id in core["core_milestones"]) + + assert report["active_scope"]["core_raw_points_total"] == expected_total + assert report["active_scope"]["core_raw_points_verified"] == expected_total + assert core["core_raw_points_total"] == expected_total + assert core["core_raw_points_verified"] == expected_total + + _defer_milestone(report, "P3-M11", "missing_live_observability_scheduler_store") + blocked_core = build_phase3_core_readiness(report["milestone_reports"]) + + assert blocked_core["core_raw_points_total"] == expected_total + assert blocked_core["core_raw_points_verified"] == expected_total - PHASE3_MILESTONE_POINTS["P3-M11"] + +def test_final_report_core_readiness_blocks_when_a_milestone_is_deferred(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + _defer_milestone(report, "P3-M11", "missing_live_observability_scheduler_store") + report["core_readiness"] = build_phase3_core_readiness(report["milestone_reports"]) + report["active_scope"] = report["core_readiness"] + + core = report["core_readiness"] + p3m11_summary = next(summary for summary in report["milestone_summaries"] if summary["milestone_id"] == "P3-M11") + + assert report["milestone_reports"]["P3-M11"]["passed"] is False + assert p3m11_summary["passed"] is False + assert core["ready"] is False + assert core["report_label"] == "active-artifact-audit-blocked" + assert core["artifact_audit_clean"] is False + assert core["deferred_milestones"] == [] + assert "P3-M11" in core["core_milestones"] + assert "P3-M11" in core["blocked_core_milestones"] + + +def test_final_report_core_readiness_blocks_p3m7_fixture_benchmark_credit(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + report["milestone_reports"]["P3-M7"]["benchmark_report"] = {"source_pin": {"benchmark_version": "fixture-v2"}} + report["core_readiness"] = build_phase3_core_readiness(report["milestone_reports"]) + report["active_scope"] = report["core_readiness"] + + core = report["core_readiness"] + p3m7 = next(status for status in core["milestone_statuses"] if status["milestone_id"] == "P3-M7") + + assert core["ready"] is False + assert core["report_label"] == "active-artifact-audit-blocked" + assert core["blocked_active_milestones"] == ["P3-M7"] + assert p3m7["blocker"] == "fixture_benchmark_not_external_core_credit" + assert validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) == [] + + + +def test_final_report_core_readiness_blocks_hand_written_benchmark_candidates(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + report["milestone_reports"]["P3-M7"]["benchmark_report"] = { + "prompt_solution_leakage_scan": { + "candidate_source": "hand_written_baseline_in_target_payload", + } + } + report["core_readiness"] = build_phase3_core_readiness(report["milestone_reports"]) + report["active_scope"] = report["core_readiness"] + + core = report["core_readiness"] + p3m7 = next(status for status in core["milestone_statuses"] if status["milestone_id"] == "P3-M7") + + assert core["ready"] is False + assert core["blocked_active_milestones"] == ["P3-M7"] + assert p3m7["blocker"] == "benchmark_candidate_not_phase3_model_pipeline" + assert validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) == [ + "P3-M7 benchmark candidate source must come from the Phase 3 model pipeline" + ] + +def test_collect_milestone_reports_prefers_canonical_directory(tmp_path: Path) -> None: + runs = tmp_path / "runs" + write_report(runs / "live_provider_reports" / "P3-M8_retired_provider_milestone.json", milestone_id="P3-M8", report_id="auxiliary", passed=True) + write_report(runs / "milestone_reports" / "P3-M8_retired_provider_milestone.json", milestone_id="P3-M8", report_id="canonical", passed=False) + + reports = _collect_milestone_reports(runs) + + assert reports["P3-M8"]["report_id"] == "canonical" + assert reports["P3-M8"]["passed"] is False + +def test_collect_milestone_reports_ignores_unknown_milestone_ids(tmp_path: Path) -> None: + runs = tmp_path / "runs" + write_report(runs / "milestone_reports" / "stray.json", milestone_id="P3-M99", report_id="stray", passed=True) + write_report(runs / "milestone_reports" / "P3-M8_retired_provider_milestone.json", milestone_id="P3-M8", report_id="canonical", passed=False) + + reports = _collect_milestone_reports(runs) + + assert list(reports) == ["P3-M8"] + assert reports["P3-M8"]["report_id"] == "canonical" + + +def test_final_report_script_ignores_non_object_json_inputs(tmp_path: Path) -> None: + runs = tmp_path / "runs" + (runs / "milestone_reports").mkdir(parents=True) + (runs / "phase3_command_log_manifest.json").write_text("[]") + (runs / "milestone_reports" / "array.json").write_text("[]") + (runs / "milestone_reports" / "object_milestone.json").write_text(json.dumps({"milestone_id": ["P3-M9"], "passed": True})) + write_report(runs / "milestone_reports" / "P3-M8_retired_provider_milestone.json", milestone_id="P3-M8", report_id="canonical", passed=True) + + reports = _collect_milestone_reports(runs) + + assert _load_json(runs / "phase3_command_log_manifest.json") == {} + assert list(reports) == ["P3-M8"] + assert reports["P3-M8"]["report_id"] == "canonical" + +def test_final_report_cli_handles_directory_inputs(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + runs = phase_dir / "runs" + runs.mkdir(parents=True) + (runs / "phase3_command_log_manifest.json").mkdir() + (phase_dir / "BB_ZYPHRA_RL_PHASE_3_CLAIM_LEDGER.md").mkdir() + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/build_phase3_final_report.py", + "--phase-dir", + str(phase_dir), + ], + check=False, + cwd=Path(__file__).resolve().parents[3], + text=True, + capture_output=True, + ) + + assert result.returncode == 0 + report = json.loads((runs / "p3_m12_final_report.json").read_text()) + assert { + "schema_version must be bb.rl.phase3.command_log_manifest.v1", + "target_run_id must match Phase 3 Slurm target run id pattern", + "commands must contain at least one command row", + }.issubset(set(report["validation_errors"])) + assert report["command_log_manifest"] == {} + +def test_final_report_cli_require_ready_rejects_schema_valid_incomplete_active_scope(tmp_path: Path) -> None: + evidence_root = tmp_path / "docs_tmp" + phase_dir = evidence_root / "ZYPHRA" / "RL_PHASE_3" + runs = phase_dir / "runs" + milestone_dir = runs / "milestone_reports" + milestone_dir.mkdir(parents=True) + report = valid_final_report(evidence_root) + _defer_milestone(report, "P3-M11", "missing_live_observability_scheduler_store") + for milestone_id, milestone_report in report["milestone_reports"].items(): + (milestone_dir / f"{milestone_id}.json").write_text(json.dumps(milestone_report, sort_keys=True) + "\n") + (runs / "phase3_command_log_manifest.json").write_text(json.dumps(report["command_log_manifest"], sort_keys=True) + "\n") + (phase_dir / "BB_ZYPHRA_RL_PHASE_3_CLAIM_LEDGER.md").write_text(report["claim_ledger_text"]) + (phase_dir / "BB_ZYPHRA_RL_PHASE_3_SCORECARD.yaml").write_text(f"reviewed_final_report_id: {PHASE3_FINAL_REPORT_ID}\n") + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/build_phase3_final_report.py", + "--phase-dir", + str(phase_dir), + "--require-ready", + ], + check=False, + cwd=Path(__file__).resolve().parents[3], + text=True, + capture_output=True, + ) + + assert result.returncode == 1 + cli_payload = json.loads(result.stdout) + assert cli_payload["validation_errors"] == [] + assert cli_payload["readiness_errors"] == [ + "active_scope.ready must be true for --require-ready", + "core_raw_points_verified must equal core_raw_points_total for --require-ready", + ] + generated_report = json.loads((runs / "p3_m12_final_report.json").read_text()) + assert generated_report["validation_errors"] == [] + assert generated_report["active_scope"]["ready"] is False + assert generated_report["active_scope"]["core_raw_points_verified"] != generated_report["active_scope"]["core_raw_points_total"] + + +def test_scorecard_malformed_total_points_defaults_to_zero(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + phase_dir.mkdir(parents=True) + (phase_dir / "BB_ZYPHRA_RL_PHASE_3_SCORECARD.yaml").write_text( + "current_verified_points: 1000\n" + "total_points: not-a-number\n" + f"reviewed_final_report_id: {PHASE3_FINAL_REPORT_ID}\n" + ) + + scorecard = _scorecard(phase_dir) + + assert scorecard["current_verified_points"] == 1000 + assert scorecard["total_points"] == 0 + + +def test_final_report_validator_rejects_wrong_schema(tmp_path: Path) -> None: + report = build_phase3_final_report( + target_run_id="20260624T040000Z-slurm-243958", + milestone_reports={}, + command_log_manifest={}, + scorecard={}, + claim_ledger_text="", + ) + report["schema_version"] = "stale" + + assert "schema_version must be Phase 3 final report schema" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + +def test_final_report_validator_requires_final_report_id_in_ledger(tmp_path: Path) -> None: + report = build_phase3_final_report( + target_run_id="20260624T040000Z-slurm-243958", + milestone_reports={}, + command_log_manifest={}, + scorecard={}, + claim_ledger_text=f"20260624T040000Z-slurm-243958\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n", + ) + + assert "claim ledger must contain target_run_id, final report id, and final claim boundary" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + +def test_final_report_builder_coerces_non_mapping_inputs() -> None: + report = build_phase3_final_report( + target_run_id="20260624T040000Z-slurm-243958", + milestone_reports=[], # type: ignore[arg-type] + command_log_manifest=[], # type: ignore[arg-type] + scorecard=[], # type: ignore[arg-type] + claim_ledger_text={"bad": "container"}, # type: ignore[arg-type] + ) + + assert report["milestone_reports"] == {} + assert report["command_log_manifest"] == {} + assert report["scorecard"] == {} + assert report["claim_ledger_text"] == "" + + +def test_final_report_validator_rejects_non_string_claim_ledger(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + report["claim_ledger_text"] = {"bad": "container"} + + assert "claim ledger must contain target_run_id, final report id, and final claim boundary" in validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + +def test_valid_final_report_fixture_has_no_errors(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + + assert validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) == [] + + +def test_final_report_validator_rejects_mutated_milestone_claim_boundary(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + report["milestone_reports"]["P3-M8"]["claim_boundary"] = "phase3_wrong_scope" + + assert ( + "P3-M8 claim_boundary must be phase3_retired_provider_milestone_accepted_scope" + in validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + ) + + + +def test_final_report_validator_rejects_missing_p3m9_provider_kind(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + del report["milestone_reports"]["P3-M9"]["provider_kind"] + + assert "P3-M9 provider_kind must be harbor_facade for the active Harbor/NeMo Gym path" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + +def test_final_report_validator_rejects_p3m9_native_benchflow_backend(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + report["milestone_reports"]["P3-M9"]["provider_report"]["attestation_backend"] = "native_benchflow" + + errors = validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + assert "P3-M9 provider_report.attestation_backend must be harbor_facade for the active Harbor/NeMo Gym path" in errors + assert "P3-M9 native BenchFlow evidence is contract-only and cannot satisfy the active Harbor/NeMo Gym milestone" in errors + + +def test_final_report_validator_rejects_missing_p3m9_report_backend(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + del report["milestone_reports"]["P3-M9"]["provider_report"]["attestation_backend"] + + assert "P3-M9 provider_report.attestation_backend must be harbor_facade for the active Harbor/NeMo Gym path" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + +def test_final_report_validator_rejects_p3m9_missing_trial_hash(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + del report["milestone_reports"]["P3-M9"]["provider_report"]["trial_id_sha256"] + + assert "P3-M9 harbor provider_report.trial_id_sha256 must be present" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + +def test_final_report_validator_rejects_p3m9_harbor_missing_finalize_route(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + p3m9 = report["milestone_reports"]["P3-M9"] + p3m9["provider_report"]["harbor_routes"] = [ + "GET /health", + "GET /metrics.json", + "POST /trial/create", + "POST /trial/{trial_id}/exec", + ] + + assert "P3-M9 harbor provider_report.harbor_routes must match the Harbor service proof route sequence" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + + +def test_final_report_validator_accepts_p3m9_harbor_facade_routes(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + + assert validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) == [] + + +def test_final_report_validator_rejects_p3m11_observability_errors(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + report["milestone_reports"]["P3-M11"]["observability_evidence"]["errors"] = ["production_object_store_endpoint_missing"] + + assert "P3-M11 observability_evidence.errors must be empty" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + +def test_final_report_validator_rejects_p3m11_local_object_store_backend(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + object_store_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["object_store_metrics"] + object_store_metrics["object_store"] = "local_object_store" + + assert "P3-M11 object_store_metrics.object_store must be a production object-store backend" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + +def test_final_report_validator_rejects_p3m11_target_workspace_local_object_store_backend(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + object_store_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["object_store_metrics"] + object_store_metrics["object_store"] = "target_workspace_local_object_store" + + assert "P3-M11 object_store_metrics.object_store must be a production object-store backend" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + +def test_final_report_validator_rejects_p3m11_local_object_store_class_backend(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + object_store_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["object_store_metrics"] + object_store_metrics["object_store"] = "LocalObjectStore" + + assert "P3-M11 object_store_metrics.object_store must be a production object-store backend" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + +def test_final_report_validator_rejects_p3m11_local_metric_urls(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + metric_sections = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"] + metric_sections["verifier_metrics"] = {"endpoint": "http://localhost:8080/verifier"} + metric_sections["object_store_metrics"].update( + { + "endpoint": "http://127.0.0.1:9000", + "put_endpoint": "http://127.0.0.1:9000/put", + "get_endpoint": "http://127.0.0.1:9000/get", + "delete_endpoint": "http://127.0.0.1:9000/delete", + } + ) + metric_sections["scheduler_metrics"]["scheduler_control"]["endpoint"] = "http://localhost:8081/scheduler" + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert { + "P3-M11 verifier_metrics.endpoint must not be local", + "P3-M11 object_store_metrics.endpoint must not be local", + "P3-M11 object_store_metrics.put_endpoint must not be local", + "P3-M11 object_store_metrics.get_endpoint must not be local", + "P3-M11 object_store_metrics.delete_endpoint must not be local", + "P3-M11 scheduler_metrics.scheduler_control.endpoint must not be local", + }.issubset(errors) + + +def test_final_report_validator_rejects_p3m11_node_hostname_metric_url(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("SLURMD_NODENAME", "cnode-143") + report = valid_final_report(tmp_path) + object_store_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["object_store_metrics"] + object_store_metrics["endpoint"] = "https://cnode-143:9000" + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert "P3-M11 object_store_metrics.endpoint must not be local" in errors + + +def test_final_report_validator_rejects_p3m11_schemeless_local_metric_url(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + object_store_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["object_store_metrics"] + object_store_metrics["endpoint"] = "127.0.0.1:9000" + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert "P3-M11 object_store_metrics.endpoint must not be local" in errors + + +def test_final_report_validator_rejects_p3m11_missing_scheduler_endpoint(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + scheduler_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["scheduler_metrics"] + scheduler_metrics["scheduler_control"]["endpoint_present"] = False + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert "P3-M11 observability_evidence.metric_sections.scheduler_metrics.scheduler_control_endpoint_missing" in errors + + +def test_final_report_validator_rejects_p3m11_missing_scheduler_token(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + scheduler_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["scheduler_metrics"] + scheduler_metrics["scheduler_control"]["token_present"] = False + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert "P3-M11 observability_evidence.metric_sections.scheduler_metrics.scheduler_control_token_missing" in errors + + + +def test_final_report_validator_rejects_p3m11_missing_object_store_token_proof(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + object_store_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["object_store_metrics"] + object_store_metrics["env_presence"]["BREADBOARD_OBJECT_STORE_TOKEN"] = False + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert any("object_store" in error and "token" in error.lower() for error in errors) + + +def test_final_report_validator_rejects_p3m11_missing_write_read_round_trip_proof(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + object_store_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["object_store_metrics"] + object_store_metrics.pop("write_read_verified") + object_store_metrics.pop("readback_sha256") + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert any("write_read_verified" in error for error in errors) + assert any("readback_sha256" in error or "round" in error.lower() for error in errors) + + +def test_final_report_validator_rejects_p3m11_legacy_artifact_hash_without_readback_hash(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + object_store_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["object_store_metrics"] + object_store_metrics["artifact_sha256"] = object_store_metrics["written_sha256"] + object_store_metrics.pop("readback_sha256") + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert any("readback_sha256" in error for error in errors) + + +def test_final_report_validator_rejects_p3m11_failed_delete_proof(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + object_store_metrics = report["milestone_reports"]["P3-M11"]["observability_evidence"]["metric_sections"]["object_store_metrics"] + object_store_metrics["delete_status"] = 500 + object_store_metrics["delete_verified"] = False + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert any("delete" in error.lower() for error in errors) + +def test_final_report_validator_rejects_misfiled_milestone_id(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + report["milestone_reports"]["P3-M10"]["milestone_id"] = "P3-M9" + + assert "P3-M10 report milestone_id must match outer milestone key" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + +def test_final_report_validator_rejects_unknown_milestone_report_key(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + report["milestone_reports"]["P3-M99"] = dict(report["milestone_reports"]["P3-M0"]) + + assert "P3-M99 report is not a Phase 3 milestone" in validate_phase3_final_report( + report, + repo_root=tmp_path, + evidence_root=tmp_path, + ) + + +def test_final_report_validator_rejects_bad_parity_artifact_on_disk(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + parity_path = Path(report["milestone_reports"]["P3-M2"]["artifact_paths"]["parity_report"]) + parity_payload = json.loads(parity_path.read_text()) + parity_payload["infra"]["introspection"]["devices"] = ["AMD Instinct MI300X"] * 7 + parity_path.write_text(json.dumps(parity_payload) + "\n") + bad_sha = "sha256:" + hashlib.sha256(parity_path.read_bytes()).hexdigest() + for milestone in ("P3-M2", "P3-M3", "P3-M4"): + report["milestone_reports"][milestone]["input_hashes"]["parity_report"] = bad_sha + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert "parity_report: infra.introspection.devices must list 8 AMD Instinct MI300X devices" in errors + + +def test_final_report_validator_rejects_stale_parity_hash(tmp_path: Path) -> None: + report = valid_final_report(tmp_path) + parity_path = Path(report["milestone_reports"]["P3-M2"]["artifact_paths"]["parity_report"]) + parity_payload = json.loads(parity_path.read_text()) + parity_payload["limitations"].append("changed after milestone snapshot") + parity_path.write_text(json.dumps(parity_payload) + "\n") + + errors = validate_phase3_final_report(report, repo_root=tmp_path, evidence_root=tmp_path) + + assert "parity_report input hash must match artifact content" in errors diff --git a/tests/rl/phase3/test_harbor_local_lifecycle.py b/tests/rl/phase3/test_harbor_local_lifecycle.py new file mode 100644 index 00000000..e9781aab --- /dev/null +++ b/tests/rl/phase3/test_harbor_local_lifecycle.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from typing import Any +from urllib.parse import urlparse + +from scripts.rl_phase3.run_phase3_harbor_local_lifecycle import build_lifecycle_report + + +def _passing_transport(url: str, method: str, payload: Any, timeout_s: float) -> tuple[int, Any]: + path = urlparse(url).path + if path == "/health": + return 200, {"ok": True, "n_tasks": 1} + if path == "/metrics.json": + return 200, {"requests_total": 1} + if path == "/list_tasks": + return 200, ["task.alpha"] + if path == "/score" and method == "POST": + return 200, {"reward": 1.0, "raw": {"reward": 1.0}, "error": None} + if path == "/trial/create" and method == "POST": + name = "trial-delete" if payload and payload.get("task_name") == "task.delete" else "trial-main" + return 200, {"trial_id": name, "task_name": payload["task_name"], "instruction": "solve", "expires_at_ts": 1.0} + if path == "/trial/trial-main/exec" and method == "POST": + return 200, {"stdout": "", "stderr": "", "return_code": 0, "stdout_truncated": False, "stderr_truncated": False, "duration_sec": 0.01} + if path == "/trial/trial-main" and method == "GET": + return 200, {"trial_id": "trial-main", "task_name": "task.alpha", "n_exec_calls": 1, "finalized": False} + if path == "/trial_stats": + return 200, {"n_active": 1} + if path == "/trial/trial-main/finalize" and method == "POST": + return 200, {"reward": 1.0, "raw": {"reward": 1.0}, "error": None} + if path == "/trial/trial-delete" and method == "DELETE": + return 200, {"ok": True} + return 404, {"error": f"unexpected {method} {path}"} + + +def test_harbor_local_lifecycle_passes_as_non_promotional_local_report() -> None: + calls: list[tuple[str, str, Any]] = [] + + def transport(url: str, method: str, payload: Any, timeout_s: float) -> tuple[int, Any]: + calls.append((method, urlparse(url).path, payload)) + if urlparse(url).path == "/trial/create" and len([call for call in calls if call[1] == "/trial/create"]) == 2: + return 200, {"trial_id": "trial-delete", "task_name": payload["task_name"], "instruction": "solve", "expires_at_ts": 1.0} + return _passing_transport(url, method, payload, timeout_s) + + report = build_lifecycle_report( + base_url="http://127.0.0.1:5050", + target_run_id="local-diagnostic", + task_name=None, + answer="ok", + exec_cmd="true", + timeout_s=1.0, + transport=transport, + ) + + assert report["passed"] is True + assert report["promotional"] is False + assert report["scorecard_update_allowed"] is False + assert report["claim_boundary"] == "local_harbor_api_lifecycle_only" + assert report["target_endpoint_proven"] is False + assert report["auth_proven"] is False + assert report["docker_execution_proven"] is True + assert report["base_url_identity"]["is_loopback"] is True + assert report["selected_task"] == "task.alpha" + assert report["route_statuses"]["trial_delete"] == "passed" + assert all(route.get("response") is None for route in report["routes"].values() if route) + + +def test_harbor_local_lifecycle_blocks_when_health_fails() -> None: + def transport(url: str, method: str, payload: Any, timeout_s: float) -> tuple[int, Any]: + if urlparse(url).path == "/health": + return 503, {"ok": False} + return _passing_transport(url, method, payload, timeout_s) + + report = build_lifecycle_report( + base_url="http://127.0.0.1:5050", + target_run_id="local-diagnostic", + task_name="task.alpha", + answer="ok", + exec_cmd="true", + timeout_s=1.0, + transport=transport, + ) + + assert report["passed"] is False + assert report["route_statuses"]["health"] == "failed" + assert "health" in report["blocked_reason"] + assert report["scorecard_update_allowed"] is False + + +def test_harbor_local_lifecycle_blocks_without_tasks() -> None: + def transport(url: str, method: str, payload: Any, timeout_s: float) -> tuple[int, Any]: + if urlparse(url).path == "/list_tasks": + return 200, [] + return _passing_transport(url, method, payload, timeout_s) + + report = build_lifecycle_report( + base_url="http://127.0.0.1:5050", + target_run_id="local-diagnostic", + task_name=None, + answer="ok", + exec_cmd="true", + timeout_s=1.0, + transport=transport, + ) + + assert report["passed"] is False + assert report["selected_task"] == "" + assert report["route_statuses"]["score"] == "skipped" + assert report["route_statuses"]["trial_create"] == "skipped" + assert "score" in report["blocked_reason"] + + +def test_harbor_local_lifecycle_blocks_on_trial_create_failure() -> None: + def transport(url: str, method: str, payload: Any, timeout_s: float) -> tuple[int, Any]: + if urlparse(url).path == "/trial/create": + return 500, {"error": "trial open failed"} + return _passing_transport(url, method, payload, timeout_s) + + report = build_lifecycle_report( + base_url="http://127.0.0.1:5050", + target_run_id="local-diagnostic", + task_name="task.alpha", + answer="ok", + exec_cmd="true", + timeout_s=1.0, + transport=transport, + ) + + assert report["passed"] is False + assert report["route_statuses"]["trial_create"] == "failed" + assert report["route_statuses"]["trial_exec"] == "skipped" + assert report["route_statuses"]["trial_delete"] == "skipped" + assert report["target_endpoint_proven"] is False diff --git a/tests/rl/phase3/test_live_integrations.py b/tests/rl/phase3/test_live_integrations.py new file mode 100644 index 00000000..1e02a9d3 --- /dev/null +++ b/tests/rl/phase3/test_live_integrations.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +import json +import hashlib +import http.server +import socket +import socketserver +import threading +from pathlib import Path + +from breadboard.rl.phase3.integrations import call_ors_openreward, run_benchflow_harbor_attestation, run_harbor_service_proof, run_live_verifier_campaign + +TARGET = "20260623T000000Z-slurm-234555" + + +class Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self): # noqa: N802 + length = int(self.headers.get("content-length", "0")) + self.rfile.read(length) + self.send_response(200) + self.end_headers() + self.wfile.write(b'{"ok":true}') + + def log_message(self, *args): + pass + + +def server(): + srv = socketserver.TCPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=srv.serve_forever, daemon=True) + thread.start() + return srv + + +def test_provider_env_absence_blocks(monkeypatch) -> None: + monkeypatch.delenv("BREADBOARD_ORS_BASE_URL", raising=False) + monkeypatch.delenv("BREADBOARD_ORS_TOKEN", raising=False) + monkeypatch.delenv("BREADBOARD_OPENREWARD_BASE_URL", raising=False) + monkeypatch.delenv("BREADBOARD_OPENREWARD_TOKEN", raising=False) + report = run_live_verifier_campaign([{"id": 1}], target_run_id=TARGET) + assert report["passed"] is False + assert report["blocked_reason"] == "missing_live_provider_credentials" + + +def test_campaign_preserves_unsafe_endpoint_reason(monkeypatch) -> None: + monkeypatch.setenv("BREADBOARD_ORS_BASE_URL", "https://127.0.0.1:8443") + monkeypatch.setenv("BREADBOARD_ORS_TOKEN", "token") + monkeypatch.setenv("BREADBOARD_OPENREWARD_BASE_URL", "https://127.0.0.1:9443") + monkeypatch.setenv("BREADBOARD_OPENREWARD_TOKEN", "token") + report = run_live_verifier_campaign([{"id": 1}], target_run_id=TARGET) + assert report["passed"] is False + assert report["blocked_reason"] == "provider_endpoint_must_not_be_loopback_or_private" + +def test_campaign_rejects_dns_resolved_loopback(monkeypatch) -> None: + monkeypatch.setenv("BREADBOARD_ORS_BASE_URL", "https://provider.example.test") + monkeypatch.setenv("BREADBOARD_ORS_TOKEN", "token") + monkeypatch.setenv("BREADBOARD_OPENREWARD_BASE_URL", "https://provider.example.test") + monkeypatch.setenv("BREADBOARD_OPENREWARD_TOKEN", "token") + monkeypatch.setattr( + "breadboard.rl.phase3.integrations.socket.getaddrinfo", + lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("127.0.0.1", 443))], + ) + report = run_live_verifier_campaign([{"id": 1}], target_run_id=TARGET) + assert report["passed"] is False + assert report["blocked_reason"] == "provider_endpoint_must_not_be_loopback_or_private" + +def test_provider_calls_disable_proxy_handlers(monkeypatch) -> None: + seen = {} + + class FakeOpener: + def open(self, request, timeout): # noqa: ANN001 + raise RuntimeError("stop-before-network") + + def fake_build_opener(*handlers): # noqa: ANN001 + seen["proxies"] = getattr(handlers[0], "proxies", None) + return FakeOpener() + + monkeypatch.setenv("BREADBOARD_ORS_BASE_URL", "https://provider.example.test") + monkeypatch.setenv("BREADBOARD_ORS_TOKEN", "token") + monkeypatch.setenv("HTTPS_PROXY", "http://127.0.0.1:9999") + monkeypatch.setattr( + "breadboard.rl.phase3.integrations.socket.getaddrinfo", + lambda *args, **kwargs: [(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("93.184.216.34", 443))], + ) + monkeypatch.setattr("breadboard.rl.phase3.integrations.urllib.request.build_opener", fake_build_opener) + evidence = call_ors_openreward({"id": 1}, provider="ors", timeout_s=1) + assert evidence.blocked_reason == "RuntimeError" + assert seen["proxies"] == {} + +def test_loopback_http_provider_is_blocked(monkeypatch) -> None: + srv = server() + monkeypatch.setenv("BREADBOARD_ORS_BASE_URL", f"http://127.0.0.1:{srv.server_address[1]}") + monkeypatch.setenv("BREADBOARD_ORS_TOKEN", "token") + evidence = call_ors_openreward({"x": 1}, provider="ors", timeout_s=2) + srv.shutdown() + assert evidence.passed is False + assert evidence.blocked_reason == "provider_endpoint_must_be_https" + + +def test_harbor_missing_credentials(monkeypatch, tmp_path: Path) -> None: + monkeypatch.delenv("BREADBOARD_BENCHFLOW_BASE_URL", raising=False) + monkeypatch.delenv("BREADBOARD_BENCHFLOW_TOKEN", raising=False) + monkeypatch.delenv("BREADBOARD_HARBOR_BASE_URL", raising=False) + monkeypatch.delenv("BREADBOARD_HARBOR_TOKEN", raising=False) + report = run_harbor_service_proof(tmp_path / "env.tar", target_run_id=TARGET) + assert report["blocked_reason"] == "missing_harbor_credentials" + +def test_harbor_loopback_provider_is_blocked(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setenv("BREADBOARD_HARBOR_BASE_URL", "https://127.0.0.1:8443") + monkeypatch.setenv("BREADBOARD_HARBOR_TOKEN", "token") + report = run_harbor_service_proof(tmp_path / "env.tar", target_run_id=TARGET) + assert report["passed"] is False + assert report["blocked_reason"] == "provider_endpoint_must_not_be_loopback_or_private" + + +def test_harbor_facade_attestation_reports_routes(monkeypatch, tmp_path: Path) -> None: + env_package = tmp_path / "env.tar" + env_package.write_bytes(b"harbor env package") + seen_requests: list[tuple[str, str]] = [] + expected_routes = [ + "GET /health", + "GET /metrics.json", + "GET /list_tasks", + "POST /score", + "POST /trial/create", + "POST /trial/{trial_id}/exec", + "GET /trial/{trial_id}", + "POST /trial/{trial_id}/finalize", + ] + + class Response: + status = 200 + + def __init__(self, body: bytes = b'{"ok":true}') -> None: + self._body = body + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self) -> bytes: + return self._body + + def fake_open(request, *, timeout_s: float): # noqa: ANN001 + assert timeout_s == 10.0 + seen_requests.append((request.get_method(), request.full_url)) + if request.full_url.endswith("/list_tasks"): + return Response(b'["phase3-harbor-smoke"]') + if request.full_url.endswith("/score"): + body = json.loads(request.data) + assert body == {"answer": "phase3 harbor service proof", "task_name": "phase3-harbor-smoke"} + return Response(b'{"reward":1.0,"raw":{"reward":1.0},"error":null}') + if request.full_url.endswith("/trial/create"): + body = json.loads(request.data) + assert body == {"task_name": "phase3-harbor-smoke"} + return Response(b'{"trial_id":"trial-1","task_name":"phase3-harbor-smoke","instruction":"solve","expires_at_ts":1.0}') + if request.full_url.endswith("/trial/trial-1/exec"): + body = json.loads(request.data) + assert body == { + "cmd": "printf phase3-harbor-proof", + "timeout_sec": 30, + } + return Response(b'{"stdout":"phase3-harbor-proof","stderr":"","return_code":0,"stdout_truncated":false,"stderr_truncated":false,"duration_sec":0.1}') + if request.full_url.endswith("/trial/trial-1/finalize"): + body = json.loads(request.data) + assert body == {"answer": "phase3 harbor service proof"} + return Response(b'{"reward":1.0,"raw":{"reward":1.0},"error":null}') + if request.full_url.endswith("/trial/trial-1"): + return Response(b'{"trial_id":"trial-1","task_name":"phase3-harbor-smoke","created_at_ts":0.0,"expires_at_ts":1.0,"n_exec_calls":1,"finalized":false}') + return Response(b'{"dataset":"harbor-test","ok":true}') + + monkeypatch.setenv("BREADBOARD_HARBOR_BASE_URL", "https://1.1.1.1/harbor") + monkeypatch.setenv("BREADBOARD_HARBOR_TOKEN", "harbor-token") + monkeypatch.delenv("BREADBOARD_BENCHFLOW_BASE_URL", raising=False) + monkeypatch.delenv("BREADBOARD_BENCHFLOW_TOKEN", raising=False) + monkeypatch.setattr("breadboard.rl.phase3.integrations._open_without_proxies", fake_open) + + report = run_benchflow_harbor_attestation(env_package, target_run_id=TARGET) + + assert report["passed"] is True + assert report["attestation_backend"] == "harbor_facade" + assert report["provider_kind"] == "harbor_facade" + assert report["env_package_sha256"] == "sha256:" + hashlib.sha256(env_package.read_bytes()).hexdigest() + assert report["harbor_routes"] == expected_routes + assert report["status_codes"] == {route: 200 for route in expected_routes} + assert seen_requests == [ + ("GET", "https://1.1.1.1/harbor/health"), + ("GET", "https://1.1.1.1/harbor/metrics.json"), + ("GET", "https://1.1.1.1/harbor/list_tasks"), + ("POST", "https://1.1.1.1/harbor/score"), + ("POST", "https://1.1.1.1/harbor/trial/create"), + ("POST", "https://1.1.1.1/harbor/trial/trial-1/exec"), + ("GET", "https://1.1.1.1/harbor/trial/trial-1"), + ("POST", "https://1.1.1.1/harbor/trial/trial-1/finalize"), + ] + + +def test_harbor_facade_attestation_rejects_200_status_failed_semantics(monkeypatch, tmp_path: Path) -> None: + env_package = tmp_path / "env.tar" + env_package.write_bytes(b"harbor env package") + + class Response: + status = 200 + + def __init__(self, body: bytes = b'{"ok":true}') -> None: + self._body = body + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self) -> bytes: + return self._body + + def fake_open(request, *, timeout_s: float): # noqa: ANN001, ARG001 + if request.full_url.endswith("/list_tasks"): + return Response(b'["phase3-harbor-smoke"]') + if request.full_url.endswith("/score"): + return Response(b'{"score":0.0,"passed":false}') + if request.full_url.endswith("/trial/create"): + return Response(b'{"trial_id":"trial-1","task_name":"phase3-harbor-smoke"}') + if request.full_url.endswith("/trial/trial-1/exec"): + return Response(b'{"stdout":"phase3-harbor-proof","return_code":0}') + if request.full_url.endswith("/trial/trial-1/finalize"): + return Response(b'{"reward":1.0}') + if request.full_url.endswith("/trial/trial-1"): + return Response(b'{"trial_id":"trial-1","task_name":"phase3-harbor-smoke","n_exec_calls":1}') + return Response(b'{"dataset":"harbor-test","ok":true}') + + monkeypatch.setenv("BREADBOARD_HARBOR_BASE_URL", "https://1.1.1.1/harbor") + monkeypatch.setenv("BREADBOARD_HARBOR_TOKEN", "harbor-token") + monkeypatch.setattr("breadboard.rl.phase3.integrations._open_without_proxies", fake_open) + + report = run_benchflow_harbor_attestation(env_package, target_run_id=TARGET) + + assert report["passed"] is False + assert report["blocked_reason"] == "harbor_response_semantics_failed" + + +def test_harbor_facade_attestation_rejects_missing_exec_count(monkeypatch, tmp_path: Path) -> None: + env_package = tmp_path / "env.tar" + env_package.write_bytes(b"harbor env package") + + class Response: + status = 200 + + def __init__(self, body: bytes = b'{"ok":true}') -> None: + self._body = body + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self) -> bytes: + return self._body + + def fake_open(request, *, timeout_s: float): # noqa: ANN001, ARG001 + if request.full_url.endswith("/list_tasks"): + return Response(b'["phase3-harbor-smoke"]') + if request.full_url.endswith("/score"): + return Response(b'{"score":1.0,"passed":true}') + if request.full_url.endswith("/trial/create"): + return Response(b'{"trial_id":"trial-1","task_name":"phase3-harbor-smoke"}') + if request.full_url.endswith("/trial/trial-1/exec"): + return Response(b'{"stdout":"phase3-harbor-proof","return_code":0}') + if request.full_url.endswith("/trial/trial-1/finalize"): + return Response(b'{"reward":1.0}') + if request.full_url.endswith("/trial/trial-1"): + return Response(b'{"trial_id":"trial-1","task_name":"phase3-harbor-smoke"}') + return Response(b'{"dataset":"harbor-test","ok":true}') + + monkeypatch.setenv("BREADBOARD_HARBOR_BASE_URL", "https://1.1.1.1/harbor") + monkeypatch.setenv("BREADBOARD_HARBOR_TOKEN", "harbor-token") + monkeypatch.setattr("breadboard.rl.phase3.integrations._open_without_proxies", fake_open) + + report = run_benchflow_harbor_attestation(env_package, target_run_id=TARGET) + + assert report["passed"] is False + assert report["blocked_reason"] == "harbor_response_semantics_failed" diff --git a/tests/rl/phase3/test_live_provider_runner.py b/tests/rl/phase3/test_live_provider_runner.py new file mode 100644 index 00000000..7fd23328 --- /dev/null +++ b/tests/rl/phase3/test_live_provider_runner.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +import json +import hashlib +import subprocess +import sys +from pathlib import Path + +import io +from breadboard.rl.phase3 import integrations +from scripts.rl_phase3.run_phase3_live_provider_reports import _component +TARGET = "20260623T000000Z-slurm-234555" + + +def test_live_provider_runner_blocks_without_credentials(tmp_path: Path, monkeypatch) -> None: + for key in [ + "BREADBOARD_ORS_BASE_URL", + "BREADBOARD_ORS_TOKEN", + "BREADBOARD_OPENREWARD_BASE_URL", + "BREADBOARD_OPENREWARD_TOKEN", + "BREADBOARD_BENCHFLOW_BASE_URL", + "BREADBOARD_BENCHFLOW_TOKEN", + "BREADBOARD_HARBOR_BASE_URL", + "BREADBOARD_HARBOR_TOKEN", + ]: + monkeypatch.delenv(key, raising=False) + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/run_phase3_live_provider_reports.py", + "--phase-dir", + str(phase_dir), + "--target-run-id", + TARGET, + "--rows-jsonl", + str(tmp_path / "missing_rows.jsonl"), + ], + cwd=Path(__file__).resolve().parents[3], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert result.returncode == 2 + out = phase_dir / "runs" / "live_provider_reports" + p3m8 = json.loads((out / "P3-M8_retired_provider_milestone.json").read_text()) + p3m9 = json.loads((out / "P3-M9_harbor_nemo_gym.json").read_text()) + assert p3m8["passed"] is False + assert p3m8["required_artifact_keys"] == ["provider_report"] + assert Path(p3m8["artifact_paths"]["provider_report"]).exists() + assert p3m8["component"] == "retired_provider_milestone" + assert p3m8["provider_kind"] == "none" + assert p3m8["blocked_reason"] == "retired_provider_milestone_pending_accepted_rubric_change" + assert not (out / "live_verifier_campaign.json").exists() + assert p3m9["passed"] is False + assert p3m9["required_artifact_keys"] == ["provider_report"] + assert Path(p3m9["artifact_paths"]["provider_report"]).exists() + assert not (out / "benchflow_env_package_placeholder.txt").exists() + + +def test_live_provider_runner_ignores_native_benchflow_without_harbor(tmp_path: Path, monkeypatch) -> None: + for key in [ + "BREADBOARD_ORS_BASE_URL", + "BREADBOARD_ORS_TOKEN", + "BREADBOARD_OPENREWARD_BASE_URL", + "BREADBOARD_OPENREWARD_TOKEN", + "BREADBOARD_HARBOR_BASE_URL", + "BREADBOARD_HARBOR_TOKEN", + ]: + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("BREADBOARD_BENCHFLOW_BASE_URL", "https://1.1.1.1/benchflow") + monkeypatch.setenv("BREADBOARD_BENCHFLOW_TOKEN", "benchflow-token") + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/run_phase3_live_provider_reports.py", + "--phase-dir", + str(phase_dir), + "--target-run-id", + TARGET, + ], + cwd=Path(__file__).resolve().parents[3], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert result.returncode == 2 + out = phase_dir / "runs" / "live_provider_reports" + harbor = json.loads((out / "harbor_service_proof.json").read_text()) + p3m9 = json.loads((out / "P3-M9_harbor_nemo_gym.json").read_text()) + assert harbor["blocked_reason"] == "missing_harbor_credentials" + assert p3m9["blocked_reason"] == "missing_harbor_credentials" + assert not (out / "harbor_env_package_required.tar").exists() + + +def test_native_benchflow_helper_hashes_real_env_package_as_contract_only(tmp_path: Path, monkeypatch) -> None: + env_package = tmp_path / "env-package.tar" + env_package.write_bytes(b"phase3 env package bytes") + captured: dict[str, object] = {} + + class Response: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self) -> bytes: + return b'{"attested":true}' + + def fake_open(request, *, timeout_s: float): + assert isinstance(request.data, bytes) + captured["url"] = request.full_url + captured["body_bytes"] = request.data + captured["timeout_s"] = timeout_s + return Response() + + monkeypatch.setattr(integrations, "_open_without_proxies", fake_open) + + report = integrations._run_native_benchflow_attestation("https://1.1.1.1/benchflow", "benchflow-token", env_package, target_run_id=TARGET) + + assert report["passed"] is True + assert report["claim_boundary"] == "phase3_native_benchflow_contract_only_scope" + assert captured["url"] == "https://1.1.1.1/benchflow/attest" + response_body = b'{"attested":true}' + body_bytes = captured["body_bytes"] + assert isinstance(body_bytes, bytes) + body = json.loads(body_bytes) + assert body == { + "target_run_id": TARGET, + "env_package_sha256": "sha256:" + hashlib.sha256(env_package.read_bytes()).hexdigest(), + } + assert report["request_sha256"] == "sha256:" + hashlib.sha256(body_bytes).hexdigest() + assert report["response_sha256"] == "sha256:" + hashlib.sha256(response_body).hexdigest() + assert captured["timeout_s"] == 10.0 + +def test_p3m9_runner_component_sets_provider_kind_on_success(tmp_path: Path) -> None: + provider_report_path = tmp_path / "benchflow.json" + provider_report_path.write_text('{"passed": true}\n') + report = _component( + milestone_id="P3-M9", + component="harbor_nemo_gym", + points=60, + claim_boundary="phase3_harbor_nemo_gym_named_endpoint_scope", + blocked_claim_boundary="phase3_harbor_nemo_gym_blocked_scope", + target_run_id=TARGET, + provider_report_path=provider_report_path, + provider_report={ + "passed": True, + "attestation_backend": "harbor_facade", + }, + ) + + assert report["provider_kind"] == "harbor_facade" + + +def _http_error(url: str, status: int, body: bytes) -> integrations.urllib.error.HTTPError: + return integrations.urllib.error.HTTPError(url, status, "provider error", {}, io.BytesIO(body)) + + +def test_ors_openreward_http_error_keeps_response_hash(monkeypatch) -> None: + response_body = b'{"ok":false}' + monkeypatch.setenv("BREADBOARD_OPENREWARD_BASE_URL", "https://1.1.1.1/openreward") + monkeypatch.setenv("BREADBOARD_OPENREWARD_TOKEN", "openreward-token") + + def fake_open(request, *, timeout_s: float): + raise _http_error(request.full_url, 503, response_body) + + monkeypatch.setattr(integrations, "_open_without_proxies", fake_open) + + report = integrations.call_ors_openreward({"score": 1}, provider="openreward", timeout_s=3.0) + + assert report.passed is False + assert report.status_code == 503 + assert report.response_sha256 == "sha256:" + hashlib.sha256(response_body).hexdigest() + assert report.blocked_reason == "" + + +def test_native_benchflow_http_error_keeps_response_hash(tmp_path: Path, monkeypatch) -> None: + env_package = tmp_path / "env-package.tar" + env_package.write_bytes(b"phase3 env package bytes") + response_body = b'{"attested":false}' + + def fake_open(request, *, timeout_s: float): + raise _http_error(request.full_url, 409, response_body) + + monkeypatch.setattr(integrations, "_open_without_proxies", fake_open) + + report = integrations._run_native_benchflow_attestation("https://1.1.1.1/benchflow", "benchflow-token", env_package, target_run_id=TARGET) + + assert report["passed"] is False + assert report["status_code"] == 409 + assert report["response_sha256"] == "sha256:" + hashlib.sha256(response_body).hexdigest() + assert report["blocked_reason"] == "" + + +def test_provider_redirect_handler_rejects_same_origin_redirect_without_dns_resolution(monkeypatch) -> None: + def fail_getaddrinfo(*args, **kwargs): + raise AssertionError("same-origin redirect must not trigger DNS resolution") + + monkeypatch.setattr(integrations.socket, "getaddrinfo", fail_getaddrinfo) + handler = integrations._NoUnsafeRedirectHandler() + request = integrations.urllib.request.Request( + "https://provider.example/verify", + data=b"{}", + method="POST", + ) + + try: + handler.redirect_request(request, None, 302, "Found", {}, "https://provider.example/private") + except integrations.ProviderRedirectBlocked as exc: + assert exc.block_reason == "provider_redirect_not_followed" + else: + raise AssertionError("same-origin provider redirect was followed") + +def test_provider_redirect_handler_rejects_public_cross_origin_redirect(monkeypatch) -> None: + monkeypatch.setattr( + integrations.socket, + "getaddrinfo", + lambda *args, **kwargs: [(None, None, None, None, ("1.1.1.1", 443))], + ) + handler = integrations._NoUnsafeRedirectHandler() + request = integrations.urllib.request.Request( + "https://provider.example/verify", + data=b"{}", + method="POST", + headers={"Authorization": "Bearer secret-token"}, + ) + + try: + handler.redirect_request(request, None, 302, "Found", {}, "https://other.example/verify") + except integrations.ProviderRedirectBlocked as exc: + assert exc.block_reason == "provider_redirect_cross_origin_blocked" + else: + raise AssertionError("public cross-origin provider redirect was allowed") + +def test_provider_redirect_handler_rejects_same_origin_redirect_before_post_rewrite() -> None: + handler = integrations._NoUnsafeRedirectHandler() + request = integrations.urllib.request.Request( + "https://provider.example/verify", + data=b'{"score":1}', + method="POST", + ) + + try: + handler.redirect_request(request, None, 302, "Found", {}, "https://provider.example/verify-v2") + except integrations.ProviderRedirectBlocked as exc: + assert exc.block_reason == "provider_redirect_not_followed" + else: + raise AssertionError("same-origin provider redirect was followed") + +def test_hostname_provider_connect_pins_tcp_ip_but_keeps_sni(monkeypatch) -> None: + captured: dict[str, object] = {} + + def fake_create_connection(endpoint, timeout, source_address=None): + captured["endpoint"] = endpoint + captured["timeout"] = timeout + captured["source_address"] = source_address + return object() + + class Context: + def wrap_socket(self, sock, *, server_hostname: str): + captured["sock"] = sock + captured["server_hostname"] = server_hostname + return "tls-socket" + + monkeypatch.setattr(integrations.socket, "create_connection", fake_create_connection) + connection = integrations._PinnedHTTPSConnection( + "provider.example", + {("provider.example", 443): ("1.1.1.1",)}, + timeout=2.0, + ) + connection._context = Context() + + connection.connect() + + assert captured["endpoint"] == ("1.1.1.1", 443) + assert captured["server_hostname"] == "provider.example" + assert connection.sock == "tls-socket" + + + +def test_ors_openreward_redirect_block_reason_is_canonical(monkeypatch) -> None: + monkeypatch.setenv("BREADBOARD_ORS_BASE_URL", "https://1.1.1.1/provider") + monkeypatch.setenv("BREADBOARD_ORS_TOKEN", "ors-token") + + def fake_open(_request, *, timeout_s: float): + raise integrations.ProviderRedirectBlocked("provider_endpoint_must_not_be_loopback_or_private") + + monkeypatch.setattr(integrations, "_open_without_proxies", fake_open) + + report = integrations.call_ors_openreward({"x": 1}, provider="ors", timeout_s=3.0) + + assert report.passed is False + assert report.blocked_reason == "provider_endpoint_must_not_be_loopback_or_private" + + +def test_native_benchflow_redirect_block_reason_is_canonical(tmp_path: Path, monkeypatch) -> None: + env_package = tmp_path / "env-package.tar" + env_package.write_bytes(b"phase3 env package bytes") + + def fake_open(_request, *, timeout_s: float): + raise integrations.ProviderRedirectBlocked("provider_endpoint_must_not_be_loopback_or_private") + + monkeypatch.setattr(integrations, "_open_without_proxies", fake_open) + + report = integrations._run_native_benchflow_attestation("https://1.1.1.1/benchflow", "benchflow-token", env_package, target_run_id=TARGET) + + assert report["passed"] is False + assert report["blocked_reason"] == "provider_endpoint_must_not_be_loopback_or_private" \ No newline at end of file diff --git a/tests/rl/phase3/test_live_service_store.py b/tests/rl/phase3/test_live_service_store.py new file mode 100644 index 00000000..a6ed597d --- /dev/null +++ b/tests/rl/phase3/test_live_service_store.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import pytest + +from breadboard.rl.phase2.service import ArtifactRecord, ResourceCaps, RunSubmission +from breadboard.rl.phase3.service_live import LiveRLRunService +from breadboard.rl.phase3.store import SQLiteRLRunStore + +TARGET = "20260623T000000Z-slurm-234555" + + +def sub(run_id: str = "run-1", **overrides) -> RunSubmission: + values = { + "run_id": run_id, + "tenant_id": "tenant-a", + "workspace_id": "ws", + "env_package_ref": "ws/env.tar", + "target_run_id": TARGET, + "requested_tasks": 1, + "requested_gpus": 1, + "requested_budget_usd": 1.0, + "requested_duration_seconds": 60, + "metadata": {}, + } + values.update(overrides) + return RunSubmission(**values) + + +def test_sqlite_persistence_restart(tmp_path) -> None: + db = tmp_path / "runs.db" + service = LiveRLRunService(SQLiteRLRunStore(db)) + service.submit(sub()) + service.complete("run-1") + service.store.close() + restarted = LiveRLRunService(SQLiteRLRunStore(db)) + assert restarted.status("run-1", tenant_id="tenant-a", workspace_id="ws").state == "succeeded" + + +def test_artifact_replay_persists(tmp_path) -> None: + service = LiveRLRunService(SQLiteRLRunStore(tmp_path / "runs.db")) + service.submit(sub()) + service.add_artifact(ArtifactRecord("run-1", "replay", "ws/replay/a.json", "sha256:abc", 10, True), tenant_id="tenant-a", workspace_id="ws") + assert service.collect("run-1", tenant_id="tenant-a", workspace_id="ws")[0].artifact_id == "replay" + assert service.replay("run-1", "replay", tenant_id="tenant-a", workspace_id="ws")["available"] is True + + +def test_cross_tenant_artifact_denied(tmp_path) -> None: + service = LiveRLRunService(SQLiteRLRunStore(tmp_path / "runs.db")) + service.submit(sub()) + try: + service.collect("run-1", tenant_id="tenant-b", workspace_id="ws") + except PermissionError: + return + raise AssertionError("tenant mismatch should fail") + + + +def test_live_service_rejects_invalid_terminal_state_transitions(tmp_path) -> None: + service = LiveRLRunService(SQLiteRLRunStore(tmp_path / "runs.db")) + service.submit(sub()) + service.complete("run-1") + + with pytest.raises(ValueError, match="cannot complete from state succeeded"): + service.complete("run-1") + with pytest.raises(ValueError, match="already terminal: succeeded"): + service.cancel("run-1") + + assert service.status("run-1", tenant_id="tenant-a", workspace_id="ws").state == "succeeded" + + +def test_live_service_rejected_run_cannot_start(tmp_path) -> None: + service = LiveRLRunService( + SQLiteRLRunStore(tmp_path / "runs.db"), + caps=ResourceCaps(max_tasks=1, max_gpus=1, max_budget_usd=10.0, max_duration_seconds=60, max_artifact_bytes=1024), + ) + rejected = service.submit(sub(run_id="run-rejected", requested_gpus=2)) + + assert rejected.state == "rejected" + with pytest.raises(ValueError, match="cannot start from state rejected"): + service.start("run-rejected") + assert service.status("run-rejected", tenant_id="tenant-a", workspace_id="ws").state == "rejected" + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("requested_tasks", 0), + ("requested_tasks", -1), + ("requested_gpus", 0), + ("requested_gpus", -1), + ("requested_budget_usd", 0.0), + ("requested_budget_usd", -1.0), + ("requested_duration_seconds", 0), + ("requested_duration_seconds", -1), + ], +) +def test_live_service_rejects_non_positive_submission_resources(tmp_path, field: str, value: int | float) -> None: + service = LiveRLRunService(SQLiteRLRunStore(tmp_path / "runs.db")) + + status = service.submit(sub(run_id=f"run-{field}-{abs(int(value))}", **{field: value})) + + assert status.state == "rejected" + assert status.accepted is False + assert field in status.reason \ No newline at end of file diff --git a/tests/rl/phase3/test_observability_runner.py b/tests/rl/phase3/test_observability_runner.py new file mode 100644 index 00000000..d4b752c6 --- /dev/null +++ b/tests/rl/phase3/test_observability_runner.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import json +import subprocess +import sys +import pytest +from pathlib import Path + +TARGET = "20260623T000000Z-slurm-234555" + + +def test_observability_runner_blocks_without_live_inputs(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("BREADBOARD_VERIFIER_BASE_URL", "https://controller-only.example/verifier") + monkeypatch.setenv("BREADBOARD_VERIFIER_TOKEN", "controller-token") + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/run_phase3_observability_scheduler_store.py", + "--phase-dir", + str(phase_dir), + "--target-run-id", + TARGET, + ], + cwd=Path(__file__).resolve().parents[3], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert result.returncode == 2 + report = json.loads((phase_dir / "runs" / "observability_scheduler_store_runner" / "P3-M11_observability_scheduler_store.json").read_text()) + assert report["passed"] is False + assert "verifier_metrics" in report["blocked_reason"] + assert report["required_artifact_keys"] == ["blocker_evidence"] + assert Path(report["artifact_paths"]["blocker_evidence"]).exists() + blocker = json.loads(Path(report["artifact_paths"]["blocker_evidence"]).read_text()) + assert blocker["missing_inputs"] == [ + "slurm_metrics", + "gpu_metrics", + "verifier_metrics", + "service_metrics", + "object_store_metrics", + "scheduler_metrics", + "budget_caps", + ] + assert blocker["controller_env_verifier_base_url_present"] is True + assert blocker["controller_env_verifier_token_present"] is True + assert "env_live_verifier_endpoint_present" not in blocker + assert blocker["object_store_metrics_present"] is False + assert blocker["production_object_store_endpoint_present"] is False + + +@pytest.mark.parametrize("backend", ["LocalObjectStore", "target_workspace_local_object_store", "local_object_store"]) +def test_observability_runner_rejects_local_object_store(backend: str, tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + inputs = tmp_path / "inputs" + inputs.mkdir() + payloads = { + "slurm": {"source": "slurm_sacct", "sacct_stdout": "ok", "queue_wait_seconds": 1, "scheduler_retry_count": 0}, + "gpu": {"source": "rocm_smi", "gpu_utilization": {"card0": {"GPU use (%)": "1"}}}, + "verifier": {"source": "verifier_client", "verifier_latency_seconds": [0.1]}, + "service": {"source": "service_event_log", "task_throughput": 1, "failure_taxonomy": {}}, + "object_store": {"source": "object_store", "object_store": backend, "object_store_writes": 1, "artifact_bytes": 1}, + "scheduler": {"source": "scheduler_control", "scheduler_control": {"endpoint_present": True, "token_present": True}}, + "budget": {"remaining_usd": 1}, + } + paths = {} + for name, payload in payloads.items(): + path = inputs / f"{name}.json" + path.write_text(json.dumps(payload)) + paths[name] = path + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/run_phase3_observability_scheduler_store.py", + "--phase-dir", + str(phase_dir), + "--target-run-id", + TARGET, + "--slurm-metrics", + str(paths["slurm"]), + "--gpu-metrics", + str(paths["gpu"]), + "--verifier-metrics", + str(paths["verifier"]), + "--service-metrics", + str(paths["service"]), + "--object-store-metrics", + str(paths["object_store"]), + "--scheduler-metrics", + str(paths["scheduler"]), + "--budget-caps", + str(paths["budget"]), + ], + cwd=Path(__file__).resolve().parents[3], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert result.returncode == 2 + report = json.loads((phase_dir / "runs" / "observability_scheduler_store_runner" / "P3-M11_observability_scheduler_store.json").read_text()) + assert report["passed"] is False + assert "production_object_store_endpoint_missing" in report["blocked_reason"] + assert "live_observability_report" in report["required_artifact_keys"] + live = json.loads((phase_dir / "runs" / "observability_scheduler_store_runner" / "live_observability_report.json").read_text()) + assert live["passed"] is False + assert "production_object_store_endpoint_missing" in live["errors"] + + +def test_observability_runner_rejects_empty_object_store_metrics(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + inputs = tmp_path / "inputs" + inputs.mkdir() + payloads = { + "slurm": {"source": "slurm_sacct", "sacct_stdout": "ok", "queue_wait_seconds": 1, "scheduler_retry_count": 0}, + "gpu": {"source": "rocm_smi", "gpu_utilization": {"card0": {"GPU use (%)": "1"}}}, + "verifier": {"source": "verifier_client", "verifier_latency_seconds": [0.1]}, + "service": {"source": "service_event_log", "task_throughput": 1, "failure_taxonomy": {}}, + "object_store": {}, + "scheduler": {"source": "scheduler_control", "scheduler_control": {"endpoint_present": True, "token_present": True}}, + "budget": {"remaining_usd": 1}, + } + paths = {} + for name, payload in payloads.items(): + path = inputs / f"{name}.json" + path.write_text(json.dumps(payload)) + paths[name] = path + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/run_phase3_observability_scheduler_store.py", + "--phase-dir", + str(phase_dir), + "--target-run-id", + TARGET, + "--slurm-metrics", + str(paths["slurm"]), + "--gpu-metrics", + str(paths["gpu"]), + "--verifier-metrics", + str(paths["verifier"]), + "--service-metrics", + str(paths["service"]), + "--object-store-metrics", + str(paths["object_store"]), + "--scheduler-metrics", + str(paths["scheduler"]), + "--budget-caps", + str(paths["budget"]), + ], + cwd=Path(__file__).resolve().parents[3], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert result.returncode == 2 + live = json.loads((phase_dir / "runs" / "observability_scheduler_store_runner" / "live_observability_report.json").read_text()) + assert live["passed"] is False + assert "object_store_metrics_missing" in live["errors"] + + +def test_observability_runner_preserves_invalid_metric_sources(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + inputs = tmp_path / "inputs" + inputs.mkdir() + payloads = { + "slurm": {"source": "target_probe", "sacct_stdout": "ok", "queue_wait_seconds": 1, "scheduler_retry_count": 0}, + "gpu": {"source": "rocm_smi", "gpu_utilization": {"card0": {"GPU use (%)": "1"}}}, + "verifier": {"source": "verifier_client", "verifier_latency_seconds": [0.1]}, + "service": {"source": "service_event_log", "task_throughput": 1, "failure_taxonomy": {}}, + "object_store": {"source": "object_store", "object_store": "S3ObjectStore", "object_store_writes": 1, "artifact_bytes": 1}, + "scheduler": {"source": "scheduler_control", "scheduler_control": {"endpoint_present": True, "token_present": True}}, + "budget": {"remaining_usd": 1}, + } + paths = {} + for name, payload in payloads.items(): + path = inputs / f"{name}.json" + path.write_text(json.dumps(payload)) + paths[name] = path + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/run_phase3_observability_scheduler_store.py", + "--phase-dir", + str(phase_dir), + "--target-run-id", + TARGET, + "--slurm-metrics", + str(paths["slurm"]), + "--gpu-metrics", + str(paths["gpu"]), + "--verifier-metrics", + str(paths["verifier"]), + "--service-metrics", + str(paths["service"]), + "--object-store-metrics", + str(paths["object_store"]), + "--scheduler-metrics", + str(paths["scheduler"]), + "--budget-caps", + str(paths["budget"]), + ], + cwd=Path(__file__).resolve().parents[3], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert result.returncode == 2 + report_path = phase_dir / "runs" / "observability_scheduler_store_runner" / "P3-M11_observability_scheduler_store.json" + report = json.loads(report_path.read_text()) + live = json.loads((phase_dir / "runs" / "observability_scheduler_store_runner" / "live_observability_report.json").read_text()) + assert live["passed"] is False + assert "slurm_metrics.source must be 'slurm_sacct'" in ";".join(live["errors"]) + assert live["metric_sections"]["slurm_metrics"]["source"] == "target_probe" + + +def test_observability_runner_fills_only_missing_metric_sources() -> None: + from breadboard.rl.phase3.evidence import normalize_phase3_metric_sources + + metrics = { + "slurm": {"sacct_stdout": "ok"}, + "gpu": {"source": "target_probe", "gpu_utilization": {}}, + } + + normalized = normalize_phase3_metric_sources(metrics) + + assert normalized["slurm"]["source"] == "slurm_sacct" + assert normalized["gpu"]["source"] == "target_probe" + assert normalized["scheduler"]["source"] == "scheduler_control" + diff --git a/tests/rl/phase3/test_p3m11_live_endpoint_probe.py b/tests/rl/phase3/test_p3m11_live_endpoint_probe.py new file mode 100644 index 00000000..1478eda9 --- /dev/null +++ b/tests/rl/phase3/test_p3m11_live_endpoint_probe.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import ClassVar + +import pytest + +from scripts.rl_phase3 import p3m11_live_endpoint_probe as probe + + +class _ProbeHandler(BaseHTTPRequestHandler): + store: ClassVar[dict[str, bytes]] = {} + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 - stdlib signature + return + + def do_GET(self) -> None: # noqa: N802 - stdlib hook + if self.path == "/verifier/ready" or self.path == "/scheduler/ready": + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + return + if self.path.startswith("/objects/"): + payload = self.store.get(self.path) + if payload is None: + self.send_response(404) + self.end_headers() + return + self.send_response(200) + self.end_headers() + self.wfile.write(payload) + return + self.send_response(404) + self.end_headers() + + def do_PUT(self) -> None: # noqa: N802 - stdlib hook + if not self.path.startswith("/objects/"): + self.send_response(404) + self.end_headers() + return + length = int(self.headers.get("Content-Length", "0")) + self.store[self.path] = self.rfile.read(length) + self.send_response(201) + self.end_headers() + + def do_DELETE(self) -> None: # noqa: N802 - stdlib hook + self.store.pop(self.path, None) + self.send_response(204) + self.end_headers() + + +@pytest.fixture() +def probe_server() -> str: + _ProbeHandler.store = {} + server = ThreadingHTTPServer(("127.0.0.1", 0), _ProbeHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + host, port = server.server_address + yield f"http://{host}:{port}" + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_probe_requires_explicit_paths(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BREADBOARD_VERIFIER_BASE_URL", "https://verifier.example") + monkeypatch.setenv("BREADBOARD_VERIFIER_TOKEN", "redacted") + + metrics, errors = probe.collect_verifier_metrics() + + assert metrics["verifier_latency_seconds"] is None + assert "BREADBOARD_VERIFIER_PROBE_PATH" in errors + assert "verifier_probe_not_configured" in errors + + +def test_probe_redacts_query_and_userinfo() -> None: + assert probe._redacted_url("https://user:secret@example.test:9443/ready?token=secret#frag") == "https://example.test:9443/ready" + + +def test_probe_rejects_schemeless_local_host_port() -> None: + assert probe._endpoint_is_local("localhost:8080") + assert probe._endpoint_is_local("127.0.0.1:9000") + + +def test_probe_collects_live_endpoint_metrics(monkeypatch: pytest.MonkeyPatch, probe_server: str) -> None: + monkeypatch.setenv("PHASE3_TARGET_RUN_ID", "20260624T040000Z-slurm-243958") + monkeypatch.setenv("PHASE3_COMMAND_ID", "phase3_p3m11_live_endpoint_probe_test") + monkeypatch.setenv("BREADBOARD_VERIFIER_BASE_URL", probe_server) + monkeypatch.setenv("BREADBOARD_VERIFIER_PROBE_PATH", "/verifier/ready") + monkeypatch.setenv("BREADBOARD_VERIFIER_TOKEN", "redacted") + monkeypatch.setenv("BREADBOARD_VERIFIER_TOKEN_SCHEME", "") + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_BASE_URL", probe_server) + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_BUCKET", "phase3-test") + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_TOKEN", "redacted") + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_TOKEN_SCHEME", "") + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_PUT_URL_TEMPLATE", f"{probe_server}/objects/{{bucket}}/{{key}}") + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_GET_URL_TEMPLATE", f"{probe_server}/objects/{{bucket}}/{{key}}") + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_DELETE_URL_TEMPLATE", f"{probe_server}/objects/{{bucket}}/{{key}}") + monkeypatch.setenv("BREADBOARD_SCHEDULER_BASE_URL", probe_server) + monkeypatch.setenv("BREADBOARD_SCHEDULER_PROBE_PATH", "/scheduler/ready") + monkeypatch.setenv("BREADBOARD_SCHEDULER_TOKEN", "redacted") + monkeypatch.setenv("BREADBOARD_SCHEDULER_TOKEN_SCHEME", "") + + monkeypatch.setattr(probe, "_endpoint_is_local", lambda url: False) + verifier, verifier_errors = probe.collect_verifier_metrics() + object_store, object_store_errors = probe.collect_object_store_metrics() + scheduler, scheduler_errors = probe.collect_scheduler_metrics() + + assert verifier_errors == [] + assert object_store_errors == [] + assert scheduler_errors == [] + assert verifier["verifier_latency_seconds"] and verifier["verifier_latency_seconds"][0] >= 0 + assert verifier["endpoint"].endswith("/verifier/ready") + assert object_store["object_store"] == "configured_http_object_store" + assert object_store["object_store_writes"] == 1 + assert object_store["artifact_bytes"] > 0 + assert object_store["write_read_verified"] is True + assert object_store["written_sha256"] == object_store["artifact_sha256"] + assert object_store["readback_sha256"] == object_store["artifact_sha256"] + assert scheduler["scheduler_control"]["endpoint_present"] is True + assert scheduler["scheduler_control"]["token_present"] is True + assert scheduler["scheduler_control"]["status"] == "ready" + + +def test_probe_rejects_local_object_store_urls(monkeypatch: pytest.MonkeyPatch, probe_server: str) -> None: + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_BASE_URL", probe_server) + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_BUCKET", "phase3-test") + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_TOKEN", "redacted") + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_PUT_URL_TEMPLATE", f"{probe_server}/objects/{{bucket}}/{{key}}") + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_GET_URL_TEMPLATE", f"{probe_server}/objects/{{bucket}}/{{key}}") + monkeypatch.setenv("BREADBOARD_OBJECT_STORE_DELETE_URL_TEMPLATE", f"{probe_server}/objects/{{bucket}}/{{key}}") + + object_store, errors = probe.collect_object_store_metrics() + + assert { + "object_store_base_endpoint_is_local", + "object_store_put_endpoint_is_local", + "object_store_get_endpoint_is_local", + "object_store_delete_endpoint_is_local", + }.issubset(errors) + assert "object_store_probe_not_configured" not in errors + assert object_store["write_read_verified"] is False + assert object_store["object_store_writes"] == 0 + assert _ProbeHandler.store == {} + + +def test_probe_main_reports_blocked_without_credentials(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + for key in list(probe.os.environ): + if key.startswith("BREADBOARD_VERIFIER") or key.startswith("BREADBOARD_OBJECT_STORE") or key.startswith("BREADBOARD_SCHEDULER"): + monkeypatch.delenv(key, raising=False) + monkeypatch.setattr(probe, "OUT", tmp_path / "out") + + assert probe.main() == 0 + out = capsys.readouterr().out + + assert "PHASE3_COMPONENT_REPORT_JSON=" in out + assert "verifier_latency_unavailable" in out + assert "production_object_store_write_read_unavailable" in out + assert "scheduler_control_unavailable" in out diff --git a/tests/rl/phase3/test_parity_report_builder.py b/tests/rl/phase3/test_parity_report_builder.py new file mode 100644 index 00000000..04e17c76 --- /dev/null +++ b/tests/rl/phase3/test_parity_report_builder.py @@ -0,0 +1,328 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from breadboard.rl.phase3.parity import PHASE3_PARITY_CLAIM_BOUNDARY, PHASE3_PARITY_REPORT_ID, PHASE3_PARITY_SCHEMA, build_phase3_parity_report, validate_phase3_parity_report +from scripts.rl_phase3.build_phase3_parity_report import MILESTONE_FILES, _attach_parity, _runtime_evidence + + +def _write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, sort_keys=True) + "\n") + + +def test_update_milestones_attaches_same_parity_metadata(tmp_path: Path) -> None: + evidence_root = tmp_path / "docs_tmp" + reports_dir = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs" / "milestone_reports" + parity_path = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs" / "parity" / "phase3_parity_report.json" + _write_json(parity_path, {"schema_version": "bb.rl.phase3.parity_report.v1"}) + parity_sha = "sha256:" + "a" * 64 + + for filename in (MILESTONE_FILES["P3-M2"], MILESTONE_FILES["P3-M3"], MILESTONE_FILES["P3-M4"]): + _write_json(reports_dir / filename, {"artifact_paths": {}, "input_hashes": {}, "required_artifact_keys": []}) + _attach_parity(reports_dir / filename, parity_path=parity_path, parity_sha256=parity_sha, evidence_root=evidence_root) + + attached = [json.loads((reports_dir / filename).read_text()) for filename in (MILESTONE_FILES["P3-M2"], MILESTONE_FILES["P3-M3"], MILESTONE_FILES["P3-M4"])] + parity_paths = {report["artifact_paths"]["parity_report"] for report in attached} + parity_hashes = {report["input_hashes"]["parity_report"] for report in attached} + parity_ids = {report["parity_report_id"] for report in attached} + + assert parity_paths == {"ZYPHRA/RL_PHASE_3/runs/parity/phase3_parity_report.json"} + assert parity_hashes == {parity_sha} + assert parity_ids == {"phase3_parity_report"} + assert all("parity_report" in report["required_artifact_keys"] for report in attached) + + +def test_infrastructure_parity_rejects_same_basename_different_runtime_roots(tmp_path: Path) -> None: + evidence_root = tmp_path / "docs_tmp" + runtime_a = "/shared/runtime/phase3_vllm_verl_py312" + runtime_b = "/alternate/runtime/phase3_vllm_verl_py312" + command_log = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs" / "command_logs" / "cmd.log" + command_log.parent.mkdir(parents=True, exist_ok=True) + command_log.write_text("Digest: sha256:" + "b" * 64 + "\n") + + closed_stdout = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs" / "closed" / "trainer_stdout.log" + closed_stdout.parent.mkdir(parents=True, exist_ok=True) + closed_stdout.write_text("'tensor_model_parallel_size': 2\n'memory_limit_mb': 1024\n'default_local_dir': '/tmp/x'\n'storage_backend': 'SimpleStorage'\n") + closed_stderr = closed_stdout.with_name("trainer_stderr.log") + closed_stderr.write_text("Started a local Ray instance\n") + reward = closed_stdout.with_name("reward.py") + projection = closed_stdout.with_name("accepted_projection_rows.jsonl") + manifest = closed_stdout.with_name("evidence_manifest.json") + metrics = closed_stdout.with_name("metrics.json") + for artifact in (reward, projection, metrics): + artifact.write_text("{}\n") + _write_json(manifest, {"checkpoint_dir": "/remote/checkpoint", "files": {"train.parquet": {"bytes": 1}}}) + + base_report = { + "report_id": "report", + "trainer_backend": "verl_ppo", + "rollout_name": "vllm", + "model_ref": "Qwen/Qwen2.5-0.5B-Instruct", + "n_gpus_per_node": 8, + "device_count": 8, + "optimizer_step_count": 1, + "checkpoint_before_sha256": "sha256:" + "0" * 64, + "checkpoint_after_sha256": "sha256:" + "1" * 64, + "checkpoint_changed": True, + "artifact_paths": {"target_command_log": "ZYPHRA/RL_PHASE_3/runs/command_logs/cmd.log"}, + } + closed_loop = { + **base_report, + "accepted_count": 1, + "quarantined_count": 0, + "rejected_count": 0, + "dataproto_ok": True, + "artifact_paths": { + "target_command_log": "ZYPHRA/RL_PHASE_3/runs/command_logs/cmd.log", + "trainer_stdout": "ZYPHRA/RL_PHASE_3/runs/closed/trainer_stdout.log", + "trainer_stderr": "ZYPHRA/RL_PHASE_3/runs/closed/trainer_stderr.log", + "reward_function": "ZYPHRA/RL_PHASE_3/runs/closed/reward.py", + "accepted_projection_rows": "ZYPHRA/RL_PHASE_3/runs/closed/accepted_projection_rows.jsonl", + "evidence_manifest": "ZYPHRA/RL_PHASE_3/runs/closed/evidence_manifest.json", + "metrics": "ZYPHRA/RL_PHASE_3/runs/closed/metrics.json", + }, + } + introspection = { + "torch": {"version": "2.9.1+rocm6.4", "device_count": 8, "devices": ["AMD Instinct MI300X"] * 8}, + "symbols": {"verl.__version__": "0.8.0"}, + } + + report = build_phase3_parity_report( + target_run_id="20260624T040000Z-slurm-243958", + ppo_report=base_report, + grpo_report=base_report, + closed_loop_report=closed_loop, + introspection_report=introspection, + runtime_evidence={ + "container_image": "vllm/vllm-openai-rocm:nightly", + "runtime_path": runtime_a, + "runtime_install_runtime": runtime_b, + "runtime_install_report_path": "/tmp/install.json", + "runtime_install_passed": True, + "vllm_version": "0.23.1", + }, + evidence_root=evidence_root, + ) + + missing = report["checklist"]["C10_infrastructure_parity"]["missing"] + assert "single_runtime_install_for_trainer_runtime" in missing + assert report["passed"] is False + + + +def test_infrastructure_parity_rejects_scratch_runtime_install_evidence(tmp_path: Path) -> None: + evidence_root = tmp_path / "docs_tmp" + command_log = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs" / "command_logs" / "cmd.log" + command_log.parent.mkdir(parents=True, exist_ok=True) + command_log.write_text("Digest: sha256:" + "b" * 64 + "\n") + + closed_stdout = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs" / "closed" / "trainer_stdout.log" + closed_stdout.parent.mkdir(parents=True, exist_ok=True) + closed_stdout.write_text("'tensor_model_parallel_size': 2\n'memory_limit_mb': 1024\n'default_local_dir': '/tmp/x'\n'storage_backend': 'SimpleStorage'\n") + closed_stderr = closed_stdout.with_name("trainer_stderr.log") + closed_stderr.write_text("Started a local Ray instance\n") + for name in ("reward.py", "accepted_projection_rows.jsonl", "metrics.json"): + closed_stdout.with_name(name).write_text("{}\n") + _write_json(closed_stdout.with_name("evidence_manifest.json"), {"checkpoint_dir": "/tmp/x", "files": {"train.parquet": {"bytes": 1}}}) + + base_report = { + "report_id": "report", + "trainer_backend": "verl_ppo", + "rollout_name": "vllm", + "model_ref": "Qwen/Qwen2.5-0.5B-Instruct", + "n_gpus_per_node": 8, + "device_count": 8, + "optimizer_step_count": 1, + "checkpoint_before_sha256": "sha256:" + "0" * 64, + "checkpoint_after_sha256": "sha256:" + "1" * 64, + "checkpoint_changed": True, + "artifact_paths": {"target_command_log": "ZYPHRA/RL_PHASE_3/runs/command_logs/cmd.log"}, + } + closed_loop = { + **base_report, + "accepted_count": 1, + "quarantined_count": 0, + "rejected_count": 0, + "dataproto_ok": True, + "artifact_paths": { + "target_command_log": "ZYPHRA/RL_PHASE_3/runs/command_logs/cmd.log", + "trainer_stdout": "ZYPHRA/RL_PHASE_3/runs/closed/trainer_stdout.log", + "trainer_stderr": "ZYPHRA/RL_PHASE_3/runs/closed/trainer_stderr.log", + "reward_function": "ZYPHRA/RL_PHASE_3/runs/closed/reward.py", + "accepted_projection_rows": "ZYPHRA/RL_PHASE_3/runs/closed/accepted_projection_rows.jsonl", + "evidence_manifest": "ZYPHRA/RL_PHASE_3/runs/closed/evidence_manifest.json", + "metrics": "ZYPHRA/RL_PHASE_3/runs/closed/metrics.json", + }, + } + report = build_phase3_parity_report( + target_run_id="20260624T040000Z-slurm-243958", + ppo_report=base_report, + grpo_report=base_report, + closed_loop_report=closed_loop, + introspection_report={"torch": {"version": "2.9.1+rocm6.4", "device_count": 8, "devices": ["AMD Instinct MI300X"] * 8}, "symbols": {"verl.__version__": "0.8.0"}}, + runtime_evidence={ + "container_image": "vllm/vllm-openai-rocm:nightly", + "runtime_path": "/shared/bb-p3-root/phase3_vllm_verl_py312", + "runtime_install_runtime": "/shared/bb-p3-root/phase3_vllm_verl_py312", + "runtime_install_report_path": str(evidence_root / "ZYPHRA" / "RL_PHASE_3" / "scratch_runs" / "phase3_verl_vllm_container_install" / "phase3_verl_vllm_container_install.json"), + "runtime_install_passed": True, + "vllm_version": "0.23.1", + }, + evidence_root=evidence_root, + ) + + checklist = report["checklist"]["C10_infrastructure_parity"] + assert checklist["status"] == "open" + assert "target_run_bound_runtime_install" in checklist["missing"] + assert report["passed"] is False + + +def test_runtime_evidence_prefers_target_bound_probe_over_legacy_scratch(tmp_path: Path) -> None: + evidence_root = tmp_path / "docs_tmp" + runs = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs" + scratch = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "scratch_runs" + introspection = runs / "phase3_verl_api_introspection" / "phase3_verl_api_introspection.json" + introspection.parent.mkdir(parents=True, exist_ok=True) + introspection.write_text("{}\n") + for rel in ( + "payloads/phase3_container_ppo_8gpu_stage/run.sh", + "payloads/phase3_container_grpo_8gpu_stage/run.sh", + "payloads/closed_loop_verl_train_stage/run.sh", + ): + script = runs / rel + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text('IMAGE="vllm/vllm-openai-rocm:nightly"\nsource /shared/bb-p3-root/phase3_vllm_verl_py312/bin/activate\n') + target_report = runs / "phase3_vllm_runtime_parity_probe" / "phase3_vllm_runtime_parity_probe.json" + _write_json(target_report, {"target_run_id": "target-run", "passed": True, "runtime": "/shared/bb-p3-root/phase3_vllm_verl_py312", "imports": {"vllm": "target-version"}}) + legacy_report = scratch / "phase3_verl_vllm_container_install" / "phase3_verl_vllm_container_install.json" + _write_json(legacy_report, {"passed": True, "runtime": "/scratch/legacy", "imports": {"vllm": "legacy-version"}}) + + evidence = _runtime_evidence( + evidence_root=evidence_root, + p1_report={"artifact_paths": {"introspection_report": "ZYPHRA/RL_PHASE_3/runs/phase3_verl_api_introspection/phase3_verl_api_introspection.json"}}, + target_run_id="target-run", + ) + + assert evidence["runtime_install_report_artifact"] == "ZYPHRA/RL_PHASE_3/runs/phase3_vllm_runtime_parity_probe/phase3_vllm_runtime_parity_probe.json" + assert evidence["runtime_install_runtime"] == "/shared/bb-p3-root/phase3_vllm_verl_py312" + assert evidence["vllm_version"] == "target-version" + + +def test_runtime_evidence_ignores_mismatched_target_probe_and_scratch(tmp_path: Path) -> None: + evidence_root = tmp_path / "docs_tmp" + runs = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs" + scratch = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "scratch_runs" + introspection = runs / "phase3_verl_api_introspection" / "phase3_verl_api_introspection.json" + introspection.parent.mkdir(parents=True, exist_ok=True) + introspection.write_text("{}\n") + for rel in ( + "payloads/phase3_container_ppo_8gpu_stage/run.sh", + "payloads/phase3_container_grpo_8gpu_stage/run.sh", + "payloads/closed_loop_verl_train_stage/run.sh", + ): + script = runs / rel + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text('IMAGE="vllm/vllm-openai-rocm:nightly"\nsource /shared/bb-p3-root/phase3_vllm_verl_py312/bin/activate\n') + target_report = runs / "phase3_vllm_runtime_parity_probe" / "phase3_vllm_runtime_parity_probe.json" + _write_json(target_report, {"target_run_id": "other-run", "passed": True, "runtime": "/shared/bb-p3-root/phase3_vllm_verl_py312", "imports": {"vllm": "target-version"}}) + legacy_report = scratch / "phase3_verl_vllm_container_install" / "phase3_verl_vllm_container_install.json" + _write_json(legacy_report, {"target_run_id": "target-run", "passed": True, "runtime": "/scratch/legacy", "imports": {"vllm": "legacy-version"}}) + + evidence = _runtime_evidence( + evidence_root=evidence_root, + p1_report={"artifact_paths": {"introspection_report": "ZYPHRA/RL_PHASE_3/runs/phase3_verl_api_introspection/phase3_verl_api_introspection.json"}}, + target_run_id="target-run", + ) + + assert evidence["runtime_install_report_artifact"] == "" + assert evidence["runtime_install_runtime"] == "" + assert evidence["vllm_version"] == "" + +def _minimal_valid_parity_report(evidence_root: Path) -> dict: + artifact_paths = { + "reward_function": "ZYPHRA/RL_PHASE_3/runs/parity/reward.py", + "accepted_projection_rows": "ZYPHRA/RL_PHASE_3/runs/parity/accepted_projection_rows.jsonl", + "evidence_manifest": "ZYPHRA/RL_PHASE_3/runs/parity/evidence_manifest.json", + "metrics": "ZYPHRA/RL_PHASE_3/runs/parity/metrics.json", + "introspection_report": "ZYPHRA/RL_PHASE_3/runs/parity/introspection.json", + "runtime_ppo_script": "ZYPHRA/RL_PHASE_3/runs/parity/run_ppo.sh", + "runtime_grpo_script": "ZYPHRA/RL_PHASE_3/runs/parity/run_grpo.sh", + "runtime_closed_loop_script": "ZYPHRA/RL_PHASE_3/runs/parity/run_closed_loop.sh", + "runtime_install_report": "ZYPHRA/RL_PHASE_3/runs/parity/runtime_install.json", + } + for rel in artifact_paths.values(): + path = evidence_root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}\n") + return { + "schema_version": PHASE3_PARITY_SCHEMA, + "report_id": PHASE3_PARITY_REPORT_ID, + "claim_boundary": PHASE3_PARITY_CLAIM_BOUNDARY, + "target_run_id": "20260624T040000Z-slurm-243958", + "scorecard_update_allowed": False, + "passed": True, + "scorer": {"reward_function_sha256": "sha256:" + "1" * 64}, + "rollout": {"accepted_count": 1}, + "token_logprob": {"available": False}, + "checkpoint": { + "ppo": { + "optimizer_step_count": 1, + "checkpoint_changed": True, + "checkpoint_before_sha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "checkpoint_after_sha256": "sha256:" + "2" * 64, + }, + "grpo": { + "optimizer_step_count": 1, + "checkpoint_changed": True, + "checkpoint_before_sha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "checkpoint_after_sha256": "sha256:" + "3" * 64, + }, + "closed_loop": { + "optimizer_step_count": 1, + "checkpoint_changed": True, + "checkpoint_before_sha256": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "checkpoint_after_sha256": "sha256:" + "4" * 64, + }, + }, + "model_merge": {"required": False}, + "infra": { + "introspection": { + "torch": { + "cuda_available": True, + "device_count": 8, + "devices": ["AMD Instinct MI300X"] * 8, + }, + "symbols": {"verl.__version__": "0.8.0"}, + }, + }, + "dataproto": {"dataproto_ok": True, "evidence_manifest_sha256": "sha256:" + "5" * 64}, + "limitations": [], + "checklist": { + "C7_checkpoint_parity": {"status": "satisfied"}, + "C10_infrastructure_parity": {"status": "satisfied"}, + }, + "artifact_paths": artifact_paths, + "errors": [], + } + + +def test_parity_validator_rejects_open_checkpoint_checklist(tmp_path: Path) -> None: + evidence_root = tmp_path / "docs_tmp" + report = _minimal_valid_parity_report(evidence_root) + report["checklist"]["C7_checkpoint_parity"]["status"] = "open" + + errors = validate_phase3_parity_report(report, target_run_id=report["target_run_id"], evidence_root=evidence_root) + + assert "checklist.C7_checkpoint_parity.status must be satisfied" in errors + + +def test_parity_validator_rejects_open_infrastructure_checklist(tmp_path: Path) -> None: + evidence_root = tmp_path / "docs_tmp" + report = _minimal_valid_parity_report(evidence_root) + report["checklist"]["C10_infrastructure_parity"]["status"] = "open" + + errors = validate_phase3_parity_report(report, target_run_id=report["target_run_id"], evidence_root=evidence_root) + + assert "checklist.C10_infrastructure_parity.status must be satisfied" in errors diff --git a/tests/rl/phase3/test_phase4_native_inference_payload.py b/tests/rl/phase3/test_phase4_native_inference_payload.py new file mode 100644 index 00000000..fdf6ded7 --- /dev/null +++ b/tests/rl/phase3/test_phase4_native_inference_payload.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import json +import stat +import zipfile +from pathlib import Path + +from scripts.rl_phase3.build_phase4_native_inference_payload import REQUIRED_ZIP_ENTRIES, build_payload + + +REPO_ROOT = Path(__file__).resolve().parents[3] +WRAPPER_HEAD = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +VERL_HEAD = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +NEMO_HEAD = "cccccccccccccccccccccccccccccccccccccccc" + + +def _git_runner(args: list[str], cwd: Path) -> str: + cwd_text = str(cwd).replace("\\", "/") + if args == ["submodule", "status", "--recursive"]: + return f"{VERL_HEAD} third_party/verl (heads/main)\n{NEMO_HEAD} third_party/nemo-gym (heads/main)" + if args == ["rev-parse", "HEAD"]: + if cwd_text.endswith("third_party/verl"): + return VERL_HEAD + if cwd_text.endswith("third_party/nemo-gym"): + return NEMO_HEAD + return WRAPPER_HEAD + if args == ["rev-parse", "--abbrev-ref", "HEAD"]: + return "main" + return "" + + +def _write_canonical_wrapper(wrapper_root: Path, *, include_submodules: bool = True) -> None: + (wrapper_root / "src/zyphra_verl/configs").mkdir(parents=True) + (wrapper_root / "src/zyphra_verl/nemo_gym_loop.py").write_text( + 'register("nemo_gym_tool_use")\nToolParser\nToolCallComparator\nreward_score\n' + ) + (wrapper_root / "src/zyphra_verl/configs/agent_loops.yaml").write_text("agent_loop: native\n") + (wrapper_root / "deps.yaml").write_text(f"verl:\n pin: {VERL_HEAD}\nnemo_gym:\n pin: {NEMO_HEAD}\n") + if include_submodules: + (wrapper_root / "third_party" / "verl").mkdir(parents=True) + (wrapper_root / "third_party" / "verl" / "pyproject.toml").write_text("[project]\nname='verl'\n") + (wrapper_root / "third_party" / "nemo-gym").mkdir(parents=True) + (wrapper_root / "third_party" / "nemo-gym" / "pyproject.toml").write_text("[project]\nname='nemo-gym'\n") + + +def test_phase4_native_payload_zip_contains_native_lane_and_excludes_bytecode(tmp_path: Path) -> None: + source_payload_dir = tmp_path / "source_payload" + wrapper_root = source_payload_dir / "verl_wrapper" + _write_canonical_wrapper(wrapper_root) + (wrapper_root / "src/zyphra_verl/__pycache__").mkdir() + (wrapper_root / "src/zyphra_verl/__pycache__/nemo_gym_loop.cpython-312.pyc").write_bytes(b"bytecode") + (wrapper_root / "src/zyphra_verl/configs/cache.pyc").write_bytes(b"bytecode") + output_dir = tmp_path / "out" + output_dir.mkdir() + + report = build_payload( + repo_root=REPO_ROOT, + source_payload_dir=source_payload_dir, + output_dir=output_dir, + stamp="20260707T000000Z", + git_runner=_git_runner, + ) + + assert report["passed"] is True + assert report["errors"] == [] + persisted_report = json.loads(Path(report["report_path"]).read_text()) + assert persisted_report["passed"] is True + assert persisted_report["errors"] == [] + + stage_dir = Path(report["stage_dir"]) + run_sh = stage_dir / "run.sh" + assert run_sh.exists() + assert run_sh.stat().st_mode & stat.S_IXUSR + + zip_path = Path(report["zip_path"]) + with zipfile.ZipFile(zip_path) as archive: + names = set(archive.namelist()) + assert set(REQUIRED_ZIP_ENTRIES).issubset(names) + assert "repo/breadboard/rl/phase4/native_inference.py" in names + assert "verl_wrapper/src/zyphra_verl/nemo_gym_loop.py" in names + assert "verl_wrapper/src/zyphra_verl/configs/agent_loops.yaml" in names + assert "repo/breadboard/rl/phase4/wrapper_identity.py" in names + assert "verl_wrapper/wrapper_identity.json" in names + assert "verl_wrapper/third_party/verl/pyproject.toml" in names + assert "verl_wrapper/third_party/nemo-gym/pyproject.toml" in names + assert all("__pycache__" not in name for name in names) + assert all(not name.endswith(".pyc") for name in names) + run_info = archive.getinfo("run.sh") + assert (run_info.external_attr >> 16) & stat.S_IXUSR + native_source = archive.read("repo/breadboard/rl/phase4/native_inference.py").decode("utf-8") + assert "BREADBOARD_NATIVE_INFERENCE_OWNER" in native_source + assert "NativeInferenceLane" in native_source + + identity = json.loads(archive.read("verl_wrapper/wrapper_identity.json").decode("utf-8")) + assert identity["passed"] is True + assert identity["components"]["verl"]["actual_commit"] == VERL_HEAD + + +def test_phase4_native_payload_blocks_missing_exact_wrapper_submodules(tmp_path: Path) -> None: + source_payload_dir = tmp_path / "source_payload" + wrapper_root = source_payload_dir / "verl_wrapper" + _write_canonical_wrapper(wrapper_root, include_submodules=False) + output_dir = tmp_path / "out" + output_dir.mkdir() + + report = build_payload( + repo_root=REPO_ROOT, + source_payload_dir=source_payload_dir, + output_dir=output_dir, + stamp="20260707T000001Z", + git_runner=_git_runner, + ) + + assert report["passed"] is False + assert "wrapper identity manifest must pass" in report["errors"] + assert "submodule_path_missing:third_party/verl" in report["wrapper_identity"]["blockers"] \ No newline at end of file diff --git a/tests/rl/phase3/test_promotion_audit.py b/tests/rl/phase3/test_promotion_audit.py new file mode 100644 index 00000000..ab5b1941 --- /dev/null +++ b/tests/rl/phase3/test_promotion_audit.py @@ -0,0 +1,604 @@ +from __future__ import annotations +import json +import subprocess +import sys +from pathlib import Path + +from breadboard.rl.phase3.final_report import PHASE3_CORE_CLAIM_BOUNDARY, PHASE3_CORE_READINESS_SCHEMA, PHASE3_FINAL_CLAIM_BOUNDARY, PHASE3_FINAL_REPORT_ID, PHASE3_MILESTONES +from breadboard.rl.phase3.promotion_audit import build_phase3_promotion_audit, validate_phase3_core_promotion_audit, validate_phase3_promotion_audit + + +def _scorecard(points: int) -> dict: + return {"current_verified_points": points, "total_points": 1000} + + +def test_promotion_audit_surfaces_final_report_blockers() -> None: + target = "20260624T040000Z-slurm-243958" + final_report = { + "validation_errors": ["P3-M7: passed must be true", "P3-M11: passed must be true"], + "milestone_summaries": [ + {"milestone_id": "P3-M0", "passed": True}, + {"milestone_id": "P3-M7", "passed": False}, + {"milestone_id": "P3-M11", "passed": False}, + ], + } + + audit = build_phase3_promotion_audit( + target_run_id=target, + final_report=final_report, + scorecard=_scorecard(0), + claim_ledger_text="", + bd_epic_closed=False, + ) + + assert audit["promotion_review_ready"] is False + assert audit["completed_milestones"] == ["P3-M0"] + assert audit["blocked_milestones"] == ["P3-M7", "P3-M11"] + assert audit["final_report_validation_errors"] == final_report["validation_errors"] + errors = validate_phase3_promotion_audit(audit) + assert "promotion_review_ready must be true" in errors + assert "every P3 milestone must be completed" in errors + + +def test_promotion_audit_ready_requires_all_phase3_gates() -> None: + target = "20260624T040000Z-slurm-243958" + active_milestones = list(PHASE3_MILESTONES) + final_report = { + "validation_errors": [], + "milestone_summaries": [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES], + "core_readiness": { + "schema_version": PHASE3_CORE_READINESS_SCHEMA, + "claim_boundary": PHASE3_CORE_CLAIM_BOUNDARY, + "ready": True, + "scorecard_update_allowed": False, + "active_milestones": active_milestones, + "core_milestones": active_milestones, + "deferred_milestones": [], + "blocked_active_milestones": [], + "blocked_core_milestones": [], + "core_raw_points_verified": 0, + "core_raw_points_total": 0, + "original_scorecard_total_points": 1000, + }, + } + ledger = f"{target}\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n{PHASE3_CORE_CLAIM_BOUNDARY}\n" + + audit = build_phase3_promotion_audit( + target_run_id=target, + final_report=final_report, + scorecard=_scorecard(1000), + claim_ledger_text=ledger, + bd_epic_closed=True, + ) + + assert "P3-M11" in PHASE3_MILESTONES + assert "P3-M11" in final_report["core_readiness"]["core_milestones"] + assert audit["promotion_review_ready"] is True + assert audit["active_review_ready"] is True + assert audit["active_artifact_audit_clean"] is True + assert audit["core_artifact_audit_clean"] is True + assert audit["active_completed_milestones"] == active_milestones + assert audit["blocked_milestones"] == [] + assert audit["scorecard_update_allowed"] is False + assert audit["final_report_validation_errors"] == [] + assert audit["core_validation_errors"] == [] + assert validate_phase3_promotion_audit(audit) == [] + + +def test_active_audit_ready_but_promotion_gated_until_epic_closed() -> None: + target = "20260624T040000Z-slurm-243958" + active_milestones = list(PHASE3_MILESTONES) + final_report = { + "validation_errors": [], + "milestone_summaries": [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES], + "core_readiness": { + "schema_version": PHASE3_CORE_READINESS_SCHEMA, + "claim_boundary": PHASE3_CORE_CLAIM_BOUNDARY, + "ready": True, + "scorecard_update_allowed": False, + "active_milestones": active_milestones, + "core_milestones": active_milestones, + "deferred_milestones": [], + "blocked_active_milestones": [], + "blocked_core_milestones": [], + "core_raw_points_verified": 0, + "core_raw_points_total": 0, + "original_scorecard_total_points": 1000, + }, + } + ledger = f"{target}\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n{PHASE3_CORE_CLAIM_BOUNDARY}\n" + + audit = build_phase3_promotion_audit( + target_run_id=target, + final_report=final_report, + scorecard=_scorecard(0), + claim_ledger_text=ledger, + bd_epic_closed=False, + ) + + assert audit["promotion_review_ready"] is False + assert audit["active_review_ready"] is True + assert audit["active_artifact_audit_clean"] is True + assert audit["active_review_ready_meaning"] == "Artifact-audit boundary is clean for the promoted exact-scope Phase 3 claim; broader successor claims remain separately gated." + assert audit["completed_milestones"] == active_milestones + assert audit["blocked_milestones"] == [] + assert audit["scorecard_update_allowed"] is False + assert audit["core_scorecard_update_allowed"] is False + assert audit["active_completed_milestones"] == active_milestones + assert audit["core_blocked_milestones"] == [] + assert audit["core_claim_ledger_anchored"] is True + assert audit["core_validation_errors"] == [] + assert validate_phase3_core_promotion_audit(audit) == [] + full_errors = validate_phase3_promotion_audit(audit) + assert "bd epic must be closed" in full_errors + + unanchored = build_phase3_promotion_audit( + target_run_id=target, + final_report=final_report, + scorecard=_scorecard(0), + claim_ledger_text=f"{target}\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n", + bd_epic_closed=False, + ) + assert unanchored["core_review_ready"] is False + assert "active claim boundary must be anchored in claim ledger" in unanchored["core_validation_errors"] + +def test_promotion_audit_rejects_missing_target_run_id() -> None: + audit = { + "schema_version": "bb.rl.phase3.promotion_audit.v1", + "report_id": "bb_zyphra_rl_phase3_promotion_audit_v1", + "claim_boundary": "phase3_promotion_review_only_not_scorecard_update", + "target_run_id": "", + "completed_milestones": list(PHASE3_MILESTONES), + "blocked_milestones": [], + "final_report_validation_errors": [], + "bd_epic_closed": True, + "promotion_review_ready": True, + "scorecard_update_allowed": False, + "core_scorecard_update_allowed": False, + } + + assert "target_run_id must match Phase 3 Slurm target run id pattern" in validate_phase3_promotion_audit(audit) + + final_report = { + "validation_errors": [], + "milestone_summaries": [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES], + } + audit_from_builder = build_phase3_promotion_audit( + target_run_id="", + final_report=final_report, + scorecard=_scorecard(1000), + claim_ledger_text=f"{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n", + bd_epic_closed=True, + ) + assert audit_from_builder["promotion_review_ready"] is False + + +def test_promotion_audit_non_string_target_run_id_blocks_without_crashing() -> None: + final_report = { + "validation_errors": [], + "milestone_summaries": [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES], + } + + audit = build_phase3_promotion_audit( + target_run_id=12345, # type: ignore[arg-type] + final_report=final_report, + scorecard=_scorecard(1000), + claim_ledger_text=f"12345\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n", + bd_epic_closed=True, + ) + + assert audit["promotion_review_ready"] is False + assert "target_run_id must match Phase 3 Slurm target run id pattern" in validate_phase3_promotion_audit(audit) + + +def test_promotion_audit_rejects_ready_with_stale_blockers() -> None: + audit = { + "schema_version": "bb.rl.phase3.promotion_audit.v1", + "report_id": "bb_zyphra_rl_phase3_promotion_audit_v1", + "claim_boundary": "phase3_promotion_review_only_not_scorecard_update", + "target_run_id": "20260624T040000Z-slurm-243958", + "completed_milestones": list(PHASE3_MILESTONES), + "blocked_milestones": ["P3-M7"], + "final_report_validation_errors": ["P3-M7: passed must be true"], + "bd_epic_closed": True, + "promotion_review_ready": True, + "scorecard_update_allowed": False, + "core_scorecard_update_allowed": False, + } + + errors = validate_phase3_promotion_audit(audit) + assert "ready promotion audit must not list blocked_milestones" in errors + assert "ready promotion audit must not list final_report_validation_errors" in errors + + +def test_promotion_audit_rejects_wrong_schema_version() -> None: + audit = { + "schema_version": "wrong", + "report_id": "bb_zyphra_rl_phase3_promotion_audit_v1", + "claim_boundary": "phase3_promotion_review_only_not_scorecard_update", + "target_run_id": "20260624T040000Z-slurm-243958", + "completed_milestones": list(PHASE3_MILESTONES), + "blocked_milestones": [], + "final_report_validation_errors": [], + "bd_epic_closed": True, + "promotion_review_ready": True, + "scorecard_update_allowed": False, + "core_scorecard_update_allowed": False, + } + + assert "schema_version must be Phase 3 promotion audit schema" in validate_phase3_promotion_audit(audit) + + +def test_promotion_audit_malformed_final_report_shape_blocks_without_crashing() -> None: + audit = build_phase3_promotion_audit( + target_run_id="20260624T040000Z-slurm-243958", + final_report={"validation_errors": "stale error", "milestone_summaries": {"P3-M0": {"passed": True}}}, + scorecard=_scorecard(1000), + claim_ledger_text=f"20260624T040000Z-slurm-243958\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n", + bd_epic_closed=True, + ) + + assert audit["promotion_review_ready"] is False + assert audit["completed_milestones"] == [] + assert audit["final_report_validation_errors"] == ["stale error"] + + +def test_promotion_audit_malformed_scorecard_blocks_without_crashing() -> None: + final_report = { + "validation_errors": [], + "milestone_summaries": [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES], + } + + audit = build_phase3_promotion_audit( + target_run_id="20260624T040000Z-slurm-243958", + final_report=final_report, + scorecard={"current_verified_points": "oops", "total_points": "also-oops"}, + claim_ledger_text=f"20260624T040000Z-slurm-243958\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n", + bd_epic_closed=True, + ) + + assert audit["promotion_review_ready"] is True + + +def test_promotion_audit_non_string_claim_ledger_blocks_without_crashing() -> None: + non_string_ledger = { + "text": "20260624T040000Z-slurm-243958\nbb_zyphra_rl_phase3_final_report_v1\nphase3_final_report_claim_boundary" + } + audit = build_phase3_promotion_audit( + target_run_id="20260624T040000Z-slurm-243958", + final_report={ + "validation_errors": [], + "milestone_summaries": [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES], + }, + scorecard=_scorecard(1000), + claim_ledger_text=non_string_ledger, # type: ignore[arg-type] + bd_epic_closed=True, + ) + + assert audit["promotion_review_ready"] is False + + +def test_promotion_audit_duplicate_failed_summary_blocks_readiness() -> None: + target = "20260624T040000Z-slurm-243958" + summaries = [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES] + summaries.append({"milestone_id": "P3-M7", "passed": False}) + + audit = build_phase3_promotion_audit( + target_run_id=target, + final_report={"validation_errors": [], "milestone_summaries": summaries}, + scorecard=_scorecard(1000), + claim_ledger_text=f"{target}\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n", + bd_epic_closed=True, + ) + + assert audit["promotion_review_ready"] is False + assert audit["blocked_milestones"] == ["P3-M7"] + assert "ready promotion audit must not list blocked_milestones" not in validate_phase3_promotion_audit(audit) + + + + +def test_promotion_audit_duplicate_passed_summary_blocks_readiness() -> None: + target = "20260624T040000Z-slurm-243958" + summaries = [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES] + summaries.append({"milestone_id": "P3-M0", "passed": True}) + + audit = build_phase3_promotion_audit( + target_run_id=target, + final_report={"validation_errors": [], "milestone_summaries": summaries}, + scorecard=_scorecard(1000), + claim_ledger_text=f"{target}\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n", + bd_epic_closed=True, + ) + + assert audit["promotion_review_ready"] is False + assert audit["completed_milestones"].count("P3-M0") == 2 + assert "every P3 milestone must be completed" in validate_phase3_promotion_audit(audit) + + + + +def test_promotion_audit_validator_rejects_malformed_ready_lists() -> None: + audit = { + "schema_version": "bb.rl.phase3.promotion_audit.v1", + "report_id": "bb_zyphra_rl_phase3_promotion_audit_v1", + "claim_boundary": "phase3_promotion_review_only_not_scorecard_update", + "target_run_id": "20260624T040000Z-slurm-243958", + "completed_milestones": [*PHASE3_MILESTONES, {"bad": 1}], + "blocked_milestones": {}, + "final_report_validation_errors": "", + "bd_epic_closed": True, + "promotion_review_ready": True, + "scorecard_update_allowed": False, + "core_scorecard_update_allowed": False, + } + + errors = validate_phase3_promotion_audit(audit) + assert "blocked_milestones must be a list of strings" in errors + assert "final_report_validation_errors must be a list of strings" in errors + assert "every P3 milestone must be completed" in errors + + +def test_promotion_audit_validator_handles_unhashable_completed_milestones() -> None: + audit = { + "schema_version": "bb.rl.phase3.promotion_audit.v1", + "report_id": "bb_zyphra_rl_phase3_promotion_audit_v1", + "claim_boundary": "phase3_promotion_review_only_not_scorecard_update", + "target_run_id": "20260624T040000Z-slurm-243958", + "completed_milestones": [{"bad": 1}], + "blocked_milestones": [], + "final_report_validation_errors": [], + "bd_epic_closed": True, + "promotion_review_ready": True, + "scorecard_update_allowed": False, + "core_scorecard_update_allowed": False, + } + + assert "every P3 milestone must be completed" in validate_phase3_promotion_audit(audit) + +def test_promotion_audit_unhashable_milestone_ids_block_without_crashing() -> None: + audit = build_phase3_promotion_audit( + target_run_id="20260624T040000Z-slurm-243958", + final_report={ + "validation_errors": [], + "milestone_summaries": [{"milestone_id": {"bad": 1}, "passed": True}], + }, + scorecard=_scorecard(1000), + claim_ledger_text=f"20260624T040000Z-slurm-243958\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n", + bd_epic_closed=True, + ) + + assert audit["promotion_review_ready"] is False + assert audit["completed_milestones"] == [] + assert "every P3 milestone must be completed" in validate_phase3_promotion_audit(audit) + +def test_audit_cli_revalidates_stale_final_report_errors(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + runs = phase_dir / "runs" + runs.mkdir(parents=True) + final_report = { + "report_id": PHASE3_FINAL_REPORT_ID, + "claim_boundary": PHASE3_FINAL_CLAIM_BOUNDARY, + "target_run_id": "20260624T040000Z-slurm-243958", + "validation_errors": [], + "scorecard_update_allowed": False, + "command_log_manifest": {}, + "milestone_reports": {}, + "milestone_summaries": [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES], + "scorecard": {"current_verified_points": 1000, "total_points": 1000}, + "claim_ledger_text": "20260624T040000Z-slurm-243958\nphase3_final_report_claim_boundary\n", + } + (runs / "p3_m12_final_report.json").write_text(json.dumps(final_report)) + (phase_dir / "BB_ZYPHRA_RL_PHASE_3_CLAIM_LEDGER.md").write_text( + f"20260624T040000Z-slurm-243958\n{PHASE3_FINAL_REPORT_ID}\n{PHASE3_FINAL_CLAIM_BOUNDARY}\n" + ) + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/audit_phase3_promotion.py", + "--phase-dir", + str(phase_dir), + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0 + audit = json.loads((runs / "p3_m12_promotion_audit.json").read_text()) + assert audit["promotion_review_ready"] is False + assert "commands must contain at least one command row" in audit["final_report_validation_errors"] + + +def test_audit_cli_blocks_malformed_final_report_json(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + runs = phase_dir / "runs" + runs.mkdir(parents=True) + (runs / "p3_m12_final_report.json").write_text("{not json") + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/audit_phase3_promotion.py", + "--phase-dir", + str(phase_dir), + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0 + audit = json.loads((runs / "p3_m12_promotion_audit.json").read_text()) + assert audit["promotion_review_ready"] is False + assert "JSONDecodeError" in audit["final_report_read_error"] + assert audit["final_report_validation_errors"][0].startswith("final report is not readable JSON: JSONDecodeError") + + +def test_audit_cli_blocks_non_object_final_report_json(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + runs = phase_dir / "runs" + runs.mkdir(parents=True) + (runs / "p3_m12_final_report.json").write_text(json.dumps([])) + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/audit_phase3_promotion.py", + "--phase-dir", + str(phase_dir), + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0 + audit = json.loads((runs / "p3_m12_promotion_audit.json").read_text()) + assert audit["promotion_review_ready"] is False + assert audit["final_report_validation_errors"] == ["final report must be a JSON object"] + + +def test_audit_cli_blocks_null_final_report_json(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + runs = phase_dir / "runs" + runs.mkdir(parents=True) + (runs / "p3_m12_final_report.json").write_text("null") + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/audit_phase3_promotion.py", + "--phase-dir", + str(phase_dir), + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0 + audit = json.loads((runs / "p3_m12_promotion_audit.json").read_text()) + assert audit["promotion_review_ready"] is False + assert audit["final_report_validation_errors"] == ["final report must be a JSON object"] + + +def test_audit_cli_blocks_unreadable_claim_ledger(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + runs = phase_dir / "runs" + runs.mkdir(parents=True) + final_report = { + "report_id": PHASE3_FINAL_REPORT_ID, + "claim_boundary": PHASE3_FINAL_CLAIM_BOUNDARY, + "target_run_id": "20260624T040000Z-slurm-243958", + "validation_errors": [], + "scorecard_update_allowed": False, + "command_log_manifest": {}, + "milestone_reports": {}, + "milestone_summaries": [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES], + "scorecard": {"current_verified_points": 1000, "total_points": 1000}, + "claim_ledger_text": "20260624T040000Z-slurm-243958\nphase3_final_report_claim_boundary\n", + } + (runs / "p3_m12_final_report.json").write_text(json.dumps(final_report)) + (phase_dir / "BB_ZYPHRA_RL_PHASE_3_CLAIM_LEDGER.md").mkdir() + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/audit_phase3_promotion.py", + "--phase-dir", + str(phase_dir), + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0 + audit = json.loads((runs / "p3_m12_promotion_audit.json").read_text()) + assert audit["promotion_review_ready"] is False + assert "IsADirectoryError" in audit["claim_ledger_read_error"] + + +def test_audit_cli_revalidates_empty_final_report_object(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + runs = phase_dir / "runs" + runs.mkdir(parents=True) + (runs / "p3_m12_final_report.json").write_text("{}") + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/audit_phase3_promotion.py", + "--phase-dir", + str(phase_dir), + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0 + audit = json.loads((runs / "p3_m12_promotion_audit.json").read_text()) + assert audit["promotion_review_ready"] is False + assert "report_id must be Phase 3 final report id" in audit["final_report_validation_errors"] + + +def test_audit_cli_blocks_missing_final_report(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + (phase_dir / "runs").mkdir(parents=True) + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/audit_phase3_promotion.py", + "--phase-dir", + str(phase_dir), + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0 + audit = json.loads((phase_dir / "runs" / "p3_m12_promotion_audit.json").read_text()) + assert audit["promotion_review_ready"] is False + assert "FileNotFoundError" in audit["final_report_read_error"] + assert audit["final_report_validation_errors"][0].startswith("final report is missing: FileNotFoundError") + + +def test_audit_cli_blocks_missing_claim_ledger(tmp_path: Path) -> None: + phase_dir = tmp_path / "docs_tmp" / "ZYPHRA" / "RL_PHASE_3" + runs = phase_dir / "runs" + runs.mkdir(parents=True) + final_report = { + "report_id": PHASE3_FINAL_REPORT_ID, + "claim_boundary": PHASE3_FINAL_CLAIM_BOUNDARY, + "target_run_id": "20260624T040000Z-slurm-243958", + "validation_errors": [], + "scorecard_update_allowed": False, + "command_log_manifest": {}, + "milestone_reports": {}, + "milestone_summaries": [{"milestone_id": milestone, "passed": True} for milestone in PHASE3_MILESTONES], + "scorecard": {"current_verified_points": 1000, "total_points": 1000}, + "claim_ledger_text": "20260624T040000Z-slurm-243958\nphase3_final_report_claim_boundary\n", + } + (runs / "p3_m12_final_report.json").write_text(json.dumps(final_report)) + + result = subprocess.run( + [ + sys.executable, + "scripts/rl_phase3/audit_phase3_promotion.py", + "--phase-dir", + str(phase_dir), + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0 + audit = json.loads((runs / "p3_m12_promotion_audit.json").read_text()) + assert audit["promotion_review_ready"] is False + assert "FileNotFoundError" in audit["claim_ledger_read_error"] diff --git a/tests/rl/phase3/test_runner_contract_and_nemo_smoke.py b/tests/rl/phase3/test_runner_contract_and_nemo_smoke.py new file mode 100644 index 00000000..62a0d4a2 --- /dev/null +++ b/tests/rl/phase3/test_runner_contract_and_nemo_smoke.py @@ -0,0 +1,259 @@ +from __future__ import annotations +import shutil + +from pathlib import Path + +from scripts.rl_phase3.build_phase3_runner_contract import _parse_deps_yaml, build_contract +from scripts.rl_phase3.run_phase3_nemo_agentloop_smoke import build_smoke_report + +TARGET = "20260624T040000Z-slurm-243958" +WRAPPER_HEAD = "98b09b5603d2ef84ffa8ac0baa0ee55448a69122" +VERL_HEAD = "abc123" +NEMO_HEAD = "def456" + + +def _git_runner(marker: str = ""): + def run(args: list[str], cwd: Path) -> str: + cwd_text = str(cwd).replace("\\", "/") + if args == ["submodule", "status", "--recursive"]: + return f"{marker}{VERL_HEAD} third_party/verl (heads/main)\n{marker}{NEMO_HEAD} third_party/nemo-gym (heads/main)" + if args == ["rev-parse", "HEAD"]: + if cwd_text.endswith("third_party/verl"): + return VERL_HEAD + if cwd_text.endswith("third_party/nemo-gym"): + return NEMO_HEAD + return WRAPPER_HEAD + if args == ["rev-parse", "--abbrev-ref", "HEAD"]: + return "main" + return "" + return run + + + +def _wrapper(tmp_path: Path) -> Path: + root = tmp_path / "verl_wrapper" + (root / "patches" / "verl").mkdir(parents=True) + (root / "patches" / "verl" / "0001.patch").write_text("patch bytes\n") + (root / "src" / "zyphra_verl").mkdir(parents=True) + (root / "src" / "zyphra_verl" / "nemo_gym_loop.py").write_text( + 'register("nemo_gym_tool_use")\nToolParser\nToolCallComparator\nreward_score\n' + ) + (root / "launch").mkdir() + (root / "launch" / "train.sh").write_text("#!/usr/bin/env bash\n") + (root / "deps.yaml").write_text( + "verl:\n pin: abc123\nnemo_gym:\n pin: def456\n" + ) + (root / ".gitmodules").write_text("[submodule]\n") + (root / "pyproject.toml").write_text("[project]\nname='x'\n") + (root / "third_party" / "verl").mkdir(parents=True) + (root / "third_party" / "verl" / "pyproject.toml").write_text("[project]\nname='verl'\n") + (root / "third_party" / "nemo-gym").mkdir(parents=True) + (root / "third_party" / "nemo-gym" / "pyproject.toml").write_text("[project]\nname='nemo-gym'\n") + return root + + +def test_runner_contract_deps_parser_strips_inline_comments(tmp_path: Path) -> None: + deps = tmp_path / "deps.yaml" + deps.write_text( + "\n".join( + [ + "verl:", + " pin: ed89419c23653730e95c43954c00e6c24277e1c8 # v0.8.0 branch", + "nemo_gym:", + " pin: 'fd1c91cb83256546bdef024d7dbd013c377748539' # canonical", + "reward_models:", + ' commit: \"abc#inside\" # keep quoted hash marker', + "trainer:", + " rev: main # comment", + "ignored:", + " image: should-not-appear # unsupported key", + ] + ) + + "\n" + ) + + assert _parse_deps_yaml(deps) == { + "verl_pin": "ed89419c23653730e95c43954c00e6c24277e1c8", + "nemo_gym_pin": "fd1c91cb83256546bdef024d7dbd013c377748539", + "reward_models_commit": "abc#inside", + "trainer_rev": "main", + } + + +def test_runner_contract_deps_parser_missing_file_returns_empty(tmp_path: Path) -> None: + assert _parse_deps_yaml(tmp_path / "missing.yaml") == {} + + +def test_runner_contract_requires_immutable_container_digest(tmp_path: Path) -> None: + wrapper = _wrapper(tmp_path) + + report = build_contract( + wrapper_dir=wrapper, + target_run_id=TARGET, + container_image="vllm/vllm-openai-rocm:nightly", + container_digest="", + launch_command="launch/train.sh STEPS=1", + git_runner=_git_runner(), + ) + + assert report["passed"] is False + assert "container_digest" in report["missing_required_identities"] + assert report["required_identities"]["verl_pin"] == "abc123" + assert report["required_identities"]["nemo_gym_pin"] == "def456" + assert report["patch_queue_sha256"].startswith("sha256:") + assert report["recipe_package_sha256"].startswith("sha256:") + + +def test_runner_contract_passes_with_complete_identity(tmp_path: Path) -> None: + wrapper = _wrapper(tmp_path) + + report = build_contract( + wrapper_dir=wrapper, + target_run_id=TARGET, + container_image="vllm/vllm-openai-rocm:nightly", + container_digest="vllm/vllm-openai-rocm@sha256:abc", + launch_command="launch/train.sh STEPS=1", + git_runner=_git_runner(), + ) + + assert report["passed"] is True + assert report["missing_required_identities"] == [] + + +def test_runner_contract_blocks_yaml_pins_without_checked_out_submodules(tmp_path: Path) -> None: + wrapper = _wrapper(tmp_path) + shutil.rmtree(wrapper / "third_party") + + report = build_contract( + wrapper_dir=wrapper, + target_run_id=TARGET, + container_image="vllm/vllm-openai-rocm:nightly", + container_digest="vllm/vllm-openai-rocm@sha256:abc", + launch_command="launch/train.sh STEPS=1", + git_runner=_git_runner(), + ) + + assert report["passed"] is False + assert "submodule_path_missing:third_party/verl" in report["wrapper_identity_blockers"] + + +def test_runner_contract_blocks_uninitialized_submodule_marker(tmp_path: Path) -> None: + wrapper = _wrapper(tmp_path) + + report = build_contract( + wrapper_dir=wrapper, + target_run_id=TARGET, + container_image="vllm/vllm-openai-rocm:nightly", + container_digest="vllm/vllm-openai-rocm@sha256:abc", + launch_command="launch/train.sh STEPS=1", + git_runner=_git_runner("-"), + ) + + assert report["passed"] is False + assert "submodule_uninitialized:third_party/verl" in report["wrapper_identity_blockers"] + +def test_nemo_agentloop_smoke_records_controls_and_hashes(tmp_path: Path) -> None: + wrapper = _wrapper(tmp_path) + + report = build_smoke_report( + wrapper_dir=wrapper, + target_run_id=TARGET, + require_canonical_runtime=False, + local_diagnostic=False, + ) + + assert report["controls"] == { + "gold_reward": 1.0, + "wrong_name_reward": 0.0, + "wrong_args_reward": 0.0, + "missing_call_reward": 0.0, + } + assert report["dependency_status"]["canonical_agentloop_source_present"] is True + assert report["hashes"]["row_sha256"].startswith("sha256:") + assert report["mode"] == "diagnostic" + assert report["diagnostic_passed"] is True + assert report["passed"] is False + assert report["promotional"] is False + assert "diagnostic_only_not_promotional" in report["blocked_reason"] + + +def test_nemo_agentloop_local_diagnostic_cannot_promote(tmp_path: Path) -> None: + wrapper = _wrapper(tmp_path) + + report = build_smoke_report( + wrapper_dir=wrapper, + target_run_id=TARGET, + require_canonical_runtime=False, + local_diagnostic=True, + ) + + assert report["passed"] is False + assert "local_diagnostic_not_promotional" in report["blocked_reason"] + + +def test_nemo_agentloop_canonical_mode_blocks_when_runtime_missing(tmp_path: Path, monkeypatch) -> None: + wrapper = _wrapper(tmp_path) + monkeypatch.setattr( + "scripts.rl_phase3.run_phase3_nemo_agentloop_smoke._dependency_status", + lambda wrapper_dir: { + "wrapper_dir": str(wrapper_dir), + "nemo_gym_loop_path": str(wrapper_dir / "src" / "zyphra_verl" / "nemo_gym_loop.py"), + "canonical_agentloop_source_present": True, + "nemo_gym_loop_sha256": "sha256:test", + "required_source_terms": ["register(\"nemo_gym_tool_use\"", "ToolParser", "ToolCallComparator", "reward_score"], + "canonical_runtime_imports_present": False, + "runtime_import_blocked_reason": "ModuleNotFoundError", + }, + ) + + report = build_smoke_report( + wrapper_dir=wrapper, + target_run_id=TARGET, + require_canonical_runtime=True, + local_diagnostic=False, + canonical_mode=True, + ) + + assert report["mode"] == "canonical" + assert report["passed"] is False + assert report["canonical_runtime_required"] is True + assert report["canonical_agentloop_executed"] is False + assert report["canonical_tool_parser_used"] is False + assert report["canonical_comparator_used"] is False + assert report["canonical_reward_score_observed"] is False + assert report["breadboard_toy_reward_used"] is True + assert report["scorecard_update_allowed"] is False + assert "canonical_runtime_imports_missing" in report["blocked_reason"] + + +def test_nemo_agentloop_canonical_mode_blocks_when_execution_missing(tmp_path: Path, monkeypatch) -> None: + wrapper = _wrapper(tmp_path) + monkeypatch.setattr( + "scripts.rl_phase3.run_phase3_nemo_agentloop_smoke._dependency_status", + lambda wrapper_dir: { + "wrapper_dir": str(wrapper_dir), + "nemo_gym_loop_path": str(wrapper_dir / "src" / "zyphra_verl" / "nemo_gym_loop.py"), + "canonical_agentloop_source_present": True, + "nemo_gym_loop_sha256": "sha256:test", + "required_source_terms": ["register(\"nemo_gym_tool_use\"", "ToolParser", "ToolCallComparator", "reward_score"], + "canonical_runtime_imports_present": True, + "runtime_import_blocked_reason": "", + }, + ) + + report = build_smoke_report( + wrapper_dir=wrapper, + target_run_id=TARGET, + require_canonical_runtime=True, + local_diagnostic=False, + canonical_mode=True, + ) + + assert report["mode"] == "canonical" + assert report["passed"] is False + assert report["canonical_runtime_required"] is True + assert report["canonical_agentloop_executed"] is False + assert "canonical_runtime_imports_missing" not in report["blocked_reason"] + assert "canonical_agentloop_execution_missing" in report["blocked_reason"] + + diff --git a/tests/rl/phase3/test_scheduler_observability_store.py b/tests/rl/phase3/test_scheduler_observability_store.py new file mode 100644 index 00000000..6c0ede2d --- /dev/null +++ b/tests/rl/phase3/test_scheduler_observability_store.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from breadboard.rl.phase3.object_store import LocalObjectStore +from breadboard.rl.phase3.observability_live import build_live_observability_report +from breadboard.rl.phase3.scheduler import parse_sbatch_job_id + +TARGET = "20260623T000000Z-slurm-234555" + +def _live_inputs(**overrides): + inputs = { + "target_run_id": TARGET, + "slurm_metrics": {"source": "slurm_sacct", "sacct_stdout": "ok", "queue_wait_seconds": 0, "scheduler_retry_count": 0}, + "gpu_metrics": {"source": "rocm_smi", "gpu_utilization": {"card0": {"GPU use (%)": "1"}}}, + "verifier_metrics": {"source": "verifier_client", "verifier_latency_seconds": [0.1]}, + "service_metrics": {"source": "service_event_log", "task_throughput": 1}, + "object_store_metrics": {"source": "object_store", "object_store": "production_object_store", "object_store_writes": 1, "artifact_bytes": 1}, + "budget_caps": {"remaining_usd": 1}, + "scheduler_metrics": { + "source": "scheduler_control", + "scheduler_control": {"endpoint_present": True, "token_present": True, "status": "ready"}, + "env_presence": { + "BREADBOARD_SCHEDULER_BASE_URL": True, + "BREADBOARD_SCHEDULER_TOKEN": True, + }, + }, + } + inputs.update(overrides) + return inputs + + +def test_slurm_parser_behavior() -> None: + assert parse_sbatch_job_id("Submitted batch job 12345") == "12345" + assert parse_sbatch_job_id("12345;gpu") == "12345" + + +def test_object_store_hash_validation(tmp_path: Path) -> None: + source = tmp_path / "source.txt"; source.write_text("payload") + store = LocalObjectStore(tmp_path / "store") + stat = store.put_file(source, artifact_id="artifact", metadata={"kind": "test"}) + restored = store.get_file("artifact", tmp_path / "restored.txt") + assert stat["sha256"] == store.stat("artifact")["sha256"] + assert restored.read_text() == "payload" + + +def test_metric_source_rejection() -> None: + report = build_live_observability_report(**_live_inputs(slurm_metrics={"source": "caller"})) + assert report["passed"] is False + assert any("source" in error for error in report["errors"]) + + +def test_metric_source_rejects_valid_but_wrong_section_source() -> None: + report = build_live_observability_report( + **_live_inputs(slurm_metrics={"source": "rocm_smi", "sacct_stdout": "ok", "queue_wait_seconds": 0, "scheduler_retry_count": 0}) + ) + + assert report["passed"] is False + assert "slurm_metrics.source must be 'slurm_sacct'" in report["errors"] + + + +def test_scheduler_metrics_requires_endpoint_and_token() -> None: + report = build_live_observability_report( + **_live_inputs( + scheduler_metrics={ + "source": "scheduler_control", + "scheduler_control": {"status": "connected"}, + } + ) + ) + + assert report["passed"] is False + assert "scheduler_control_endpoint_missing" in report["errors"] + assert "scheduler_control_token_missing" in report["errors"] + +def test_budget_cap_enforcement() -> None: + report = build_live_observability_report(**_live_inputs(budget_caps={"remaining_usd": -1})) + assert report["passed"] is False + assert any("budget" in error for error in report["errors"]) + + +def test_budget_cap_requires_remaining_usd() -> None: + report = build_live_observability_report(**_live_inputs(budget_caps={})) + + assert report["passed"] is False + assert "budget_caps.remaining_usd_missing" in report["errors"] + + +def test_budget_cap_rejects_invalid_remaining_usd() -> None: + report = build_live_observability_report(**_live_inputs(budget_caps={"remaining_usd": "not-a-number"})) + + assert report["passed"] is False + assert "budget_caps.remaining_usd_invalid" in report["errors"] diff --git a/tests/rl/phase3/test_security_enforcement.py b/tests/rl/phase3/test_security_enforcement.py new file mode 100644 index 00000000..42a69f18 --- /dev/null +++ b/tests/rl/phase3/test_security_enforcement.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import pytest + +from breadboard.rl.phase2.hardening import ArtifactEgressRequest, EgressPolicy, redact_mapping +from breadboard.rl.phase3.security_enforcement import enforce_artifact_egress, enforce_command_request, enforce_workspace_path + + +def policy() -> EgressPolicy: + return EgressPolicy(allowed_prefixes=("ws/replay",), max_artifact_bytes=100) + + +def test_denied_absolute_path() -> None: + with pytest.raises(PermissionError): + enforce_workspace_path("/tmp/x", tenant_id="t", workspace_id="ws") + + +def test_denied_parent_escape() -> None: + with pytest.raises(PermissionError): + enforce_workspace_path("ws/../secret", tenant_id="t", workspace_id="ws") + + +def test_denied_egress_classification() -> None: + with pytest.raises(PermissionError, match="artifact egress denied"): + enforce_artifact_egress(ArtifactEgressRequest("ws/replay/a.json", 10, "secret"), policy()) + + +def test_denied_oversize_artifact() -> None: + with pytest.raises(PermissionError): + enforce_artifact_egress(ArtifactEgressRequest("ws/replay/a.json", 101, "tenant_internal"), policy()) + + +def test_denied_destructive_command() -> None: + with pytest.raises(PermissionError): + enforce_command_request(["rm", "-rf", "/"], workspace_relative_path="ws", workspace_id="ws") + + +def test_redaction_secret_and_path_values() -> None: + result = redact_mapping({"api_token": "abc", "path": "/tmp/secret", "ok": "value"}) + assert result["api_token"] == "" + assert result["path"] == "" + assert result["ok"] == "value" + + +def test_valid_artifact_replay_path() -> None: + assert enforce_workspace_path("ws/replay/a.json", tenant_id="t", workspace_id="ws").as_posix() == "ws/replay/a.json" + enforce_artifact_egress(ArtifactEgressRequest("ws/replay/a.json", 10, "tenant_internal"), policy()) diff --git a/tests/rl/phase3/test_target_command_runner.py b/tests/rl/phase3/test_target_command_runner.py new file mode 100644 index 00000000..b74c7e88 --- /dev/null +++ b/tests/rl/phase3/test_target_command_runner.py @@ -0,0 +1,775 @@ +from __future__ import annotations + +import hashlib +import json +import shlex +import subprocess +from pathlib import Path + +from scripts.rl_phase3.run_phase3_target_command import _build_remote_command, _build_ssh_command, _safe_artifact_name, _validated_slurm_option, main + + +def test_ssh_command_quotes_remote_script_as_single_bash_argument() -> None: + remote = "set -euo pipefail; export PHASE3_TARGET_RUN_ID=run id; echo ok" + command = _build_ssh_command(ssh_alias="target", remote_command=remote) + + assert command == ["ssh", "target", f"bash -lc {shlex.quote(remote)}"] + assert "bash -lc 'set -euo pipefail;" in command[2] + + +def test_remote_command_runs_payload_under_slurm_bash_context() -> None: + remote = _build_remote_command( + target_run_id="20260624T040000Z-slurm-243958", + command_id="phase3 probe current", + remote_zip="/tmp/phase3 probe current.zip", + partition="gpu", + job_name="bb p3 probe", + ) + + assert "export PHASE3_TARGET_RUN_ID=20260624T040000Z-slurm-243958" in remote + assert "mktemp -d /shared/bb-p3-${USER:-root}/'phase3 probe current'.XXXXXX" in remote + assert "unzip -q '/tmp/phase3 probe current.zip' -d \"$WORK\"" in remote + assert "test -x ./run.sh" in remote + assert "srun --partition=gpu --job-name='bb p3 probe' --gres=gpu:8" in remote + assert shlex.quote("echo PHASE3_NODE=$(hostname); echo PHASE3_SLURM_JOB_ID=${SLURM_JOB_ID:-}; ./run.sh") in remote + + +def test_remote_command_accepts_explicit_slurm_targeting_options() -> None: + remote = _build_remote_command( + target_run_id="20260706T193804Z-slurm-pending", + command_id="phase4_nemo_agentloop_canonical_scratch", + remote_zip="/tmp/payload.zip", + partition="gpu", + job_name="bb-p4-nemo-agentloop", + nodelist="cnode-[19,148]", + constraint="mi300x", + reservation="bmoe", + qos="normal", + gres="gpu:1", + mem="512M", + ) + + assert "--gres=gpu:1" in remote + assert "--mem=512M" in remote + assert "--nodelist='cnode-[19,148]'" in remote + assert "--constraint=mi300x" in remote + assert "--reservation=bmoe" in remote + assert "--qos=normal" in remote + assert "20260706T193804Z-slurm-pending" in remote + + + +def test_slurm_targeting_option_validation_rejects_shell_characters() -> None: + assert _validated_slurm_option("cnode-[19,148]", name="nodelist") == "cnode-[19,148]" + try: + _validated_slurm_option("cnode-19;rm", name="nodelist") + except ValueError as exc: + assert "--nodelist contains unsupported characters" in str(exc) + else: # pragma: no cover + raise AssertionError("invalid Slurm option was accepted") + +def test_safe_artifact_name_rejects_path_segments() -> None: + bad_id = _safe_artifact_name("../bad id.log", fallback="fallback") + assert bad_id == f"bad_id_log-{hashlib.sha256(b'../bad id.log').hexdigest()[:12]}" + assert _safe_artifact_name("a b", fallback="fallback") != _safe_artifact_name("a/b", fallback="fallback") + assert _safe_artifact_name("...", fallback="fallback").startswith("fallback-") + + +def test_main_uses_sanitized_artifact_paths(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + safe_command_id = _safe_artifact_name("../bad id", fallback="phase3_command") + calls: list[list[str]] = [] + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + calls.append(command) + if command[0] == "scp": + assert command[1] == str(payload) + assert command[2] == f"target:/tmp/{safe_command_id}.zip" + assert kwargs["timeout"] == 20 + return subprocess.CompletedProcess(command, 0, "", "") + raise subprocess.TimeoutExpired(command, timeout=3600) + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "../bad id", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(tmp_path / "out"), + ] + ) + + assert len(calls) == 2 + ssh_payload = calls[1][2] + assert "../bad id" not in ssh_payload + assert f"/tmp/{safe_command_id}.zip" in ssh_payload + assert f"/shared/bb-p3-${{USER:-root}}/{safe_command_id}.XXXXXX" in ssh_payload + + out = tmp_path / "out" + assert result == 124 + assert (out / "command_logs" / f"{safe_command_id}.log").exists() + assert not (out / ".." / "bad id.log").exists() + assert (out / f"{safe_command_id}_blocked.json").exists() + assert not (out / "phase3_command_log_manifest.json").exists() + attempts = json.loads((out / "phase3_command_attempts_manifest.json").read_text()) + assert attempts["attempts"][0]["command_id"] == "../bad id" + assert attempts["attempts"][0]["raw_log_path"] == f"command_logs/{safe_command_id}.log" + raw_log = out / "command_logs" / f"{safe_command_id}.log" + expected_log_sha = "sha256:" + hashlib.sha256(raw_log.read_bytes()).hexdigest() + assert attempts["attempts"][0]["raw_log_sha256"] == expected_log_sha + + + +def test_main_honors_explicit_scp_timeout(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + scp_timeouts: list[int] = [] + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + scp_timeouts.append(kwargs["timeout"]) + return subprocess.CompletedProcess(command, 0, "", "") + raise subprocess.TimeoutExpired(command, timeout=3600) + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb-p3-probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(tmp_path / "out"), + "--scp-timeout-seconds", + "90", + ] + ) + + assert result == 124 + assert scp_timeouts == [90] + +def test_main_writes_only_passed_runs_to_canonical_manifest(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + out.mkdir() + (out / "phase3_command_attempts_manifest.json").write_text( + json.dumps( + { + "schema_version": "bb.rl.phase3.command_attempts_manifest.v1", + "target_run_id": "20260624T040000Z-slurm-243958", + "attempts": [ + {"command_id": "phase3_probe", "status": "failed"}, + {"command_id": "other_probe", "status": "failed"}, + ], + } + ) + + "\n" + ) + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + return subprocess.CompletedProcess(command, 0, "PHASE3_NODE=cnode-1\nPHASE3_SLURM_JOB_ID=12345\n", "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + ) + + assert result == 0 + manifest = json.loads((out / "phase3_command_log_manifest.json").read_text()) + assert manifest["commands"][0]["command_id"] == "phase3_probe" + assert manifest["commands"][0]["status"] == "passed" + assert manifest["commands"][0]["slurm_job_id"] == "12345" + assert manifest["commands"][0]["node"] == "cnode-1" + attempts = json.loads((out / "phase3_command_attempts_manifest.json").read_text()) + assert attempts["attempts"] == [{"command_id": "other_probe", "status": "failed"}] + + +def test_failed_rerun_removes_stale_canonical_manifest_row(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + attempts = 0 + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + nonlocal attempts + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + attempts += 1 + if attempts == 1: + return subprocess.CompletedProcess(command, 0, "PHASE3_NODE=cnode-1\nPHASE3_SLURM_JOB_ID=12345\n", "") + raise subprocess.TimeoutExpired(command, timeout=3600) + + monkeypatch.setattr(subprocess, "run", fake_run) + argv = [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + + assert main(argv) == 0 + assert (out / "phase3_command_log_manifest.json").exists() + + assert main(argv) == 124 + assert not (out / "phase3_command_log_manifest.json").exists() + retry_attempts = json.loads((out / "phase3_command_attempts_manifest.json").read_text()) + assert retry_attempts["attempts"][0]["command_id"] == "phase3_probe" + assert retry_attempts["attempts"][0]["status"] == "failed" + + +def test_passed_rerun_resets_stale_target_manifest(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + out.mkdir() + (out / "phase3_command_log_manifest.json").write_text( + json.dumps( + { + "schema_version": "bb.rl.phase3.command_log_manifest.v1", + "target_run_id": "20260624T040000Z-slurm-111111", + "commands": [{"command_id": "stale", "target_run_id": "20260624T040000Z-slurm-111111"}], + } + ) + + "\n" + ) + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + return subprocess.CompletedProcess(command, 0, "PHASE3_NODE=cnode-2\nPHASE3_SLURM_JOB_ID=67890\n", "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + assert ( + main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + ) + == 0 + ) + + manifest = json.loads((out / "phase3_command_log_manifest.json").read_text()) + assert manifest["target_run_id"] == "20260624T040000Z-slurm-243958" + assert [row["command_id"] for row in manifest["commands"]] == ["phase3_probe"] + + +def test_inline_component_report_paths_are_sanitized(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + report = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "a/b", + "component": "../evil", + "passed": True, + } + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + stdout = ( + "PHASE3_NODE=cnode-1\n" + "PHASE3_SLURM_JOB_ID=12345\n" + f"PHASE3_COMPONENT_REPORT_JSON={json.dumps(report)}\n" + ) + return subprocess.CompletedProcess(command, 0, stdout, "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + assert ( + main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + ) + == 0 + ) + + safe_component = _safe_artifact_name("../evil", fallback=_safe_artifact_name("a/b", fallback="phase3_probe")) + safe_report = _safe_artifact_name("a/b", fallback="phase3_probe") + report_path = out / safe_component / f"{safe_report}.json" + assert report_path.exists() + emitted = json.loads(report_path.read_text()) + assert emitted["component"] == "../evil" + assert emitted["report_id"] == "a/b" + assert emitted["target_run_id"] == "20260624T040000Z-slurm-243958" + assert not (out / ".." / "evil" / "b.json").exists() + + +def test_inline_component_report_empty_artifact_paths_are_injected(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + report = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "artifact-report", + "component": "runtime_probe", + "passed": True, + "artifact_paths": {}, + } + stdout = ( + "PHASE3_NODE=cnode-1\n" + "PHASE3_SLURM_JOB_ID=12345\n" + f"PHASE3_COMPONENT_REPORT_JSON={json.dumps(report)}\n" + ) + stderr = "remote diagnostic line\n" + raw_log = stdout + stderr + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + return subprocess.CompletedProcess(command, 0, stdout, stderr) + + def resolve_artifact_path(value: str) -> Path: + path = Path(value) + return path if path.is_absolute() else out / path + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + ) + + assert result == 0 + report_path = out / "runtime_probe" / "artifact-report.json" + emitted = json.loads(report_path.read_text()) + artifact_paths = emitted["artifact_paths"] + component_report_path = resolve_artifact_path(artifact_paths["component_report_json"]) + command_log_path = resolve_artifact_path(artifact_paths["command_log"]) + assert component_report_path == report_path.resolve() + assert component_report_path.exists() + assert command_log_path.exists() + assert command_log_path.read_text() == raw_log + manifest = json.loads((out / "phase3_command_log_manifest.json").read_text()) + assert manifest["commands"][0]["command_id"] == "phase3_probe" + assert manifest["commands"][0]["status"] == "passed" + +def test_inline_component_passed_false_blocks_canonical_manifest(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + report = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "blocked-report", + "component": "nemo_gym_agentloop_smoke", + "passed": False, + "blocked_reason": "canonical_runtime_imports_missing", + } + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + stdout = ( + "PHASE3_NODE=cnode-1\n" + "PHASE3_SLURM_JOB_ID=12345\n" + f"PHASE3_COMPONENT_REPORT_JSON={json.dumps(report)}\n" + ) + return subprocess.CompletedProcess(command, 0, stdout, "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + ) + + assert result == 1 + assert not (out / "phase3_command_log_manifest.json").exists() + attempts = json.loads((out / "phase3_command_attempts_manifest.json").read_text()) + row = attempts["attempts"][0] + assert row["command_id"] == "phase3_probe" + assert row["status"] == "failed" + assert row["blocked_reason"] == "inline_component_failed" + assert row["component_passed"] is False + assert row["component_failed_count"] == 1 + assert row["component_blocked_reasons"] == ["canonical_runtime_imports_missing"] + report_path = out / "nemo_gym_agentloop_smoke" / "blocked-report.json" + assert report_path.exists() + emitted = json.loads(report_path.read_text()) + assert emitted["passed"] is False + assert emitted["target_run_id"] == "20260624T040000Z-slurm-243958" + + +def test_inline_component_blocked_rerun_removes_stale_canonical_row(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + out.mkdir() + (out / "phase3_command_log_manifest.json").write_text( + json.dumps( + { + "schema_version": "bb.rl.phase3.command_log_manifest.v1", + "target_run_id": "20260624T040000Z-slurm-243958", + "commands": [{"command_id": "phase3_probe", "status": "passed"}], + } + ) + + "\n" + ) + report = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "blocked-report", + "component": "nemo_gym_agentloop_smoke", + "passed": False, + "blocked_reason": "canonical_runtime_imports_missing", + } + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + stdout = ( + "PHASE3_NODE=cnode-1\n" + "PHASE3_SLURM_JOB_ID=12345\n" + f"PHASE3_COMPONENT_REPORT_JSON={json.dumps(report)}\n" + ) + return subprocess.CompletedProcess(command, 0, stdout, "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + assert ( + main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + ) + == 1 + ) + assert not (out / "phase3_command_log_manifest.json").exists() + attempts = json.loads((out / "phase3_command_attempts_manifest.json").read_text()) + assert attempts["attempts"][0]["blocked_reason"] == "inline_component_failed" + + +def test_multiple_inline_component_reports_preserve_blocked_reasons(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + reports = [ + { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "passed-report", + "component": "runtime_probe", + "passed": True, + }, + { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "blocked-import", + "component": "nemo_gym_agentloop_smoke", + "passed": False, + "blocked_reason": "canonical_runtime_imports_missing", + }, + { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "blocked-harbor", + "component": "harbor_lifecycle", + "passed": False, + "blocked_reason": "target_harbor_endpoint_missing", + }, + ] + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + stdout = "PHASE3_NODE=cnode-1\nPHASE3_SLURM_JOB_ID=12345\n" + "".join( + f"PHASE3_COMPONENT_REPORT_JSON={json.dumps(report)}\n" for report in reports + ) + return subprocess.CompletedProcess(command, 0, stdout, "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + assert ( + main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + ) + == 1 + ) + attempts = json.loads((out / "phase3_command_attempts_manifest.json").read_text()) + row = attempts["attempts"][0] + assert row["component_passed"] is False + assert row["component_failed_count"] == 2 + assert row["component_blocked_reasons"] == [ + "canonical_runtime_imports_missing", + "target_harbor_endpoint_missing", + ] + + +def test_inline_component_missing_passed_blocks_canonical_manifest(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + report = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "ambiguous-report", + "component": "runtime_probe", + } + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + stdout = ( + "PHASE3_NODE=cnode-1\n" + "PHASE3_SLURM_JOB_ID=12345\n" + f"PHASE3_COMPONENT_REPORT_JSON={json.dumps(report)}\n" + ) + return subprocess.CompletedProcess(command, 0, stdout, "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + result = main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + ) + + assert result == 1 + assert not (out / "phase3_command_log_manifest.json").exists() + attempts = json.loads((out / "phase3_command_attempts_manifest.json").read_text()) + row = attempts["attempts"][0] + assert row["blocked_reason"] == "inline_component_failed" + assert row["component_passed"] is False + assert row["component_failed_count"] == 1 + assert row["component_blocked_reasons"] == ["inline_component_not_passed"] + + +def test_bad_inline_component_report_does_not_pass_canonical_manifest(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + stdout = ( + "PHASE3_NODE=cnode-1\n" + "PHASE3_SLURM_JOB_ID=12345\n" + "PHASE3_COMPONENT_REPORT_JSON={bad-json\n" + ) + return subprocess.CompletedProcess(command, 0, stdout, "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + assert ( + main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + ) + == 1 + ) + + assert not (out / "phase3_command_log_manifest.json").exists() + attempts = json.loads((out / "phase3_command_attempts_manifest.json").read_text()) + assert attempts["attempts"][0]["command_id"] == "phase3_probe" + assert attempts["attempts"][0]["status"] == "failed" + assert attempts["attempts"][0]["blocked_reason"] == "invalid_inline_report" + + +def test_mixed_good_and_bad_inline_reports_leave_no_partial_artifacts(tmp_path, monkeypatch) -> None: + payload = tmp_path / "payload.zip" + payload.write_bytes(b"zip") + out = tmp_path / "out" + report = { + "schema_version": "bb.rl.phase3.component_report.v1", + "report_id": "good-report", + "component": "component", + "passed": True, + } + + def fake_run(command, **kwargs): # noqa: ANN001, ANN202 + if command[0] == "scp": + return subprocess.CompletedProcess(command, 0, "", "") + stdout = ( + "PHASE3_NODE=cnode-1\n" + "PHASE3_SLURM_JOB_ID=12345\n" + f"PHASE3_COMPONENT_REPORT_JSON={json.dumps(report)}\n" + "PHASE3_COMPONENT_REPORT_JSON={bad-json\n" + ) + return subprocess.CompletedProcess(command, 0, stdout, "") + + monkeypatch.setattr(subprocess, "run", fake_run) + + assert ( + main( + [ + "--ssh-alias", + "target", + "--partition", + "gpu", + "--job-name", + "bb p3 probe", + "--command-id", + "phase3_probe", + "--target-run-id", + "20260624T040000Z-slurm-243958", + "--payload-zip", + str(payload), + "--output-dir", + str(out), + ] + ) + == 1 + ) + + assert not (out / "component" / "good-report.json").exists() + assert not (out / "phase3_command_log_manifest.json").exists() + attempts = json.loads((out / "phase3_command_attempts_manifest.json").read_text()) + assert attempts["attempts"][0]["status"] == "failed" + assert attempts["attempts"][0]["blocked_reason"] == "invalid_inline_report" diff --git a/tests/rl/phase3/test_trainer_live.py b/tests/rl/phase3/test_trainer_live.py new file mode 100644 index 00000000..f67a91c5 --- /dev/null +++ b/tests/rl/phase3/test_trainer_live.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path + +import pytest + +from breadboard.rl.phase2.bridge import build_verl_batch_from_projection_rows +from breadboard.rl.phase3.evidence import PHASE3_COMMAND_LOG_MANIFEST_SCHEMA, sha256_file +from breadboard.rl.phase3.trainer_live import Phase3TrainerRunSpec, build_phase3_dataproto, build_phase3_trainer_update_report, validate_phase3_trainer_update_report + +TARGET = "20260623T000000Z-slurm-234555" + + +def row(task_id: str = "task", group_id: str | None = None, quarantined: bool = False) -> dict: + payload = { + "input_ids": [1, 2, 3], + "prompt_ids": [1], + "completion_ids": [2, 3], + "attention_mask": [1, 1, 1], + "loss_mask": [False, True, True], + "assistant_mask": [False, True, True], + "tool_action_mask": [False, False, False], + "reward_mask": [False, True, True], + "completion_logprobs": [-0.1, -0.2], + "completion_logprob_status": "native_available", + "policy_snapshot_id": "policy-1", + "admission": {"quarantine_status": "quarantined" if quarantined else "clear", "row_status": "accepted", "trainable": True}, + "reward": {"scalar": 1.0}, + "rollout_id": f"roll-{task_id}", + "trajectory_id": f"traj-{task_id}", + "episode_id": f"ep-{task_id}", + "task_id": task_id, + "projection_manifest_id": "pm-1", + "trainable_candidate": True, + } + if group_id: + payload["group_id"] = group_id + return payload + + +class FakeTensorDict(dict): + def __init__(self, data, batch_size=None): + super().__init__(data) + self.batch_size = batch_size + + +class FakeDataProto: + @classmethod + def from_dict(cls, payload): + obj = cls() + obj.payload = payload + return obj + + +@pytest.fixture(autouse=True) +def fake_verl_modules(monkeypatch): + torch = types.SimpleNamespace(long="long", tensor=lambda values, **kwargs: {"values": values, "kwargs": kwargs}) + tensordict = types.ModuleType("tensordict") + tensordict.TensorDict = FakeTensorDict + verl = types.ModuleType("verl") + protocol = types.ModuleType("verl.protocol") + protocol.DataProto = FakeDataProto + monkeypatch.setitem(sys.modules, "torch", torch) + monkeypatch.setitem(sys.modules, "tensordict", tensordict) + monkeypatch.setitem(sys.modules, "verl", verl) + monkeypatch.setitem(sys.modules, "verl.protocol", protocol) + + +def test_dataproto_conversion_with_fake_verl() -> None: + batch = build_verl_batch_from_projection_rows([row("a")], target_run_id=TARGET).to_dict() + proto = build_phase3_dataproto(batch, device="cpu", require_grpo_uid=False) + assert set(proto.payload["batch"].keys()) >= {"input_ids", "attention_mask", "responses", "response_mask", "token_level_rewards", "old_log_probs"} + assert proto.payload["meta_info"]["target_run_id"] == TARGET + + +def test_grpo_single_uid_rejected() -> None: + batch = build_verl_batch_from_projection_rows([row("a")], target_run_id=TARGET).to_dict() + with pytest.raises(ValueError, match="GRPO uid groups"): + build_phase3_dataproto(batch, device="cpu", require_grpo_uid=True) + + +def test_quarantined_row_rejected() -> None: + with pytest.raises(ValueError, match="quarantined"): + build_verl_batch_from_projection_rows([row("a", quarantined=True)], target_run_id=TARGET) + + +def _manifest(evidence_root: Path) -> dict: + raw = evidence_root / "ZYPHRA" / "RL_PHASE_3" / "runs" / "command_logs" / "cmd.log" + raw.parent.mkdir(parents=True) + raw.write_text("ok") + return {"schema_version": PHASE3_COMMAND_LOG_MANIFEST_SCHEMA, "target_run_id": TARGET, "commands": [{"command_id": "cmd", "argv": ["x"], "raw_log_path": "command_logs/cmd.log", "raw_log_sha256": sha256_file(raw), "slurm_job_id": "234555", "target_run_id": TARGET, "node": "n", "started_at": "a", "completed_at": "b", "exit_code": 0, "status": "passed"}]} + + +def test_checkpoint_hash_equality_rejected(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "trainer" + out.mkdir(parents=True) + before = out / "before.pt"; after = out / "after.pt"; metrics = out / "metrics.json" + before.write_text("same"); after.write_text("same"); metrics.write_text(json.dumps({"optimizer_step_count": 1, "device_count": 8, "weight_update_performed": True})) + spec = Phase3TrainerRunSpec(TARGET, "verl_ppo", "model", out / "rows.json", out, 1) + report = build_phase3_trainer_update_report(spec, command_log_manifest=_manifest(evidence), checkpoint_before=before, checkpoint_after=after, metrics_path=metrics) + assert any("checkpoint" in error for error in validate_phase3_trainer_update_report(report)) + + +def test_stale_command_log_hash_rejected(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + manifest = _manifest(evidence) + raw = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "command_logs" / "cmd.log" + raw.write_text("changed") + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "trainer"; out.mkdir(parents=True) + before = out / "before.pt"; after = out / "after.pt"; metrics = out / "metrics.json" + before.write_text("a"); after.write_text("b"); metrics.write_text(json.dumps({"optimizer_step_count": 1, "device_count": 8, "weight_update_performed": True})) + report = build_phase3_trainer_update_report(Phase3TrainerRunSpec(TARGET, "verl_ppo", "model", out / "rows.json", out, 1), command_log_manifest=manifest, checkpoint_before=before, checkpoint_after=after, metrics_path=metrics) + assert report["passed"] is False + + +def test_valid_ppo_grpo_reports_accept(tmp_path: Path) -> None: + evidence = tmp_path / "docs_tmp" + manifest = _manifest(evidence) + out = evidence / "ZYPHRA" / "RL_PHASE_3" / "runs" / "trainer"; out.mkdir(parents=True) + metrics = out / "metrics.json"; metrics.write_text(json.dumps({"optimizer_step_count": 1, "device_count": 8, "weight_update_performed": True, "loss_metrics": {"loss": 0.1}})) + for backend in ("verl_ppo", "verl_grpo"): + before = out / f"{backend}-before.pt"; after = out / f"{backend}-after.pt" + before.write_text("a"); after.write_text("b") + report = build_phase3_trainer_update_report(Phase3TrainerRunSpec(TARGET, backend, "model", out / "rows.json", out, 1), command_log_manifest=manifest, checkpoint_before=before, checkpoint_after=after, metrics_path=metrics) + assert validate_phase3_trainer_update_report(report) == [] diff --git a/tests/rl/phase4/test_infra_hardening.py b/tests/rl/phase4/test_infra_hardening.py new file mode 100644 index 00000000..16730e66 --- /dev/null +++ b/tests/rl/phase4/test_infra_hardening.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from breadboard.rl.phase4.infra_hardening import InfraHardeningInputs, evaluate_infra_hardening + + +def test_infra_hardening_does_not_count_runsc_binary_as_registered_runtime() -> None: + report = evaluate_infra_hardening( + InfraHardeningInputs( + firecracker_version="Firecracker v1.16.1", + firecracker_non_root_boot=True, + firecracker_jailer_available=True, + firecracker_networking_configured=True, + firecracker_concurrent_microvms=2, + gvisor_runsc_version="runsc version release-20260622", + gvisor_registered_runtime=False, + cpu_squeeze_workers=48, + cpu_squeeze_success=True, + ) + ) + + assert report["passed"] is False + assert report["checks"]["firecracker"]["passed"] is True + assert report["checks"]["gvisor"]["passed"] is False + assert "gvisor_registered_runtime_missing" in report["blockers"] + assert report["scorecard_update_allowed"] is False + + +def test_infra_hardening_requires_firecracker_rootless_jailer_networking_and_concurrency() -> None: + report = evaluate_infra_hardening( + InfraHardeningInputs( + firecracker_version="Firecracker v1.16.1", + firecracker_non_root_boot=False, + firecracker_jailer_available=False, + firecracker_networking_configured=False, + firecracker_concurrent_microvms=1, + gvisor_runsc_version="runsc version release-20260622", + gvisor_registered_runtime=True, + gvisor_runtime_name="runsc", + cpu_squeeze_workers=48, + cpu_squeeze_success=True, + ) + ) + + assert report["passed"] is False + assert report["checks"]["gvisor"]["passed"] is True + assert report["checks"]["firecracker"]["blockers"] == [ + "firecracker_non_root_boot_missing", + "firecracker_jailer_missing", + "firecracker_networking_missing", + "firecracker_concurrency_missing", + ] + + +def test_infra_hardening_passes_only_when_all_production_hardening_checks_pass() -> None: + report = evaluate_infra_hardening( + InfraHardeningInputs( + firecracker_version="Firecracker v1.16.1", + firecracker_non_root_boot=True, + firecracker_jailer_available=True, + firecracker_networking_configured=True, + firecracker_concurrent_microvms=3, + gvisor_runsc_version="runsc version release-20260622", + gvisor_registered_runtime=True, + gvisor_runtime_name="runsc", + cpu_squeeze_workers=48, + cpu_squeeze_success=True, + ) + ) + + assert report["passed"] is True + assert report["blockers"] == [] + assert report["promotional"] is False + assert report["claim_boundary"] == "phase4_production_infra_hardening_non_promotional_scope" diff --git a/tests/rl/phase4/test_native_inference.py b/tests/rl/phase4/test_native_inference.py new file mode 100644 index 00000000..6405c80c --- /dev/null +++ b/tests/rl/phase4/test_native_inference.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import pytest + +from breadboard.rl.phase4.native_inference import ( + BREADBOARD_NATIVE_INFERENCE_OWNER, + BREADBOARD_NATIVE_LANE_SCHEMA, + NativeInferenceLane, +) + + +def _sha256_json(value: Any) -> str: + return "sha256:" + hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() + + +def _sha256_file(path: Path) -> str: + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +class FakeTokenizer: + def __init__(self) -> None: + self.decode_calls: list[tuple[list[int], bool]] = [] + self.encode_calls: list[tuple[str, bool]] = [] + + def decode(self, token_ids: list[int], *, skip_special_tokens: bool = False) -> str: + self.decode_calls.append((list(token_ids), skip_special_tokens)) + return "prompt<" + ",".join(str(token_id) for token_id in token_ids) + ">" + + def encode(self, text: str, *, add_special_tokens: bool = False) -> list[int]: + self.encode_calls.append((text, add_special_tokens)) + if text == "alpha beta": + return [101, 202, 303] + return [len(text)] + + +class FakeCompletionResponse: + def __init__(self, *, payload: dict[str, Any], status_code: int = 200, failure: Exception | None = None) -> None: + self._payload = payload + self.status_code = status_code + self._failure = failure + + def raise_for_status(self) -> None: + if self._failure is not None: + raise self._failure + + def json(self) -> dict[str, Any]: + return self._payload + + +class FakeCompletionSession: + def __init__(self, response: FakeCompletionResponse | None = None, failure: Exception | None = None) -> None: + self.response = response + self.failure = failure + self.posts: list[tuple[str, dict[str, Any], float]] = [] + + def post(self, url: str, *, json: dict[str, Any], timeout: float) -> FakeCompletionResponse: + self.posts.append((url, dict(json), timeout)) + if self.failure is not None: + raise self.failure + assert self.response is not None + return self.response + + +def _read_jsonl(path: Path) -> list[dict[str, Any]]: + return [json.loads(line) for line in path.read_text().splitlines()] + + +def test_native_lane_records_breadboard_ids_and_separate_backend_token_evidence(tmp_path: Path) -> None: + log_path = tmp_path / "native" / "requests.jsonl" + response_payload = { + "id": "cmpl-backend-42", + "choices": [ + { + "text": "alpha beta", + "token_ids": [9101, 9102], + "logprobs": { + "tokens": ["alpha", " beta"], + "token_logprobs": [-0.125, -0.5], + }, + } + ], + } + session = FakeCompletionSession(response=FakeCompletionResponse(payload=response_payload)) + tokenizer = FakeTokenizer() + lane = NativeInferenceLane( + model_ref="fake/native-model", + base_url="http://127.0.0.1:8000/", + tokenizer=tokenizer, + request_log_path=log_path, + target_run_id="target run/01", + session=session, + timeout_seconds=7.5, + ) + + native_response = lane.generate_completion( + upstream_request_id="rollout request/7", + prompt_ids=[11, 22], + sampling_params={"max_tokens": 3, "temperature": 0.25, "logprobs": 2}, + ) + + assert session.posts == [ + ( + "http://127.0.0.1:8000/v1/completions", + { + "model": "fake/native-model", + "prompt": "prompt<11,22>", + "max_tokens": 3, + "temperature": 0.25, + "logprobs": 2, + }, + 7.5, + ) + ] + assert tokenizer.decode_calls == [([11, 22], False)] + assert tokenizer.encode_calls == [("alpha beta", False)] + assert native_response.request_id.startswith("bbreq-target-run-01-rollout-request-7-") + assert native_response.response_id.startswith("bbresp-") + assert native_response.upstream_request_id == "rollout request/7" + assert native_response.passed is True + assert native_response.backend_completion_id == "cmpl-backend-42" + assert native_response.backend_token_texts == ["alpha", " beta"] + assert native_response.backend_token_logprobs == [-0.125, -0.5] + assert native_response.backend_token_ids == [9101, 9102] + assert native_response.posthoc_token_ids == [101, 202, 303] + + records = _read_jsonl(log_path) + assert len(records) == 1 + record = records[0] + assert record["schema_version"] == BREADBOARD_NATIVE_LANE_SCHEMA + assert record["inference_owner"] == BREADBOARD_NATIVE_INFERENCE_OWNER + assert record["breadboard_native_lane_used"] is True + assert record["request_id"] == native_response.request_id + assert record["response_id"] == native_response.response_id + assert record["passed"] is True + assert record["http_status"] == 200 + assert record["posthoc_token_count"] == 3 + assert record["backend_token_text_count"] == 2 + assert record["backend_token_id_count"] == 2 + assert record["backend_token_logprob_count"] == 2 + assert record["backend_token_texts_sha256"] == _sha256_json(["alpha", " beta"]) + assert record["backend_token_ids_sha256"] == _sha256_json([9101, 9102]) + assert record["backend_token_logprobs_sha256"] == _sha256_json([-0.125, -0.5]) + assert record["posthoc_token_ids_sha256"] == _sha256_json([101, 202, 303]) + assert record["backend_token_texts_sha256"] != record["posthoc_token_ids_sha256"] + assert record["backend_token_logprobs_sha256"] != record["posthoc_token_ids_sha256"] + + status = lane.status() + assert status["schema_version"] == BREADBOARD_NATIVE_LANE_SCHEMA + assert status["inference_owner"] == BREADBOARD_NATIVE_INFERENCE_OWNER + assert status["generate_calls"] == 1 + assert status["last_request_id"] == native_response.request_id + assert status["last_response_id"] == native_response.response_id + assert status["last_backend_token_text_count"] == 2 + assert status["last_backend_token_id_count"] == 2 + assert status["last_backend_token_logprob_count"] == 2 + assert status["request_log_sha256"] == _sha256_file(log_path) + + +def test_native_lane_appends_failed_evidence_row_before_raising(tmp_path: Path) -> None: + log_path = tmp_path / "requests.jsonl" + session = FakeCompletionSession(failure=ConnectionError("target session refused connection")) + lane = NativeInferenceLane( + model_ref="fake/native-model", + base_url="http://127.0.0.1:8000", + tokenizer=FakeTokenizer(), + request_log_path=log_path, + target_run_id="target-run-02", + session=session, + timeout_seconds=1.25, + ) + + with pytest.raises(RuntimeError, match="native inference request failed: ConnectionError: target session refused connection"): + lane.generate_completion( + upstream_request_id="rollout-99", + prompt_ids=[5], + sampling_params={"max_tokens": 1, "temperature": 0.0}, + ) + + assert session.posts == [ + ( + "http://127.0.0.1:8000/v1/completions", + { + "model": "fake/native-model", + "prompt": "prompt<5>", + "max_tokens": 1, + "temperature": 0.0, + "logprobs": 1, + }, + 1.25, + ) + ] + records = _read_jsonl(log_path) + assert len(records) == 1 + record = records[0] + assert record["inference_owner"] == BREADBOARD_NATIVE_INFERENCE_OWNER + assert record["request_id"].startswith("bbreq-target-run-02-rollout-99-") + assert record["response_id"].startswith("bbresp-") + assert record["passed"] is False + assert record["http_status"] == 0 + assert record["error_type"] == "ConnectionError" + assert record["error_message"] == "target session refused connection" + assert record["posthoc_token_count"] == 0 + assert record["backend_token_text_count"] == 0 + assert record["backend_token_id_count"] == 0 + assert record["backend_token_logprob_count"] == 0 + + status = lane.status() + assert status["generate_calls"] == 1 + assert status["last_request_id"] == record["request_id"] + assert status["last_response_id"] == record["response_id"] + assert status["last_http_status"] == 0 + assert status["request_log_sha256"] == _sha256_file(log_path) diff --git a/tests/rl/phase4/test_wrapper_identity.py b/tests/rl/phase4/test_wrapper_identity.py new file mode 100644 index 00000000..7ff37a6f --- /dev/null +++ b/tests/rl/phase4/test_wrapper_identity.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.phase4.wrapper_identity import collect_wrapper_identity, parse_deps_pins, runtime_module_provenance + +WRAPPER_HEAD = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +VERL_HEAD = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +NEMO_HEAD = "cccccccccccccccccccccccccccccccccccccccc" + + +def _wrapper(tmp_path: Path, *, deps: bool = True, submodules: bool = True) -> Path: + root = tmp_path / "verl_wrapper" + (root / "src" / "zyphra_verl").mkdir(parents=True) + (root / "src" / "zyphra_verl" / "nemo_gym_loop.py").write_text('register("nemo_gym_tool_use")\nToolParser\nToolCallComparator\nreward_score\n') + if deps: + (root / "deps.yaml").write_text(f"verl:\n pin: {VERL_HEAD}\nnemo_gym:\n pin: {NEMO_HEAD}\n") + if submodules: + (root / "third_party" / "verl").mkdir(parents=True) + (root / "third_party" / "verl" / "pyproject.toml").write_text("[project]\nname='verl'\n") + (root / "third_party" / "nemo-gym").mkdir(parents=True) + (root / "third_party" / "nemo-gym" / "pyproject.toml").write_text("[project]\nname='nemo-gym'\n") + return root + + +def _git_runner(status_marker: str = ""): + def run(args: list[str], cwd: Path) -> str: + cwd_text = str(cwd).replace("\\", "/") + if args == ["submodule", "status", "--recursive"]: + return f"{status_marker}{VERL_HEAD} third_party/verl (heads/main)\n{status_marker}{NEMO_HEAD} third_party/nemo-gym (heads/main)" + if args == ["rev-parse", "HEAD"]: + if cwd_text.endswith("third_party/verl"): + return VERL_HEAD + if cwd_text.endswith("third_party/nemo-gym"): + return NEMO_HEAD + return WRAPPER_HEAD + if args == ["rev-parse", "--abbrev-ref", "HEAD"]: + return "main" + return "" + return run + + +def test_parse_deps_pins_requires_real_yaml_values(tmp_path: Path) -> None: + deps = tmp_path / "deps.yaml" + deps.write_text("verl:\n pin: abc123 # comment\nnemo_gym:\n commit: def456\n") + + assert parse_deps_pins(deps) == {"verl_pin": "abc123", "nemo_gym_commit": "def456"} + + +def test_wrapper_identity_passes_with_expected_pins_and_clean_submodules(tmp_path: Path) -> None: + wrapper = _wrapper(tmp_path) + + identity = collect_wrapper_identity(wrapper, git_runner=_git_runner()) + payload = identity.to_dict() + + assert payload["passed"] is True + assert payload["wrapper_commit"] == WRAPPER_HEAD + assert payload["components"]["verl"]["expected_commit"] == VERL_HEAD + assert payload["components"]["verl"]["actual_commit"] == VERL_HEAD + assert payload["components"]["nemo_gym"]["expected_commit"] == NEMO_HEAD + assert payload["components"]["nemo_gym"]["actual_commit"] == NEMO_HEAD + + +def test_wrapper_identity_blocks_when_deps_yaml_or_expected_pins_are_missing(tmp_path: Path) -> None: + wrapper = _wrapper(tmp_path, deps=False) + + identity = collect_wrapper_identity(wrapper, git_runner=_git_runner()).to_dict() + + assert identity["passed"] is False + assert "deps_yaml_missing" in identity["blockers"] + assert "submodule_expected_pin_missing:third_party/verl" in identity["blockers"] + assert "submodule_expected_pin_missing:third_party/nemo-gym" in identity["blockers"] + + +def test_wrapper_identity_blocks_on_uninitialized_dirty_or_conflicted_submodule(tmp_path: Path) -> None: + wrapper = _wrapper(tmp_path) + + for marker, blocker in ( + ("-", "submodule_uninitialized:third_party/verl"), + ("+", "submodule_dirty:third_party/verl"), + ("U", "submodule_conflicted:third_party/verl"), + ): + identity = collect_wrapper_identity(wrapper, git_runner=_git_runner(marker)).to_dict() + assert identity["passed"] is False + assert blocker in identity["blockers"] + + +def test_runtime_module_provenance_requires_staged_roots(tmp_path: Path) -> None: + wrapper = _wrapper(tmp_path) + inside = runtime_module_provenance( + wrapper, + { + "zyphra_verl": str(wrapper / "src" / "zyphra_verl" / "__init__.py"), + "verl": str(wrapper / "third_party" / "verl" / "verl" / "__init__.py"), + "nemo_gym": str(wrapper / "third_party" / "nemo-gym" / "nemo_gym" / "__init__.py"), + }, + ) + outside = runtime_module_provenance( + wrapper, + { + "zyphra_verl": str(wrapper / "src" / "zyphra_verl" / "__init__.py"), + "verl": "/usr/local/lib/python3.12/site-packages/verl/__init__.py", + "nemo_gym": str(wrapper / "third_party" / "nemo-gym" / "nemo_gym" / "__init__.py"), + }, + ) + + assert inside["passed"] is True + assert outside["passed"] is False + assert "runtime_identity_mismatch:verl" in outside["blockers"] diff --git a/tests/rl/renderer/__init__.py b/tests/rl/renderer/__init__.py new file mode 100644 index 00000000..e7aeb488 --- /dev/null +++ b/tests/rl/renderer/__init__.py @@ -0,0 +1 @@ +"""Renderer test helpers.""" diff --git a/tests/rl/renderer/helpers.py b/tests/rl/renderer/helpers.py new file mode 100644 index 00000000..e4f72b34 --- /dev/null +++ b/tests/rl/renderer/helpers.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any + + +def base_rendered_turn_payload(*, fidelity_class: str = "F3") -> dict[str, Any]: + provider_by_class = { + "F0": {"token_ids_source": "unavailable", "logprobs_source": "unavailable"}, + "F1": {"token_ids_source": "posthoc_tokenizer", "logprobs_source": "unavailable"}, + "F2": {"token_ids_source": "provider_native", "logprobs_source": "unavailable"}, + "F3": {"token_ids_source": "provider_native", "logprobs_source": "provider_native"}, + } + provider = provider_by_class[fidelity_class] + payload: dict[str, Any] = { + "rollout_id": "rollout-1", + "trajectory_id": "traj-1", + "task_id": "task-1", + "split_id": "train_probe", + "env_package_hash": "sha256:env-package", + "turn_id": "turn-1", + "renderer": { + "renderer_id": "renderer-1", + "renderer_version": "0.1.0", + "renderer_config_hash": "sha256:renderer-config", + "tokenizer_id": "tok-1", + "tokenizer_hash": "sha256:tokenizer", + "chat_template_id": "template-1", + "chat_template_hash": "sha256:template", + "stop_token_ids": [2], + }, + "provider_fidelity": { + "fidelity_class": fidelity_class, + "provider": "local", + "model_requested": "test-model", + "model_served": "test-model", + "sampling_config": {"temperature": 1.0}, + **provider, + }, + "prompt_ids": [10, 11, 12], + "completion_ids": [20, 21], + "input_ids": [10, 11, 12, 20, 21], + "attention_mask": [1, 1, 1, 1, 1], + "loss_mask": [False, False, False, True, True], + "assistant_mask": [False, False, False, True, True], + "tool_action_mask": [False, False, False, False, False], + "reward_mask": [False, False, False, False, True], + "sampled_mask": [False, False, False, True, True], + "message_indices": [0, 0, 0, 1, 1], + "bridge_to_next_turn": {"attempted": True, "success": True}, + "tool_parse_status": "not_applicable", + "parsed_completion": "42", + "tool_calls": [], + "finish_reason": "stop", + "is_truncated": False, + "overlong_prompt": False, + "metadata": {"source": "unit_test"}, + } + if fidelity_class == "F3": + payload["completion_logprobs"] = [-0.1, -0.2] + return payload + + +def cloned_payload(*, fidelity_class: str = "F3") -> dict[str, Any]: + return deepcopy(base_rendered_turn_payload(fidelity_class=fidelity_class)) diff --git a/tests/rl/renderer/test_bridge_to_next_turn.py b/tests/rl/renderer/test_bridge_to_next_turn.py new file mode 100644 index 00000000..bd47bf41 --- /dev/null +++ b/tests/rl/renderer/test_bridge_to_next_turn.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import pytest + +from breadboard.rl.renderer.records import classify_rendered_turn_trainability +from breadboard.rl.renderer.schema import BridgeToNextTurn, RenderedTurnRecord +from tests.rl.renderer.helpers import cloned_payload + + +def test_bridge_success_is_recorded() -> None: + record = RenderedTurnRecord.from_dict(cloned_payload()) + + assert record.bridge_to_next_turn.attempted is True + assert record.bridge_to_next_turn.success is True + + +def test_bridge_failure_requires_reason() -> None: + with pytest.raises(ValueError, match="failure requires failure_reason"): + BridgeToNextTurn(attempted=True, success=False) + + +def test_bridge_failure_blocks_trainability() -> None: + payload = cloned_payload() + payload["bridge_to_next_turn"] = { + "attempted": True, + "success": False, + "failure_reason": "parser_state_lost", + } + + decision = classify_rendered_turn_trainability(RenderedTurnRecord.from_dict(payload)) + assert decision.sft_trainable is False + assert decision.on_policy_trainable is False + assert "bridge_to_next_turn_failed" in decision.blocked_reasons diff --git a/tests/rl/renderer/test_message_indices.py b/tests/rl/renderer/test_message_indices.py new file mode 100644 index 00000000..a76f2e6f --- /dev/null +++ b/tests/rl/renderer/test_message_indices.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from breadboard.rl.renderer.records import validate_rendered_turn +from breadboard.rl.renderer.schema import RenderedTurnRecord +from tests.rl.renderer.helpers import cloned_payload + + +def test_message_indices_map_every_token() -> None: + record = RenderedTurnRecord.from_dict(cloned_payload()) + + assert len(record.message_indices) == len(record.input_ids) + assert validate_rendered_turn(record) == [] + + +def test_message_indices_length_mismatch_fails() -> None: + payload = cloned_payload() + payload["message_indices"] = [0, 0] + + errors = validate_rendered_turn(RenderedTurnRecord.from_dict(payload)) + assert "message_indices length must equal input_ids length" in errors diff --git a/tests/rl/renderer/test_provider_message_mode.py b/tests/rl/renderer/test_provider_message_mode.py new file mode 100644 index 00000000..8b724260 --- /dev/null +++ b/tests/rl/renderer/test_provider_message_mode.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from breadboard.rl.renderer.records import classify_rendered_turn_trainability, validate_rendered_turn +from breadboard.rl.renderer.schema import RenderedTurnRecord +from tests.rl.renderer.helpers import cloned_payload + + +def test_message_only_f0_is_debug_only_not_trainable() -> None: + record = RenderedTurnRecord.from_dict(cloned_payload(fidelity_class="F0")) + + assert validate_rendered_turn(record) == [] + decision = classify_rendered_turn_trainability(record) + assert decision.sft_trainable is False + assert decision.on_policy_trainable is False + assert "message_only_provider_fidelity" in decision.blocked_reasons + + +def test_posthoc_f1_can_be_sft_but_not_on_policy() -> None: + record = RenderedTurnRecord.from_dict(cloned_payload(fidelity_class="F1")) + + decision = classify_rendered_turn_trainability(record) + assert decision.sft_trainable is True + assert decision.on_policy_trainable is False + assert "on_policy_requires_f3_token_native_logprobs" in decision.blocked_reasons + + +def test_f2_token_native_without_logprobs_is_not_on_policy() -> None: + record = RenderedTurnRecord.from_dict(cloned_payload(fidelity_class="F2")) + + decision = classify_rendered_turn_trainability(record) + assert decision.sft_trainable is True + assert decision.on_policy_trainable is False + assert "on_policy_requires_f3_token_native_logprobs" in decision.blocked_reasons + + +def test_f3_requires_completion_logprobs() -> None: + payload = cloned_payload(fidelity_class="F3") + payload.pop("completion_logprobs") + + errors = validate_rendered_turn(RenderedTurnRecord.from_dict(payload)) + assert "F3 provider fidelity requires completion_logprobs" in errors diff --git a/tests/rl/renderer/test_token_mask_lengths.py b/tests/rl/renderer/test_token_mask_lengths.py new file mode 100644 index 00000000..88bc1f0f --- /dev/null +++ b/tests/rl/renderer/test_token_mask_lengths.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from breadboard.rl.renderer.records import validate_rendered_turn +from breadboard.rl.renderer.schema import RenderedTurnRecord +from tests.rl.renderer.helpers import cloned_payload + + +def test_valid_token_mask_lengths_pass() -> None: + record = RenderedTurnRecord.from_dict(cloned_payload()) + + assert validate_rendered_turn(record) == [] + + +def test_input_ids_must_match_prompt_plus_completion() -> None: + payload = cloned_payload() + payload["input_ids"] = [10, 11, 12, 99, 21] + + errors = validate_rendered_turn(RenderedTurnRecord.from_dict(payload)) + assert "input_ids must equal prompt_ids + completion_ids" in errors + + +def test_masks_must_align_to_input_ids_length() -> None: + payload = cloned_payload() + payload["loss_mask"] = [False, False] + + errors = validate_rendered_turn(RenderedTurnRecord.from_dict(payload)) + assert "loss_mask length must equal input_ids length" in errors + + +def test_completion_logprobs_must_align_to_completion_ids() -> None: + payload = cloned_payload() + payload["completion_logprobs"] = [-0.1] + + errors = validate_rendered_turn(RenderedTurnRecord.from_dict(payload)) + assert "completion_logprobs length must equal completion_ids length" in errors diff --git a/tests/rl/renderer/test_tool_action_mask.py b/tests/rl/renderer/test_tool_action_mask.py new file mode 100644 index 00000000..f60ae97d --- /dev/null +++ b/tests/rl/renderer/test_tool_action_mask.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from breadboard.rl.renderer.records import validate_rendered_turn +from breadboard.rl.renderer.schema import RenderedTurnRecord +from tests.rl.renderer.helpers import cloned_payload + + +def test_tool_call_requires_tool_action_mask() -> None: + payload = cloned_payload() + payload["tool_parse_status"] = "ok" + payload["tool_calls"] = [{"name": "python", "arguments": {"code": "1+1"}}] + + errors = validate_rendered_turn(RenderedTurnRecord.from_dict(payload)) + assert "tool_calls require at least one true tool_action_mask entry" in errors + + +def test_tool_call_with_action_mask_passes() -> None: + payload = cloned_payload() + payload["tool_parse_status"] = "ok" + payload["tool_calls"] = [{"name": "python", "arguments": {"code": "1+1"}}] + payload["tool_action_mask"] = [False, False, False, True, True] + + assert validate_rendered_turn(RenderedTurnRecord.from_dict(payload)) == [] + + +def test_tool_calls_require_parse_status_ok() -> None: + payload = cloned_payload() + payload["tool_parse_status"] = "failed" + payload["tool_calls"] = [{"name": "python", "arguments": {}}] + payload["tool_action_mask"] = [False, False, False, True, True] + + errors = validate_rendered_turn(RenderedTurnRecord.from_dict(payload)) + assert "tool_calls require tool_parse_status=ok" in errors diff --git a/tests/rl/renderer/test_truncation_flags.py b/tests/rl/renderer/test_truncation_flags.py new file mode 100644 index 00000000..854b5808 --- /dev/null +++ b/tests/rl/renderer/test_truncation_flags.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from breadboard.rl.renderer.records import classify_rendered_turn_trainability +from breadboard.rl.renderer.schema import RenderedTurnRecord +from tests.rl.renderer.helpers import cloned_payload + + +def test_f3_non_truncated_record_is_on_policy_trainable() -> None: + decision = classify_rendered_turn_trainability(RenderedTurnRecord.from_dict(cloned_payload())) + + assert decision.sft_trainable is True + assert decision.on_policy_trainable is True + assert decision.blocked_reasons == [] + + +def test_finish_reason_length_blocks_trainability() -> None: + payload = cloned_payload() + payload["finish_reason"] = "length" + + decision = classify_rendered_turn_trainability(RenderedTurnRecord.from_dict(payload)) + assert "finish_reason=length" in decision.blocked_reasons + assert decision.sft_trainable is False + assert decision.on_policy_trainable is False + + +def test_overlong_prompt_blocks_trainability() -> None: + payload = cloned_payload() + payload["overlong_prompt"] = True + + decision = classify_rendered_turn_trainability(RenderedTurnRecord.from_dict(payload)) + assert "overlong_prompt" in decision.blocked_reasons + assert decision.sft_trainable is False diff --git a/tests/rl/replay/test_admission_rules.py b/tests/rl/replay/test_admission_rules.py new file mode 100644 index 00000000..e768e593 --- /dev/null +++ b/tests/rl/replay/test_admission_rules.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from breadboard.rl.replay import ReplayParityReport, decide_export_admission + + +def test_admission_accepts_exportable_debug_for_clean_replay() -> None: + report = ReplayParityReport(parity_tier="T2_deterministic_runtime_verifier", passed=True) + + decision = decide_export_admission(replay_report=report) + + assert decision.exportable is True + assert decision.trainable is False + assert decision.blocked_reasons == [] + + +def test_admission_rejects_replay_mismatch() -> None: + report = ReplayParityReport( + parity_tier="T2_deterministic_runtime_verifier", + passed=False, + mismatches=["reward_mismatch"], + ) + + decision = decide_export_admission(replay_report=report) + + assert decision.exportable is False + assert decision.trainable is False + assert "replay_mismatch" in decision.blocked_reasons + + +def test_admission_rejects_quarantine_and_bad_token_records() -> None: + report = ReplayParityReport(parity_tier="T2_deterministic_runtime_verifier", passed=True) + + decision = decide_export_admission( + replay_report=report, + quarantine_status="quarantined", + token_records_valid=False, + ) + + assert decision.exportable is False + assert "quarantine_status=quarantined" in decision.blocked_reasons + assert "token_records_invalid" in decision.blocked_reasons diff --git a/tests/rl/replay/test_live_replay_parity.py b/tests/rl/replay/test_live_replay_parity.py new file mode 100644 index 00000000..f16e0003 --- /dev/null +++ b/tests/rl/replay/test_live_replay_parity.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from dataclasses import replace + +from breadboard.rl.replay import compare_replay_parity +from breadboard.rl.trace import build_graph_from_session_events +from breadboard.rl.trace.graph import TrajectoryGraph +from tests.rl.session.helpers import build_successful_toy_session + + +def test_identical_live_and_replay_graphs_pass_t2_parity() -> None: + session = build_successful_toy_session() + live = build_graph_from_session_events( + graph_id="toy.live", + session_id=session.session_id, + events=session.events, + ) + replay = TrajectoryGraph(graph_id="toy.replay", nodes=live.nodes, edges=live.edges) + + report = compare_replay_parity(live, replay) + + assert report.passed is True + assert report.parity_tier == "T2_deterministic_runtime_verifier" + + +def test_reward_mismatch_fails_replay_parity() -> None: + session = build_successful_toy_session() + live = build_graph_from_session_events( + graph_id="toy.live", + session_id=session.session_id, + events=session.events, + ) + mutated_nodes = list(live.nodes) + eval_node = mutated_nodes[-1] + payload = eval_node.payload.copy() + nested_payload = payload["payload"].copy() + nested_payload["reward"] = 0.0 + payload["payload"] = nested_payload + mutated_nodes[-1] = replace(eval_node, payload=payload) + replay = TrajectoryGraph(graph_id="toy.replay", nodes=mutated_nodes, edges=live.edges) + + report = compare_replay_parity(live, replay) + + assert report.passed is False + assert "reward_mismatch" in report.mismatches diff --git a/tests/rl/runtime/__init__.py b/tests/rl/runtime/__init__.py new file mode 100644 index 00000000..de18cf20 --- /dev/null +++ b/tests/rl/runtime/__init__.py @@ -0,0 +1 @@ +"""Runtime test helpers.""" diff --git a/tests/rl/runtime/test_backend_contract.py b/tests/rl/runtime/test_backend_contract.py new file mode 100644 index 00000000..7fca9be1 --- /dev/null +++ b/tests/rl/runtime/test_backend_contract.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.runtime.local_process import LocalProcessToyRuntime + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_local_process_runtime_exposes_backend_contract_methods() -> None: + runtime = LocalProcessToyRuntime(load_env_package(PYTHON_TOY), "py_toy_001") + + for method_name in [ + "health", + "reset", + "observe", + "step", + "evaluate", + "snapshot", + "restore", + "terminate", + ]: + assert callable(getattr(runtime, method_name)) + + +def test_evaluate_without_submission_is_structured_error() -> None: + runtime = LocalProcessToyRuntime(load_env_package(PYTHON_TOY), "py_toy_001") + runtime.reset() + + result = runtime.evaluate() + + assert result.success is False + assert result.error["kind"] == "no_submission" diff --git a/tests/rl/runtime/test_pool_routing.py b/tests/rl/runtime/test_pool_routing.py new file mode 100644 index 00000000..1c855d06 --- /dev/null +++ b/tests/rl/runtime/test_pool_routing.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from breadboard.rl.runtime.pool import ASSIGNED, QUARANTINED, READY, RuntimePool, WorkerRecord + + +def test_pool_routes_exact_ready_signature() -> None: + pool = RuntimePool() + pool.register(WorkerRecord(worker_id="w1", signature_digest="sig-a")) + pool.register(WorkerRecord(worker_id="w2", signature_digest="sig-b")) + + worker = pool.route("sig-a") + + assert worker is not None + assert worker.worker_id == "w1" + assert worker.state == ASSIGNED + pool.release("w1") + assert pool.workers["w1"].state == READY + + +def test_pool_does_not_route_quarantined_worker() -> None: + pool = RuntimePool() + pool.register(WorkerRecord(worker_id="w1", signature_digest="sig-a")) + pool.quarantine("w1", "poisoned") + + assert pool.route("sig-a") is None + assert pool.workers["w1"].state == QUARANTINED diff --git a/tests/rl/runtime/test_ray_worker_smoke.py b/tests/rl/runtime/test_ray_worker_smoke.py new file mode 100644 index 00000000..82ade312 --- /dev/null +++ b/tests/rl/runtime/test_ray_worker_smoke.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.runtime import run_local_ray_toy_probe + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_local_ray_worker_executes_toy_sessions() -> None: + package = load_env_package(PYTHON_TOY) + report = run_local_ray_toy_probe( + package=package, + task_ids=["py_toy_001", "py_toy_002", "py_toy_003", "py_toy_004"], + num_workers=2, + ) + + assert report["row_count"] == 4 + assert report["worker_count"] == 2 + assert report["ray_local_mode"] is True + assert all(row["reward"] == 1.0 for row in report["rows"]) + assert all(row["event_count"] == 3 for row in report["rows"]) diff --git a/tests/rl/runtime/test_runtime_health.py b/tests/rl/runtime/test_runtime_health.py new file mode 100644 index 00000000..e332e6b1 --- /dev/null +++ b/tests/rl/runtime/test_runtime_health.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.runtime.local_process import LocalProcessToyRuntime + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" +SWE_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "swe_toy_patch" / "env_package.yaml" + + +def test_local_process_runtime_health_ready_for_python_toy() -> None: + runtime = LocalProcessToyRuntime(load_env_package(PYTHON_TOY), "py_toy_001") + + assert runtime.health().ready is True + + +def test_local_process_runtime_health_rejects_docker_swe_package() -> None: + runtime = LocalProcessToyRuntime(load_env_package(SWE_TOY), "swe_toy_001") + + health = runtime.health() + assert health.ready is False + assert "runtime.backend must be local_process" in health.reasons + assert "local toy runtime requires verifier.kind=exact_match" in health.reasons diff --git a/tests/rl/runtime/test_runtime_signature.py b/tests/rl/runtime/test_runtime_signature.py new file mode 100644 index 00000000..604865d8 --- /dev/null +++ b/tests/rl/runtime/test_runtime_signature.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.runtime import build_runtime_signature + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_runtime_signature_is_stable_and_hash_addressed() -> None: + package = load_env_package(PYTHON_TOY) + sig1 = build_runtime_signature(package) + sig2 = build_runtime_signature(package) + + assert sig1.digest().startswith("sha256:") + assert sig1.digest() == sig2.digest() + + +def test_runtime_signature_changes_with_resource_class() -> None: + package = load_env_package(PYTHON_TOY) + + assert build_runtime_signature(package, resource_class="cpu_local").digest() != build_runtime_signature( + package, + resource_class="cpu_large", + ).digest() diff --git a/tests/rl/runtime/test_telemetry_report.py b/tests/rl/runtime/test_telemetry_report.py new file mode 100644 index 00000000..2de08b12 --- /dev/null +++ b/tests/rl/runtime/test_telemetry_report.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from breadboard.rl.runtime import build_warm_vs_cold_report, summarize_stage_metrics + + +def test_stage_metrics_report_p50_p95() -> None: + rows = [ + {"metrics_ms": {"total_ms": 10.0, "reset_ms": 2.0}}, + {"metrics_ms": {"total_ms": 20.0, "reset_ms": 4.0}}, + {"metrics_ms": {"total_ms": 30.0, "reset_ms": 6.0}}, + ] + + summary = summarize_stage_metrics(rows) + + assert summary["total_ms"]["p50"] == 20.0 + assert summary["total_ms"]["p95"] == 30.0 + assert summary["reset_ms"]["count"] == 3.0 + + +def test_warm_vs_cold_report_preserves_claim_boundary() -> None: + warm = [{"metrics_ms": {"total_ms": 10.0}}] + cold = [{"metrics_ms": {"total_ms": 15.0}}] + + report = build_warm_vs_cold_report(warm_rows=warm, cold_rows=cold) + + assert report["warm"]["total_ms"]["p50"] == 10.0 + assert report["cold"]["total_ms"]["p50"] == 15.0 + assert report["claim_boundary"] == "local_ray_warm_pool_probe_not_production_scale" diff --git a/tests/rl/security/__init__.py b/tests/rl/security/__init__.py new file mode 100644 index 00000000..3b765b93 --- /dev/null +++ b/tests/rl/security/__init__.py @@ -0,0 +1 @@ +"""Security test helpers.""" diff --git a/tests/rl/security/helpers.py b/tests/rl/security/helpers.py new file mode 100644 index 00000000..de2572cc --- /dev/null +++ b/tests/rl/security/helpers.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package + + +REPO_ROOT = Path(__file__).resolve().parents[3] +SWE_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "swe_toy_patch" / "env_package.yaml" + + +def load_swe_hardening_policy(): + package = load_env_package(SWE_TOY) + assert package.hardening is not None + return package.hardening diff --git a/tests/rl/security/test_process_cleanup_contract.py b/tests/rl/security/test_process_cleanup_contract.py new file mode 100644 index 00000000..78132ac1 --- /dev/null +++ b/tests/rl/security/test_process_cleanup_contract.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from breadboard.rl.security import validate_process_cleanup_before_verify + + +def test_process_cleanup_required_before_verify() -> None: + assert validate_process_cleanup_before_verify(["reset", "verify"]) == [ + "process_cleanup event is required before verify" + ] + assert validate_process_cleanup_before_verify(["reset", "verify", "process_cleanup"]) == [ + "process_cleanup must occur before verify" + ] + assert validate_process_cleanup_before_verify(["reset", "process_cleanup", "verify"]) == [] diff --git a/tests/rl/security/test_python_import_hook_cleanup.py b/tests/rl/security/test_python_import_hook_cleanup.py new file mode 100644 index 00000000..6cf2eb36 --- /dev/null +++ b/tests/rl/security/test_python_import_hook_cleanup.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from breadboard.rl.security import scan_python_import_hooks + + +def test_python_import_hook_files_are_detected(tmp_path) -> None: + (tmp_path / "sitecustomize.py").write_text("pass\n", encoding="utf-8") + (tmp_path / "usercustomize.py").write_text("pass\n", encoding="utf-8") + (tmp_path / "poison.pth").write_text("import x\n", encoding="utf-8") + (tmp_path / "conftest.py").write_text("pytest_plugins=[]\n", encoding="utf-8") + + finding_ids = {item.finding_id for item in scan_python_import_hooks(tmp_path)} + + assert {"sitecustomize", "usercustomize", "pth_injection", "conftest_outside_tests"} <= finding_ids + + +def test_conftest_under_tests_is_allowed(tmp_path) -> None: + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + (tests_dir / "conftest.py").write_text("pass\n", encoding="utf-8") + + assert scan_python_import_hooks(tmp_path) == [] diff --git a/tests/rl/security/test_quarantine_rules.py b/tests/rl/security/test_quarantine_rules.py new file mode 100644 index 00000000..a6222d8a --- /dev/null +++ b/tests/rl/security/test_quarantine_rules.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from breadboard.rl.security.hardening import HardeningFinding +from breadboard.rl.security.quarantine import quarantine_on_findings +from tests.rl.security.helpers import load_swe_hardening_policy + + +def test_policy_listed_finding_quarantines_row() -> None: + decision = quarantine_on_findings( + row_id="row-1", + policy=load_swe_hardening_policy(), + findings=[ + HardeningFinding( + finding_id="sitecustomize_shadow", + severity="medium", + path="sitecustomize.py", + message="detected", + ) + ], + ) + + assert decision.quarantined is True + assert "sitecustomize_shadow" in decision.reasons + + +def test_high_severity_unknown_finding_quarantines_by_default() -> None: + decision = quarantine_on_findings( + row_id="row-1", + policy=load_swe_hardening_policy(), + findings=[ + HardeningFinding( + finding_id="unknown_high_risk", + severity="high", + path="x", + message="detected", + ) + ], + ) + + assert decision.quarantined is True + assert "unknown_high_risk" in decision.reasons diff --git a/tests/rl/security/test_reward_hack_probe_suite.py b/tests/rl/security/test_reward_hack_probe_suite.py new file mode 100644 index 00000000..a33b2de3 --- /dev/null +++ b/tests/rl/security/test_reward_hack_probe_suite.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from breadboard.rl.security import run_probe_suite +from breadboard.rl.security.reward_hack_suites import ( + SWE_REWARD_HACK_PROBES, + build_swe_reward_hack_probe_suite, +) +from tests.rl.security.helpers import load_swe_hardening_policy + + +def test_swe_reward_hack_probe_suite_executes_and_quarantines_adversarial_fixtures(tmp_path) -> None: + results = run_probe_suite( + workspace=tmp_path, + policy=load_swe_hardening_policy(), + probes=build_swe_reward_hack_probe_suite(), + ) + + assert [item.probe_id for item in results] == SWE_REWARD_HACK_PROBES + assert all(item.status == "quarantined" for item in results) + assert all(item.findings for item in results) diff --git a/tests/rl/security/test_swe_hardening_policy.py b/tests/rl/security/test_swe_hardening_policy.py new file mode 100644 index 00000000..154e5a93 --- /dev/null +++ b/tests/rl/security/test_swe_hardening_policy.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from breadboard.rl.security import build_hardening_report +from tests.rl.security.helpers import load_swe_hardening_policy + + +def test_clean_workspace_hardening_passes_with_baseline_and_cleanup(tmp_path) -> None: + policy = load_swe_hardening_policy() + report = build_hardening_report( + report_id="clean", + workspace=tmp_path, + policy=policy, + clean_baseline_passed=True, + reference_solution_passed=True, + process_cleanup_observed=True, + ) + + assert report.status == "passed" + assert report.findings == [] + + +def test_hardening_fails_if_clean_baseline_breaks(tmp_path) -> None: + policy = load_swe_hardening_policy() + report = build_hardening_report( + report_id="broken-clean", + workspace=tmp_path, + policy=policy, + clean_baseline_passed=False, + reference_solution_passed=True, + process_cleanup_observed=True, + ) + + assert report.status == "failed" + assert report.clean_baseline_passed is False + + +def test_missing_process_cleanup_quarantines_when_required(tmp_path) -> None: + policy = load_swe_hardening_policy() + report = build_hardening_report( + report_id="no-cleanup", + workspace=tmp_path, + policy=policy, + clean_baseline_passed=True, + reference_solution_passed=True, + process_cleanup_observed=False, + ) + + assert report.status == "quarantined" + assert any(item.finding_id == "missing_process_cleanup" for item in report.findings) diff --git a/tests/rl/security/test_symlink_escape.py b/tests/rl/security/test_symlink_escape.py new file mode 100644 index 00000000..c0b574cb --- /dev/null +++ b/tests/rl/security/test_symlink_escape.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from breadboard.rl.security import scan_symlink_escapes + + +def test_symlink_escape_is_detected(tmp_path) -> None: + (tmp_path / "escape").symlink_to("/etc/passwd") + + findings = scan_symlink_escapes(tmp_path) + + assert len(findings) == 1 + assert findings[0].finding_id == "symlink_escape" + + +def test_internal_symlink_is_allowed(tmp_path) -> None: + target = tmp_path / "target.txt" + target.write_text("ok\n", encoding="utf-8") + (tmp_path / "inside").symlink_to(target) + + assert scan_symlink_escapes(tmp_path) == [] diff --git a/tests/rl/security/test_verifier_evidence_hashes.py b/tests/rl/security/test_verifier_evidence_hashes.py new file mode 100644 index 00000000..1798a5de --- /dev/null +++ b/tests/rl/security/test_verifier_evidence_hashes.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from breadboard.rl.security.reports import VerifierRunReport, build_verifier_run_report + + +def test_verifier_report_hash_matches_output() -> None: + report = build_verifier_run_report( + report_id="verify-1", + verifier_id="pytest", + status="passed", + output="passed", + rerun_output="passed", + ) + + assert report.verify_evidence_hash() is True + assert report.rerun_agreement is True + + +def test_verifier_report_hash_mismatch_is_detected() -> None: + report = VerifierRunReport( + report_id="verify-1", + verifier_id="pytest", + status="passed", + output="tampered", + evidence_sha256="sha256:bad", + ) + + assert report.verify_evidence_hash() is False + assert report.to_dict()["evidence_hash_valid"] is False diff --git a/tests/rl/session/__init__.py b/tests/rl/session/__init__.py new file mode 100644 index 00000000..e7228d2c --- /dev/null +++ b/tests/rl/session/__init__.py @@ -0,0 +1 @@ +"""Session test helpers.""" diff --git a/tests/rl/session/helpers.py b/tests/rl/session/helpers.py new file mode 100644 index 00000000..9a1079c6 --- /dev/null +++ b/tests/rl/session/helpers.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.session.controller import LocalSession, create_local_session + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def build_successful_toy_session() -> LocalSession: + session = create_local_session(load_env_package(PYTHON_TOY), "py_toy_001") + session.reset() + session.observe() + session.step({"tool": "submit_answer", "answer": "42"}) + session.evaluate() + return session diff --git a/tests/rl/session/test_lifecycle_toy.py b/tests/rl/session/test_lifecycle_toy.py new file mode 100644 index 00000000..4f2be2e3 --- /dev/null +++ b/tests/rl/session/test_lifecycle_toy.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.session import SessionStatus, create_local_session + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_full_toy_lifecycle_success_path() -> None: + package = load_env_package(PYTHON_TOY) + session = create_local_session(package, "py_toy_001") + + assert session.health().ready is True + reset = session.reset() + assert reset.success is True + assert session.lifecycle.status == SessionStatus.READY + + observed = session.observe() + assert observed.success is True + assert observed.observation["task_id"] == "py_toy_001" + + step = session.step({"tool": "submit_answer", "answer": "42"}) + assert step.success is True + assert step.done is True + assert session.lifecycle.status == SessionStatus.RUNNING + + snapshot = session.snapshot() + assert snapshot.state["submitted_answer"] == "42" + + evaluation = session.evaluate() + assert evaluation.success is True + assert evaluation.reward == 1.0 + assert session.lifecycle.status == SessionStatus.EVALUATED + assert session.export_admission()["exportable_debug"] is True + assert session.export_admission()["trainable"] is False + + termination = session.terminate() + assert termination.success is True + assert session.lifecycle.status == SessionStatus.TERMINATED + assert [event.event_kind for event in session.events] == [ + "reset", + "observe", + "step", + "snapshot", + "evaluate", + "terminate", + ] + + +def test_snapshot_restore_round_trip() -> None: + package = load_env_package(PYTHON_TOY) + session = create_local_session(package, "py_toy_001") + session.reset() + snapshot = session.snapshot() + session.step({"tool": "submit_answer", "answer": "wrong"}) + + restored = session.restore(snapshot) + assert restored.success is True + assert restored.observation["submitted"] is False + + session.step({"tool": "submit_answer", "answer": "42"}) + assert session.evaluate().reward == 1.0 diff --git a/tests/rl/session/test_state_machine.py b/tests/rl/session/test_state_machine.py new file mode 100644 index 00000000..b3e1faa2 --- /dev/null +++ b/tests/rl/session/test_state_machine.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.session import SessionStatus, create_local_session + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_evaluate_before_reset_is_rejected() -> None: + session = create_local_session(load_env_package(PYTHON_TOY), "py_toy_001") + + with pytest.raises(ValueError, match="evaluate requires ready/running"): + session.evaluate() + + +def test_step_after_terminate_is_rejected() -> None: + session = create_local_session(load_env_package(PYTHON_TOY), "py_toy_001") + session.terminate() + + with pytest.raises(ValueError, match="step requires ready/running"): + session.step({"tool": "submit_answer", "answer": "42"}) + assert session.lifecycle.status == SessionStatus.TERMINATED + + +def test_invalid_second_reset_after_evaluation_is_rejected() -> None: + session = create_local_session(load_env_package(PYTHON_TOY), "py_toy_001") + session.reset() + session.step({"tool": "submit_answer", "answer": "42"}) + session.evaluate() + + with pytest.raises(ValueError, match="invalid session transition"): + session.reset() diff --git a/tests/rl/session/test_termination_cleanup.py b/tests/rl/session/test_termination_cleanup.py new file mode 100644 index 00000000..ac1d0290 --- /dev/null +++ b/tests/rl/session/test_termination_cleanup.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.session import SessionStatus, create_local_session + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_termination_cleanup_runs_after_error() -> None: + session = create_local_session(load_env_package(PYTHON_TOY), "py_toy_001") + session.reset() + session.step({"tool": "runtime_crash"}) + + result = session.terminate() + + assert result.success is True + assert session.lifecycle.status == SessionStatus.TERMINATED + assert session.events[-1].event_kind == "terminate" + assert session.events[-1].status_before == SessionStatus.FAILED + + +def test_terminate_is_idempotent() -> None: + session = create_local_session(load_env_package(PYTHON_TOY), "py_toy_001") + first = session.terminate() + second = session.terminate() + + assert first.success is True + assert second.success is True + assert second.evidence["already_terminated"] is True diff --git a/tests/rl/session/test_timeout_policy.py b/tests/rl/session/test_timeout_policy.py new file mode 100644 index 00000000..7cb9d5d0 --- /dev/null +++ b/tests/rl/session/test_timeout_policy.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from pathlib import Path + +from breadboard.rl.env_package.validate import load_env_package +from breadboard.rl.session import SessionStatus, create_local_session + + +REPO_ROOT = Path(__file__).resolve().parents[3] +PYTHON_TOY = REPO_ROOT / "examples" / "rl_env_packages" / "python_console_toy" / "env_package.yaml" + + +def test_step_timeout_produces_structured_failure_and_failed_session() -> None: + session = create_local_session(load_env_package(PYTHON_TOY), "py_toy_001") + session.reset() + + result = session.step({"tool": "sleep", "seconds": 999}) + + assert result.success is False + assert result.error["kind"] == "timeout" + assert session.lifecycle.status == SessionStatus.FAILED + assert session.export_admission()["trainable"] is False + assert "lifecycle_status=failed" in session.export_admission()["blocked_reasons"] + + +def test_runtime_crash_produces_structured_failure() -> None: + session = create_local_session(load_env_package(PYTHON_TOY), "py_toy_001") + session.reset() + + result = session.step({"tool": "runtime_crash"}) + + assert result.success is False + assert result.error["kind"] == "runtime_crash" + assert session.events[-1].error["kind"] == "runtime_crash" diff --git a/tests/rl/state/test_cas_refs.py b/tests/rl/state/test_cas_refs.py new file mode 100644 index 00000000..208b39f0 --- /dev/null +++ b/tests/rl/state/test_cas_refs.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import pytest + +from breadboard.rl.state import InMemoryCAS + + +def test_cas_refs_are_hash_addressed_and_retrievable() -> None: + cas = InMemoryCAS() + ref = cas.put_bytes(b"hello", media_type="text/plain") + + assert ref.sha256.startswith("sha256:") + assert ref.size_bytes == 5 + assert cas.has(ref) + assert cas.get_bytes(ref) == b"hello" + + +def test_cas_rejects_overwrite_for_existing_artifact_id() -> None: + cas = InMemoryCAS() + cas.put_bytes(b"first", artifact_id="artifact-1") + + with pytest.raises(ValueError, match="overwrite rejected"): + cas.put_bytes(b"second", artifact_id="artifact-1") diff --git a/tests/rl/state/test_snapshot_manifest.py b/tests/rl/state/test_snapshot_manifest.py new file mode 100644 index 00000000..19266eb8 --- /dev/null +++ b/tests/rl/state/test_snapshot_manifest.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from breadboard.rl.state import InMemoryCAS, build_snapshot_manifest +from tests.rl.session.helpers import build_successful_toy_session + + +def test_snapshot_manifest_records_runtime_and_package_hash() -> None: + session = build_successful_toy_session() + snapshot = session.snapshot() + cas = InMemoryCAS() + artifact = cas.put_bytes(b"state-note", artifact_id="state-note") + + manifest = build_snapshot_manifest( + snapshot=snapshot, + package_hash=session.package.package_hash or "", + runtime_backend=session.runtime.backend_id, + artifact_refs=[artifact], + event_ids=[event.event_id for event in session.events], + ) + + assert manifest.package_hash == session.package.package_hash + assert manifest.runtime_backend == "local_process" + assert manifest.state_ref.state_hash.startswith("sha256:") + assert manifest.state_ref.artifact_refs[0].artifact_id == "state-note" + assert manifest.event_ids diff --git a/tests/rl/trace/test_credit_frame.py b/tests/rl/trace/test_credit_frame.py new file mode 100644 index 00000000..f20ec993 --- /dev/null +++ b/tests/rl/trace/test_credit_frame.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from breadboard.rl.trace import build_graph_from_session_events, build_terminal_credit_frame +from tests.rl.session.helpers import build_successful_toy_session + + +def test_terminal_reward_credit_frame_targets_step_and_evaluate_nodes() -> None: + session = build_successful_toy_session() + graph = build_graph_from_session_events( + graph_id="toy.graph", + session_id=session.session_id, + events=session.events, + ) + + frame = build_terminal_credit_frame(graph, reward=1.0) + + assert frame.reward == 1.0 + assert any(node_id.endswith(".event.3") for node_id in frame.credited_node_ids) + assert any(node_id.endswith(".event.4") for node_id in frame.credited_node_ids) diff --git a/tests/rl/trace/test_graph_invariants.py b/tests/rl/trace/test_graph_invariants.py new file mode 100644 index 00000000..ca717433 --- /dev/null +++ b/tests/rl/trace/test_graph_invariants.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from breadboard.rl.trace import build_graph_from_session_events, validate_graph_invariants +from breadboard.rl.trace.edges import TraceEdge +from breadboard.rl.trace.graph import TrajectoryGraph +from breadboard.rl.trace.nodes import TraceNode +from tests.rl.session.helpers import build_successful_toy_session + + +def test_toy_session_emits_valid_trajectory_graph() -> None: + session = build_successful_toy_session() + graph = build_graph_from_session_events( + graph_id="toy.graph", + session_id=session.session_id, + events=session.events, + ) + + assert validate_graph_invariants(graph) == [] + assert [node.node_kind for node in graph.nodes] == ["reset", "observe", "step", "evaluate"] + assert len(graph.edges) == 3 + + +def test_graph_invariants_reject_missing_edge_target() -> None: + graph = TrajectoryGraph( + graph_id="bad.graph", + nodes=[TraceNode(node_id="n1", node_kind="reset")], + edges=[TraceEdge(edge_id="e1", source_id="n1", target_id="missing", edge_kind="session_order")], + ) + + assert "edge e1 references missing target_id" in validate_graph_invariants(graph) diff --git a/tests/test_rl_phase1_claim_ledger.py b/tests/test_rl_phase1_claim_ledger.py new file mode 100644 index 00000000..03e3c399 --- /dev/null +++ b/tests/test_rl_phase1_claim_ledger.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[1] +WORKSPACE_ROOT = REPO_ROOT.parent +PHASE_DIR = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" +CLAIM_LEDGER = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_CLAIM_LEDGER.md" +SCORECARD = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" +CONFIDENCE_AUDIT = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_STRATEGY_CONFIDENCE_AUDIT.md" + + +def test_claim_ledger_exists_and_tracks_current_score() -> None: + text = CLAIM_LEDGER.read_text(encoding="utf-8") + scorecard = yaml.safe_load(SCORECARD.read_text(encoding="utf-8")) + expected_score = f"{scorecard['current_verified_points']} / {scorecard['total_points']}" + + assert SCORECARD.exists() + assert "Current verified score:" in text + assert expected_score in text + assert "Current Allowed Claims" in text + assert "Forbidden Current Claims" in text + assert "Claim Update Rule" in text + + +def test_claim_ledger_keeps_support_claims_forbidden_until_evidence() -> None: + text = CLAIM_LEDGER.read_text(encoding="utf-8") + + forbidden_claims = [ + "BreadBoard supports production RL rollouts.", + "BreadBoard is VeRL/GRPO/PPO ready.", + "BreadBoard supports hardened SWE RL at scale.", + "BreadBoard supports BenchFlow as a production integration.", + "BreadBoard supports ORS/OpenReward as a production integration.", + "BreadBoard supports NeMo Gym as a production integration.", + "BreadBoard supports Prime Verifiers as a production integration.", + "BreadBoard supports Toolathlon-Gym as a production integration.", + "BreadBoard supports ProRL/Polar as a production integration.", + "BreadBoard supports kernels RL.", + "BreadBoard supports search RL.", + "BreadBoard has generalized multi-family RL environment support.", + "The RL Phase 1 strategy is guaranteed to succeed.", + "The RL Phase 1 strategy is factually 100% certain in the predictive sense.", + ] + + for claim in forbidden_claims: + assert claim in text + + +def test_claim_ledger_requires_evidence_for_future_claim_promotion() -> None: + text = CLAIM_LEDGER.read_text(encoding="utf-8") + + assert "Milestone id" in text + assert "Evidence paths" in text + assert "Test commands" in text + assert "Known caveats" in text + assert "Regression policy" in text + assert "No claim may be promoted solely because a plan says it should be possible." in text + + +def test_claim_ledger_references_confidence_audit() -> None: + text = CLAIM_LEDGER.read_text(encoding="utf-8") + + assert CONFIDENCE_AUDIT.exists() + assert "STRATEGY_CONFIDENCE_AUDIT" in text + assert "rejects literal 100% certainty" in text + assert "fail-closed confidence protocol" in text + + +def test_claim_ledger_uses_bounded_confidence_classes() -> None: + text = CLAIM_LEDGER.read_text(encoding="utf-8") + + assert "Deterministic local confidence" in text + assert "Probe-backed conditional confidence" in text + assert "External-infrastructure confidence" in text + assert "Blocked/conditional until target-environment evidence exists." in text diff --git a/tests/test_rl_phase1_scorecard_schema.py b/tests/test_rl_phase1_scorecard_schema.py new file mode 100644 index 00000000..d6947714 --- /dev/null +++ b/tests/test_rl_phase1_scorecard_schema.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[1] +WORKSPACE_ROOT = REPO_ROOT.parent +PHASE_DIR = WORKSPACE_ROOT / "docs_tmp" / "ZYPHRA" / "RL_PHASE_1" +SCORECARD = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_SCORECARD.yaml" +PLAN = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_EXECUTION_PLAN.md" +CONFIDENCE_AUDIT = PHASE_DIR / "BB_ZYPHRA_RL_PHASE_1_STRATEGY_CONFIDENCE_AUDIT.md" + + +def load_scorecard() -> dict: + return yaml.safe_load(SCORECARD.read_text(encoding="utf-8")) + + +def test_scorecard_totals_and_verified_points_are_consistent() -> None: + scorecard = load_scorecard() + milestones = scorecard["milestones"] + + assert scorecard["total_points"] == 1000 + assert sum(milestone["points"] for milestone in milestones) == 1000 + assert scorecard["current_verified_points"] == sum( + milestone["verified_points"] for milestone in milestones + ) + assert all(milestone["points"] >= 0 for milestone in milestones) + assert all(milestone["verified_points"] >= 0 for milestone in milestones) + assert all( + milestone["verified_points"] <= milestone["points"] for milestone in milestones + ) + + +def test_scorecard_preserves_phase1_claim_and_evidence_policy() -> None: + scorecard = load_scorecard() + policy = scorecard["score_policy"] + + assert policy["planning_prose_scores"] is False + assert policy["points_require_evidence"] is True + assert "committed_or_written_artifact_path" in policy["evidence_required"] + assert "test_command_or_validation_command" in policy["evidence_required"] + assert "pass_fail_summary" in policy["evidence_required"] + + forbidden = "\n".join(scorecard["forbidden_current_claims"]) + assert "production RL rollouts" in forbidden + assert "VeRL/GRPO/PPO ready" in forbidden + assert "hardened SWE RL at scale" in forbidden + + +def test_scorecard_has_all_expected_milestones_in_order() -> None: + scorecard = load_scorecard() + milestone_ids = [milestone["id"] for milestone in scorecard["milestones"]] + + assert milestone_ids == [ + "M0", + "M1", + "M2", + "M3", + "M4", + "M5", + "M6", + "M7", + "M8", + "M9", + "M10", + "M11", + "M12", + ] + assert scorecard["milestones"][-1]["status"] == "completed" + + +def test_control_plan_references_confidence_audit_and_hard_gates() -> None: + plan_text = PLAN.read_text(encoding="utf-8") + + assert CONFIDENCE_AUDIT.exists() + assert "Strategy Confidence Boundary" in plan_text + assert "Source selection fallback ladder" in plan_text + assert "Provider fidelity classes" in plan_text + assert "Non-negotiable M12 preflight" in plan_text + assert "Broad generalized RL support" in plan_text + assert "Factually Bounded Confidence Protocol" in plan_text + assert "What would falsify this milestone claim?" in plan_text + + +def test_confidence_audit_fail_closed_protocol_exists() -> None: + audit_text = CONFIDENCE_AUDIT.read_text(encoding="utf-8") + + assert "Round 3: Execution-Closure Loopholes" in audit_text + assert "Round 4: Final Confidence Test" in audit_text + assert "Factually Bounded Confidence Protocol" in audit_text + assert "Kill Switches" in audit_text + assert "No, not in the predictive sense." in audit_text + assert "fail-closed" in audit_text