diff --git a/agentic_coder_prototype/provider/runtime.py b/agentic_coder_prototype/provider/runtime.py index 1840f573..65c55811 100644 --- a/agentic_coder_prototype/provider/runtime.py +++ b/agentic_coder_prototype/provider/runtime.py @@ -3,15 +3,17 @@ from __future__ import annotations import base64 +import hashlib import datetime import json +import math import os import random import re from dataclasses import dataclass, field import time from types import SimpleNamespace -from typing import Any, Dict, List, Optional, Tuple, Type +from typing import Any, Dict, List, Mapping, Optional, Tuple, Type import textwrap try: # pragma: no cover - import guard exercised in runtime @@ -79,6 +81,115 @@ class ProviderResult: metadata: Dict[str, Any] = field(default_factory=dict) +def _provider_evidence_value(value: Any, *, seen: frozenset[int] = frozenset()) -> Any: + if value is None or type(value) in (bool, str, int, float): + json.dumps(value, allow_nan=False) + return value + if isinstance(value, dict): + if id(value) in seen or any(type(key) is not str for key in value): + raise ProviderRuntimeError("provider evidence contains a cyclic object or non-string key") + return {key: _provider_evidence_value(item, seen=seen | {id(value)}) for key, item in sorted(value.items())} + if isinstance(value, (list, tuple)): + if id(value) in seen: + raise ProviderRuntimeError("provider evidence contains a cyclic array") + return [_provider_evidence_value(item, seen=seen | {id(value)}) for item in value] + raise ProviderRuntimeError(f"provider evidence contains unsupported value type: {type(value).__name__}") + + +def provider_result_evidence(result: ProviderResult) -> Dict[str, Any]: + """Project a provider result into stable evidence without raw SDK objects.""" + if not isinstance(result, ProviderResult): + raise ProviderRuntimeError("provider returned an invalid ProviderResult") + messages: List[Dict[str, Any]] = [] + for message in result.messages: + if not isinstance(message, ProviderMessage): + raise ProviderRuntimeError("provider result contains an invalid message") + tool_calls: List[Dict[str, Any]] = [] + for call in message.tool_calls: + if not isinstance(call, ProviderToolCall): + raise ProviderRuntimeError("provider result contains an invalid tool call") + tool_calls.append({"id": call.id, "name": call.name, "arguments": call.arguments, "type": call.type}) + messages.append({ + "role": message.role, + "content": message.content, + "tool_calls": tool_calls, + "finish_reason": message.finish_reason, + "index": message.index, + "annotations": _provider_evidence_value(message.annotations), + }) + return { + "messages": messages, + "usage": _provider_evidence_value(result.usage), + "encrypted_reasoning": _provider_evidence_value(result.encrypted_reasoning), + "reasoning_summaries": _provider_evidence_value(result.reasoning_summaries), + "model": result.model, + "metadata": _provider_evidence_value(result.metadata), + } + + +def normalized_provider_usage(result: ProviderResult) -> Dict[str, Any]: + if not isinstance(result, ProviderResult): + raise ProviderRuntimeError("provider returned an invalid ProviderResult") + if result.usage is not None and not isinstance(result.usage, dict): + raise ProviderRuntimeError("provider usage must be an object when populated") + usage = result.usage or {} + + def count(*names: str) -> int: + values: List[int] = [] + for name in names: + if name not in usage: + continue + value = usage[name] + if type(value) is not int or value < 0: + raise ProviderRuntimeError(f"provider usage {name} must be a nonnegative integer") + values.append(value) + if len(set(values)) > 1: + raise ProviderRuntimeError(f"provider usage aliases disagree: {', '.join(names)}") + return values[0] if values else 0 + + input_tokens = count("input_tokens", "prompt_tokens") + output_tokens = count("output_tokens", "completion_tokens") + component_total = input_tokens + output_tokens + reported_total = count("total_tokens") + total_tokens = max(component_total, reported_total) + if not isinstance(result.metadata, dict): + raise ProviderRuntimeError("provider metadata must be an object") + metadata = result.metadata + costs: List[Union[int, float]] = [] + currencies: List[str] = [] + for source in (usage, metadata): + for name in ("cost_amount", "cost", "cost_usd"): + if name not in source: + continue + value = source[name] + if isinstance(value, bool) or not isinstance(value, (int, float)) or value < 0: + raise ProviderRuntimeError(f"provider {name} must be a finite nonnegative number") + try: + finite_cost = math.isfinite(float(value)) + except OverflowError as exc: + raise ProviderRuntimeError(f"provider {name} must be representable as a finite number") from exc + if not finite_cost: + raise ProviderRuntimeError(f"provider {name} must be a finite nonnegative number") + costs.append(value) + if name == "cost_usd": + currencies.append("USD") + if "cost_currency" in source: + currency_value = source["cost_currency"] + if not isinstance(currency_value, str) or len(currency_value) != 3 or any(character < "A" or character > "Z" for character in currency_value): + raise ProviderRuntimeError("provider cost_currency must be a three-letter uppercase code") + currencies.append(currency_value) + if len(set(currencies)) > 1: + raise ProviderRuntimeError("provider cost currencies disagree") + cost = max(costs, default=0.0) + currency = currencies[0] if currencies else "USD" + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": total_tokens, + "cost_amount": cost, + "cost_currency": currency, + } + @dataclass class ProviderRuntimeContext: """Context object passed to provider runtimes.""" @@ -847,7 +958,11 @@ def _convert_messages_to_chat(self, messages: List[Dict[str, Any]]) -> List[Dict # Some OpenAI-compatible routes reject null `content` values. # Normalize to empty string to preserve turn shape while staying valid. content = "" - converted.append({"role": role, "content": content}) + converted_message = {"role": role, "content": content} + for field in ("name", "tool_call_id", "tool_calls", "function_call"): + if field in message: + converted_message[field] = message[field] + converted.append(converted_message) return converted def _convert_tools_to_openai(self, tools: Optional[List[Dict[str, Any]]]) -> Optional[List[Dict[str, Any]]]: @@ -1700,7 +1815,7 @@ def _message_content_to_text(self, content: Any) -> Optional[str]: return None return "".join(parts) if parts else None - def _convert_messages(self, messages: List[Dict[str, Any]]) -> Tuple[Optional[str], List[Dict[str, Any]]]: + def _convert_messages(self, messages: List[Dict[str, Any]], tool_name_aliases: Optional[Dict[str, str]] = None) -> Tuple[Optional[str], List[Dict[str, Any]]]: system_prompt: Optional[str] = None converted: List[Dict[str, Any]] = [] @@ -1748,7 +1863,7 @@ def _convert_messages(self, messages: List[Dict[str, Any]]) -> Tuple[Optional[st { "type": "tool_use", "id": str(call_id), - "name": str(name), + "name": (tool_name_aliases or {}).get(str(name), str(name)), "input": input_payload if isinstance(input_payload, dict) else {}, } ) @@ -2106,14 +2221,14 @@ def _filter_anthropic_tools( self, tools: Optional[List[Dict[str, Any]]], context: ProviderRuntimeContext, - ) -> Optional[List[Dict[str, Any]]]: + ) -> Tuple[Optional[List[Dict[str, Any]]], Dict[str, str]]: """ Anthropic rejects tool names with dots or other invalid characters. Drop dotted todo* tools when todos are disabled, and strip any tool whose name fails the provider regex ^[a-zA-Z0-9_-]{1,128}$. """ if not tools: - return tools + return tools, {} agent_cfg = context.agent_config or {} features_cfg = agent_cfg.get("features") or {} @@ -2129,12 +2244,26 @@ def _filter_anthropic_tools( filtered: List[Dict[str, Any]] = [] dropped: List[str] = [] + aliases: Dict[str, str] = {} + used_names: set[str] = set() for tool in tools: - name = None - try: - name = tool.get("name") - except Exception: - name = None + if not isinstance(tool, Mapping): + continue + function = tool.get("function") if tool.get("type") == "function" else None + name = function.get("name") if isinstance(function, Mapping) else tool.get("name") if tool.get("type") != "function" else None + if isinstance(name, str): + used_names.add(name) + for tool in tools: + candidate = dict(tool) if isinstance(tool, Mapping) else tool + if isinstance(tool, Mapping) and tool.get("type") == "function" and isinstance(tool.get("function"), Mapping): + function = tool["function"] + candidate = { + "name": function.get("name"), + "input_schema": function.get("parameters", {"type": "object", "properties": {}}), + } + if isinstance(function.get("description"), str): + candidate["description"] = function["description"] + name = candidate.get("name") if isinstance(candidate, dict) else None if not name or not isinstance(name, str): continue @@ -2144,10 +2273,22 @@ def _filter_anthropic_tools( continue if not re.match(r"^[A-Za-z0-9_-]{1,128}$", name): - dropped.append(name) - continue - - filtered.append(tool) + if name != "host.execute": + dropped.append(name) + continue + alias = "host_execute" + counter = 0 + while alias in used_names: + suffix = hashlib.sha256(f"{name}:{counter}".encode("utf-8")).hexdigest()[:8] + alias = f"host_execute_{suffix}" + counter += 1 + candidate = dict(candidate) + candidate["name"] = alias + aliases[name] = alias + used_names.add(alias) + name = alias + + filtered.append(candidate) session_state = getattr(context, "session_state", None) if session_state is not None and dropped: @@ -2156,7 +2297,7 @@ def _filter_anthropic_tools( except Exception: pass - return filtered or None + return filtered or None, aliases def _build_system_prompt(self, system_prompt: str, prompt_cache_cfg: Dict[str, Any]) -> Any: apply_cache = bool(prompt_cache_cfg.get("apply_to_system", True)) @@ -2212,8 +2353,8 @@ def invoke( stream: bool, context: ProviderRuntimeContext, ) -> ProviderResult: - tools = self._filter_anthropic_tools(tools, context) - system_prompt, converted_messages = self._convert_messages(messages) + tools, tool_name_aliases = self._filter_anthropic_tools(tools, context) + system_prompt, converted_messages = self._convert_messages(messages, tool_name_aliases) anthropic_cfg = (context.agent_config.get("provider_tools") or {}).get("anthropic", {}) max_tokens = anthropic_cfg.get("max_output_tokens", 1024) @@ -2249,6 +2390,9 @@ def invoke( request["tools"] = tools resolved_tool_choice = self._resolve_tool_choice(anthropic_cfg.get("tool_choice"), tools) + if isinstance(resolved_tool_choice, dict) and isinstance(resolved_tool_choice.get("name"), str): + resolved_tool_choice = dict(resolved_tool_choice) + resolved_tool_choice["name"] = tool_name_aliases.get(resolved_tool_choice["name"], resolved_tool_choice["name"]) if resolved_tool_choice is not None: request["tool_choice"] = resolved_tool_choice @@ -2321,7 +2465,13 @@ def _respect_delay() -> None: context=context, metadata=metadata, ) - return self._normalize_response(response, usage_override=usage_override) + result = self._normalize_response(response, usage_override=usage_override) + reverse_aliases = {alias: name for name, alias in tool_name_aliases.items()} + for message in result.messages: + for call in message.tool_calls: + if call.name in reverse_aliases: + call.name = reverse_aliases[call.name] + return result except Exception as exc: is_rate_limit = AnthropicRateLimitError is not None and isinstance(exc, AnthropicRateLimitError) is_overloaded = False if is_rate_limit else self._is_overloaded_error(exc) diff --git a/agentic_coder_prototype/state/session_state.py b/agentic_coder_prototype/state/session_state.py index 9c780a2a..ef690bb2 100644 --- a/agentic_coder_prototype/state/session_state.py +++ b/agentic_coder_prototype/state/session_state.py @@ -838,6 +838,19 @@ def set_provider_metadata(self, key: str, value: Any) -> None: def get_provider_metadata(self, key: str, default: Any = None) -> Any: return self.provider_metadata.get(key, default) + def provider_metadata_snapshot(self) -> Dict[str, Any]: + replay_safe_keys = ( + "anthropic_rate_limits", + "conversation_id", + "current_turn_index", + "previous_response_id", + ) + return { + key: self.provider_metadata[key] + for key in replay_safe_keys + if key in self.provider_metadata + } + def clear_provider_metadata(self) -> None: self.provider_metadata.clear() diff --git a/breadboard/product/evidence/__init__.py b/breadboard/product/evidence/__init__.py index 86f2bcaa..a4599324 100644 --- a/breadboard/product/evidence/__init__.py +++ b/breadboard/product/evidence/__init__.py @@ -1,5 +1,8 @@ from .lane_lock import LaneLockError, LaneResolutionError, build_lane_lock, lock_lane, validate_before_capture from .lanes import LaneValidationError, MutableReferenceError, author_lane, init_lane, load_lane, validate_lane +from .replay_execution import ReplayArtifactManifest, ReplayExecution, ReplayExecutionError +from .replay_plan import ReplayPlan, ReplayPlanError, build_replay_plan +from .replay_runner import ReplayRunError, ReplayRunResult, ReplayScenario, run_replay from .stage_reports import StageReport, StageStateError from .workspace import BreadBoardWorkspace, WorkspacePathError __all__ = [ @@ -8,11 +11,21 @@ "LaneResolutionError", "LaneValidationError", "MutableReferenceError", + "ReplayArtifactManifest", + "ReplayExecution", + "ReplayExecutionError", + "ReplayPlan", + "ReplayPlanError", + "ReplayRunError", + "ReplayRunResult", + "ReplayScenario", "WorkspacePathError", "StageReport", "StageStateError", "author_lane", "build_lane_lock", + "build_replay_plan", + "run_replay", "load_lane", "validate_lane", "init_lane", diff --git a/breadboard/product/evidence/replay_execution.py b/breadboard/product/evidence/replay_execution.py new file mode 100644 index 00000000..17ae4042 --- /dev/null +++ b/breadboard/product/evidence/replay_execution.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import hashlib +import json +import math +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime +from pathlib import PurePosixPath +from typing import Any + +from .replay_plan import ReplayPlan, canonical_json + +EXECUTION_SCHEMA_VERSION = "bb.replay_execution.v1" +MANIFEST_SCHEMA_VERSION = "bb.replay_artifact_manifest.v1" +_TERMINAL_STATUSES = frozenset(("completed", "provider_failed", "tool_failed", "policy_denied", "budget_exhausted", "timed_out", "cancelled", "host_failed", "invalidated")) +_EXECUTION_FIELDS = { + "schema_version", "execution_id", "fresh_nonce", "mode", "plan_sha256", "lane_lock_sha256", "harness_lock_sha256", + "started_at_utc", "completed_at_utc", "duration_ms", "terminal_status", "completion_reason", "kernel_event_stream", + "provider_exchanges", "tool_outcomes", "workspace_before", "workspace_after", "workspace_diff", "artifact_manifest_id", + "policy_decisions", "cleanup_ledger", "nondeterminism_disclosures", "redaction_report", "schema_validation_passed", + "integrity_verified", "claimable", "reuse_attestation_id", "normalization_evidence_ids", "comparison_report_id", "problem", +} +_MANIFEST_FIELDS = {"schema_version", "manifest_id", "source_record_id", "publish_status", "created_at_utc", "entries", "integrity_verified"} +_ARTIFACT_ENTRY_FIELDS = {"artifact_id", "role", "location_kind", "location", "media_type", "schema_id", "size_bytes", "sha256", "producer", "sensitivity", "created_at_utc"} +_PROBLEM_FIELDS = {"schema_version", "error_code", "message", "record_refs", "failed_stage", "hint", "next_actions"} +_PROVIDER_EXCHANGE_FIELDS = { + "schema_version", "exchange_id", "execution_id", "attempt_id", "request_id", "route_lock_sha256", + "provider_family", "runtime_id", "runtime_version", "endpoint_class", "model_id", "model_revision", + "started_at_utc", "completed_at_utc", "duration_ms", "status", "request_payload_sha256", + "response_payload_sha256", "finish_reason", "usage", "evidence_refs", "fallback_used", "problem", +} +_REMOTE_OBJECT_REF = re.compile(r"(?![Ff][Ii][Ll][Ee]://)[A-Za-z][A-Za-z0-9+.-]*://[^/\s\\]+/[^\s\\]+") + + +class ReplayExecutionError(ValueError): + pass + + +def _portable_id(value: Any, name: str) -> str: + if not isinstance(value, str) or not value or not value[0].isalnum() or any(char not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._:-" for char in value): + raise ReplayExecutionError(f"{name} must be a portable identifier") + return value + + +def _sha256(value: Any, name: str) -> str: + if not isinstance(value, str) or len(value) != 71 or not value.startswith("sha256:") or any(char not in "0123456789abcdef" for char in value[7:]): + raise ReplayExecutionError(f"{name} must be an exact lowercase sha256 hash") + return value + + +def _timestamp(value: Any, name: str) -> str: + if not isinstance(value, str) or "T" not in value: + raise ReplayExecutionError(f"{name} must be an RFC 3339 date-time") + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise ReplayExecutionError(f"{name} must be an RFC 3339 date-time") from exc + if parsed.tzinfo is None: + raise ReplayExecutionError(f"{name} must include an offset") + return value + + +def _ref(value: Any, name: str) -> dict[str, str]: + if not isinstance(value, Mapping) or set(value) != {"ref", "sha256"} or not isinstance(value.get("ref"), str) or not value["ref"]: + raise ReplayExecutionError(f"{name} must be an artifact reference") + return {"ref": value["ref"], "sha256": _sha256(value.get("sha256"), f"{name}.sha256")} + + +def _problem(value: Any) -> dict[str, Any]: + if not isinstance(value, Mapping) or set(value) != _PROBLEM_FIELDS or value.get("schema_version") != "bb.problem.v1": + raise ReplayExecutionError("problem must match bb.problem.v1") + if not isinstance(value.get("error_code"), str) or not value["error_code"] or not isinstance(value.get("message"), str) or not value["message"]: + raise ReplayExecutionError("problem identity and message must be populated") + if any(not isinstance(value.get(name), list) or any(not isinstance(item, str) or not item for item in value[name]) for name in ("record_refs", "next_actions")): + raise ReplayExecutionError("problem record_refs and next_actions must be string arrays") + if any(value.get(name) is not None and (not isinstance(value[name], str) or not value[name]) for name in ("failed_stage", "hint")): + raise ReplayExecutionError("problem optional fields must be null or populated strings") + return dict(value) + + +def _unique_strings(value: Any, name: str, *, populated: bool = False) -> list[str]: + if not isinstance(value, list) or any(not isinstance(item, str) or not item for item in value) or len(value) != len(set(value)) or populated and not value: + raise ReplayExecutionError(f"{name} must be a unique string array") + return list(value) + + +def _validate_usage(value: Any) -> dict[str, Any]: + names = {"input_tokens", "output_tokens", "total_tokens", "cost_amount", "cost_currency"} + if not isinstance(value, Mapping) or set(value) != names: + raise ReplayExecutionError("normalized usage fields are invalid") + if any(type(value[name]) is not int or value[name] < 0 for name in ("input_tokens", "output_tokens", "total_tokens")) or isinstance(value["cost_amount"], bool) or not isinstance(value["cost_amount"], (int, float)) or value["cost_amount"] < 0: + raise ReplayExecutionError("normalized usage values must be finite and nonnegative") + try: + finite_cost = math.isfinite(float(value["cost_amount"])) + except OverflowError as exc: + raise ReplayExecutionError("normalized usage cost must be representable as a finite number") from exc + if not finite_cost: + raise ReplayExecutionError("normalized usage values must be finite and nonnegative") + if value["total_tokens"] < value["input_tokens"] + value["output_tokens"]: + raise ReplayExecutionError("total_tokens cannot understate component token counts") + if not isinstance(value["cost_currency"], str) or len(value["cost_currency"]) != 3 or any(character < "A" or character > "Z" for character in value["cost_currency"]): + raise ReplayExecutionError("cost_currency must be an uppercase ISO-like code") + return dict(value) + +def validate_provider_exchange(value: Mapping[str, Any]) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ReplayExecutionError("provider exchange must be an object") + try: + record = json.loads(canonical_json(value)) + except (TypeError, ValueError) as exc: + raise ReplayExecutionError(str(exc)) from exc + if set(record) != _PROVIDER_EXCHANGE_FIELDS or record.get("schema_version") != "bb.provider_exchange.v2": + raise ReplayExecutionError("record fields do not match bb.provider_exchange.v2") + for name in ("exchange_id", "execution_id", "attempt_id", "request_id"): + _portable_id(record.get(name), name) + for name in ("route_lock_sha256", "request_payload_sha256"): + _sha256(record.get(name), name) + for name in ("provider_family", "runtime_id", "runtime_version", "endpoint_class", "model_id"): + if not isinstance(record.get(name), str) or not record[name]: + raise ReplayExecutionError(f"{name} must be populated") + if record.get("model_revision") is not None and (not isinstance(record["model_revision"], str) or not record["model_revision"]): + raise ReplayExecutionError("model_revision must be null or populated") + _timestamp(record.get("started_at_utc"), "started_at_utc"); _timestamp(record.get("completed_at_utc"), "completed_at_utc") + if isinstance(record.get("duration_ms"), bool) or not isinstance(record.get("duration_ms"), (int, float)) or not math.isfinite(record["duration_ms"]) or record["duration_ms"] < 0: + raise ReplayExecutionError("duration_ms must be finite and nonnegative") + status = record.get("status") + if status not in ("completed", "provider_error", "invalid_response", "timed_out", "cancelled"): + raise ReplayExecutionError("provider exchange status is invalid") + usage = _validate_usage(record.get("usage")) + evidence_refs = _unique_strings(record.get("evidence_refs"), "evidence_refs", populated=True) + if record.get("fallback_used") is not False: + raise ReplayExecutionError("fallback_used must be false") + if status == "completed": + _sha256(record.get("response_payload_sha256"), "response_payload_sha256") + if not isinstance(record.get("finish_reason"), str) or not record["finish_reason"] or record.get("problem") is not None: + raise ReplayExecutionError("completed provider exchange terminal fields are invalid") + else: + if record.get("response_payload_sha256") is not None or record.get("finish_reason") is not None: + raise ReplayExecutionError("failed provider exchange cannot expose completed response fields") + _problem(record.get("problem")) + record["usage"] = usage + record["evidence_refs"] = evidence_refs + return record + + +def _validate_execution(value: Mapping[str, Any]) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ReplayExecutionError("replay execution must be an object") + try: + record = json.loads(canonical_json(value)) + except (TypeError, ValueError) as exc: + raise ReplayExecutionError(str(exc)) from exc + if set(record) != _EXECUTION_FIELDS or record.get("schema_version") != EXECUTION_SCHEMA_VERSION: + raise ReplayExecutionError("record fields do not match bb.replay_execution.v1") + _portable_id(record.get("execution_id"), "execution_id") + if record.get("mode") not in ("execute", "reuse"): + raise ReplayExecutionError("mode must be execute or reuse") + for name in ("plan_sha256", "lane_lock_sha256", "harness_lock_sha256"): + _sha256(record.get(name), name) + _timestamp(record.get("started_at_utc"), "started_at_utc"); _timestamp(record.get("completed_at_utc"), "completed_at_utc") + if type(record.get("duration_ms")) is not int or record["duration_ms"] < 0 or record.get("terminal_status") not in _TERMINAL_STATUSES or not isinstance(record.get("completion_reason"), str) or not record["completion_reason"]: + raise ReplayExecutionError("execution terminal fields are invalid") + for name in ("kernel_event_stream", "workspace_before", "workspace_after", "workspace_diff", "redaction_report"): + _ref(record.get(name), name) + if not isinstance(record.get("provider_exchanges"), list): + raise ReplayExecutionError("provider_exchanges must be an array") + for index, row in enumerate(record["provider_exchanges"]): + names = {"ref", "sha256", "request_id", "attempt_id", "normalized_usage", "status"} + if not isinstance(row, Mapping) or set(row) != names or row.get("status") not in ("completed", "provider_error", "invalid_response", "timed_out", "cancelled"): + raise ReplayExecutionError(f"provider_exchanges[{index}] is invalid") + _ref({"ref": row.get("ref"), "sha256": row.get("sha256")}, f"provider_exchanges[{index}]") + _portable_id(row.get("request_id"), "request_id"); _portable_id(row.get("attempt_id"), "attempt_id"); _validate_usage(row.get("normalized_usage")) + if not isinstance(record.get("tool_outcomes"), list): + raise ReplayExecutionError("tool_outcomes must be an array") + for index, row in enumerate(record["tool_outcomes"]): + _ref(row, f"tool_outcomes[{index}]") + _portable_id(record.get("artifact_manifest_id"), "artifact_manifest_id") + decisions = record.get("policy_decisions") + if not isinstance(decisions, list): + raise ReplayExecutionError("policy_decisions must be an array") + seen_kinds: set[str] = set() + allowed_by_kind = {"policy": {"allowed", "denied"}, "capability": {"allowed", "denied"}, "approval": {"approved", "rejected", "not_required"}} + for index, row in enumerate(decisions): + names = {"kind", "decision", "ref", "sha256"} + if not isinstance(row, Mapping) or set(row) != names or row.get("kind") not in allowed_by_kind or row.get("decision") not in allowed_by_kind[row["kind"]]: + raise ReplayExecutionError(f"policy_decisions[{index}] is invalid") + _ref({"ref": row.get("ref"), "sha256": row.get("sha256")}, f"policy_decisions[{index}]") + if row["kind"] in seen_kinds: + raise ReplayExecutionError("policy_decisions must contain each decision kind exactly once") + seen_kinds.add(row["kind"]) + if seen_kinds != set(allowed_by_kind): + raise ReplayExecutionError("policy_decisions must contain each decision kind exactly once") + _unique_strings(record.get("cleanup_ledger"), "cleanup_ledger"); _unique_strings(record.get("nondeterminism_disclosures"), "nondeterminism_disclosures") + _unique_strings(record.get("normalization_evidence_ids"), "normalization_evidence_ids") + for name in ("fresh_nonce", "reuse_attestation_id", "comparison_report_id"): + if record.get(name) is not None and (not isinstance(record[name], str) or not record[name]): + raise ReplayExecutionError(f"{name} must be null or a populated string") + if any(type(record.get(name)) is not bool for name in ("schema_validation_passed", "integrity_verified", "claimable")): + raise ReplayExecutionError("execution verification flags must be booleans") + status, mode = record["terminal_status"], record["mode"] + if mode == "execute" and (not isinstance(record.get("fresh_nonce"), str) or not record["fresh_nonce"] or record.get("reuse_attestation_id") is not None): + raise ReplayExecutionError("execute records require a fresh_nonce and prohibit reuse_attestation_id") + if mode == "reuse" and (record.get("fresh_nonce") is not None or not isinstance(record.get("reuse_attestation_id"), str) or not record["reuse_attestation_id"] or record["claimable"] or record.get("comparison_report_id") is not None or record["normalization_evidence_ids"]): + raise ReplayExecutionError("reuse records are non-claimable and require an attestation") + if status == "completed": + if record.get("problem") is not None: + raise ReplayExecutionError("completed execution must not contain a problem") + if mode == "execute" and (not record["provider_exchanges"] or not any(row["status"] == "completed" for row in record["provider_exchanges"])): + raise ReplayExecutionError("completed execute records require a completed provider exchange") + else: + _problem(record.get("problem")) + if record["claimable"] or record.get("comparison_report_id") is not None or record["normalization_evidence_ids"]: + raise ReplayExecutionError("non-completed execution cannot be compared or claimed") + if record["claimable"] or record["normalization_evidence_ids"] or record.get("comparison_report_id") is not None: + raise ReplayExecutionError("candidate replay executions cannot assert comparison evidence or claimability") + return record + + +def _validate_manifest(value: Mapping[str, Any]) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ReplayExecutionError("replay artifact manifest must be an object") + record = json.loads(canonical_json(value)) + if set(record) != _MANIFEST_FIELDS or record.get("schema_version") != MANIFEST_SCHEMA_VERSION or record.get("publish_status") not in ("complete", "quarantined") or type(record.get("integrity_verified")) is not bool: + raise ReplayExecutionError("record fields do not match bb.replay_artifact_manifest.v1") + _portable_id(record.get("manifest_id"), "manifest_id"); _portable_id(record.get("source_record_id"), "source_record_id"); _timestamp(record.get("created_at_utc"), "created_at_utc") + if record["publish_status"] == "complete" and not record["integrity_verified"] or record["publish_status"] == "quarantined" and record["integrity_verified"]: + raise ReplayExecutionError("manifest publish status conflicts with integrity_verified") + if not isinstance(record.get("entries"), list) or not record["entries"]: + raise ReplayExecutionError("artifact manifest entries must be populated") + ids: set[str] = set() + for index, row in enumerate(record["entries"]): + if not isinstance(row, Mapping) or set(row) != _ARTIFACT_ENTRY_FIELDS: + raise ReplayExecutionError(f"entries[{index}] fields are invalid") + artifact_id = _portable_id(row.get("artifact_id"), "artifact_id") + if artifact_id in ids: + raise ReplayExecutionError("artifact manifest contains duplicate artifact_id") + ids.add(artifact_id); _sha256(row.get("sha256"), "sha256"); _timestamp(row.get("created_at_utc"), "created_at_utc") + if row.get("location_kind") not in ("workspace_relative_path", "object_ref") or not isinstance(row.get("location"), str) or not row["location"] or not isinstance(row.get("media_type"), str) or not row["media_type"] or row.get("sensitivity") not in ("public", "internal", "secret_redacted") or type(row.get("size_bytes")) is not int or row["size_bytes"] < 0: + raise ReplayExecutionError(f"entries[{index}] metadata is invalid") + if not isinstance(row.get("producer"), str) or not row["producer"] or not isinstance(row.get("role"), str) or not row["role"] or row.get("schema_id") is not None and (not isinstance(row["schema_id"], str) or not row["schema_id"]): + raise ReplayExecutionError(f"entries[{index}] ownership is invalid") + if row["location_kind"] == "object_ref": + location = row["location"] + if location != row["sha256"] and _REMOTE_OBJECT_REF.fullmatch(location) is None: + raise ReplayExecutionError("object_ref locations must be a content digest or remote object URI") + if row["location_kind"] == "workspace_relative_path": + location = row["location"] + portable = PurePosixPath(location) + if location == "." or portable.is_absolute() or portable.as_posix() != location or "\\" in location or (len(location) >= 2 and location[0] in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" and location[1] == ":") or any(part in ("", ".", "..") for part in portable.parts): + raise ReplayExecutionError("workspace_relative_path locations must be canonical and stay within the workspace") + unsigned = dict(record) + actual_manifest_id = unsigned.pop("manifest_id") + expected_manifest_id = "replay_manifest:" + hashlib.sha256(canonical_json(unsigned)).hexdigest() + if actual_manifest_id != expected_manifest_id: + raise ReplayExecutionError("manifest_id does not match the canonical manifest content") + return record + + +@dataclass(frozen=True, slots=True) +class ReplayExecution: + _canonical: bytes + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ReplayExecution": + return cls(canonical_json(_validate_execution(value))) + + def as_dict(self) -> dict[str, Any]: + return json.loads(self._canonical) + + @property + def sha256(self) -> str: + return "sha256:" + hashlib.sha256(self._canonical).hexdigest() + + @property + def terminal_status(self) -> str: + return self.as_dict()["terminal_status"] + + @property + def claimable(self) -> bool: + return self.as_dict()["claimable"] + + def verify_plan(self, plan: ReplayPlan) -> None: + record, plan_record = self.as_dict(), plan.as_dict() + if record["plan_sha256"] != plan.sha256: + raise ReplayExecutionError("execution was produced from a different replay plan") + if record["mode"] != plan_record["mode"]: + raise ReplayExecutionError("execution mode does not match its replay plan") + if record["lane_lock_sha256"] != plan_record["lane_lock_sha256"] or record["harness_lock_sha256"] != plan_record["harness_lock_sha256"]: + raise ReplayExecutionError("execution lock hashes do not match its replay plan") + + def require_comparable(self) -> None: + record = self.as_dict() + if record["mode"] != "execute" or record["terminal_status"] != "completed" or not record["schema_validation_passed"] or not record["integrity_verified"]: + raise ReplayExecutionError("only completed, schema-valid, integrity-verified execute-mode replays may be compared") + + +@dataclass(frozen=True, slots=True) +class ReplayArtifactManifest: + _canonical: bytes + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ReplayArtifactManifest": + return cls(canonical_json(_validate_manifest(value))) + + def as_dict(self) -> dict[str, Any]: + return json.loads(self._canonical) + + @property + def sha256(self) -> str: + return "sha256:" + hashlib.sha256(self._canonical).hexdigest() + + +def problem(error_code: str, message: str, *, failed_stage: str = "replay", record_refs: Sequence[str] = ()) -> dict[str, Any]: + value = { + "schema_version": "bb.problem.v1", + "error_code": error_code, + "message": message, + "record_refs": list(record_refs), + "failed_stage": failed_stage, + "hint": None, + "next_actions": [], + } + return _problem(value) diff --git a/breadboard/product/evidence/replay_plan.py b/breadboard/product/evidence/replay_plan.py new file mode 100644 index 00000000..af4ba7cb --- /dev/null +++ b/breadboard/product/evidence/replay_plan.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import hashlib +import json +import math +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any + +PLAN_SCHEMA_VERSION = "bb.replay_plan.v1" +HASH_BINDING_NAMES = ( + "scenario_sha256", + "interaction_script_sha256", + "initial_workspace_sha256", + "provider_route_lock_sha256", + "capability_probe_sha256", + "model_policy_sha256", + "initial_messages_sha256", + "tool_schema_lock_sha256", + "operation_policy_sha256", + "tool_executor_identity_sha256", + "host_driver_identity_sha256", + "host_platform_sha256", + "environment_allowlist_sha256", + "secret_references_sha256", + "normalizer_config_sha256", + "comparator_config_sha256", + "schema_registry_sha256", +) +DEADLINE_NAMES = ("total", "idle", "provider_call", "tool_call") +BUDGET_NAMES = ("turns", "provider_calls", "tool_calls", "tokens", "cost") +_PLAN_FIELDS = { + "schema_version", "plan_id", "lane_lock_sha256", "harness_lock_sha256", + "hash_bindings", "deadlines_ms", "budgets", "cancellation_grace_ms", "mode", + "scenario_ref", "provider_route_ref", "operation_policy_ref", "host_ref", + "toolset_lock_ref", "reuse_attestation_ref", +} + + +class ReplayPlanError(ValueError): + pass + + +def _json_value(value: Any, *, pointer: str = "$", seen: frozenset[int] = frozenset()) -> Any: + if value is None or type(value) in (bool, str, int): + return value + if type(value) is float: + if not math.isfinite(value): + raise ReplayPlanError(f"{pointer}: non-finite number") + return value + if isinstance(value, Mapping): + if id(value) in seen or any(type(key) is not str for key in value): + raise ReplayPlanError(f"{pointer}: cyclic object or non-string key") + return {key: _json_value(item, pointer=f"{pointer}.{key}", seen=seen | {id(value)}) for key, item in value.items()} + if isinstance(value, (list, tuple)): + if id(value) in seen: + raise ReplayPlanError(f"{pointer}: cyclic array") + return [_json_value(item, pointer=f"{pointer}[{index}]", seen=seen | {id(value)}) for index, item in enumerate(value)] + raise ReplayPlanError(f"{pointer}: value is not JSON-compatible") + + +def canonical_json(value: Any) -> bytes: + return (json.dumps(_json_value(value), allow_nan=False, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") + + +def sha256_json(value: Any) -> str: + return "sha256:" + hashlib.sha256(canonical_json(value)).hexdigest() + + +def _sha256(value: Any, name: str) -> str: + if not isinstance(value, str) or len(value) != 71 or not value.startswith("sha256:") or any(char not in "0123456789abcdef" for char in value[7:]): + raise ReplayPlanError(f"{name} must be an exact lowercase sha256 hash") + return value + + +def _portable_id(value: Any, name: str) -> str: + if not isinstance(value, str) or not value or not value[0].isalnum() or any(char not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._:-" for char in value): + raise ReplayPlanError(f"{name} must be a portable identifier") + return value + + +def _plan_id(unsigned: Mapping[str, Any]) -> str: + return "replay_plan:" + hashlib.sha256(canonical_json(unsigned)).hexdigest() + + +def _validate_record(value: Mapping[str, Any]) -> dict[str, Any]: + record = _json_value(value) + if not isinstance(record, dict) or set(record) != _PLAN_FIELDS or record.get("schema_version") != PLAN_SCHEMA_VERSION: + raise ReplayPlanError("record fields do not match bb.replay_plan.v1") + _portable_id(record.get("plan_id"), "plan_id") + _sha256(record.get("lane_lock_sha256"), "lane_lock_sha256") + _sha256(record.get("harness_lock_sha256"), "harness_lock_sha256") + bindings = record.get("hash_bindings") + if not isinstance(bindings, dict) or set(bindings) != set(HASH_BINDING_NAMES): + raise ReplayPlanError("hash_bindings must contain the frozen replay binding set") + for name in HASH_BINDING_NAMES: + _sha256(bindings[name], f"hash_bindings.{name}") + deadlines = record.get("deadlines_ms") + if not isinstance(deadlines, dict) or set(deadlines) != set(DEADLINE_NAMES) or any(type(deadlines[name]) is not int or deadlines[name] < 1 for name in DEADLINE_NAMES): + raise ReplayPlanError("deadlines_ms must contain positive integer total, idle, provider_call, and tool_call values") + budgets = record.get("budgets") + if not isinstance(budgets, dict) or set(budgets) != set(BUDGET_NAMES): + raise ReplayPlanError("budgets must contain the frozen replay budget set") + if any(type(budgets[name]) is not int or budgets[name] < 1 for name in BUDGET_NAMES[:-1]) or isinstance(budgets["cost"], bool) or not isinstance(budgets["cost"], (int, float)) or budgets["cost"] <= 0: + raise ReplayPlanError("replay budgets must be positive") + if type(record.get("cancellation_grace_ms")) is not int or record["cancellation_grace_ms"] < 1: + raise ReplayPlanError("cancellation_grace_ms must be a positive integer") + mode = record.get("mode") + if mode not in ("execute", "reuse"): + raise ReplayPlanError("mode must be execute or reuse") + execute_refs = ("scenario_ref", "provider_route_ref", "operation_policy_ref", "host_ref", "toolset_lock_ref") + if mode == "execute": + if any(not isinstance(record[name], str) or not record[name] for name in execute_refs) or record.get("reuse_attestation_ref") is not None: + raise ReplayPlanError("execute plans require runtime references and prohibit reuse_attestation_ref") + elif any(record.get(name) is not None for name in execute_refs) or not isinstance(record.get("reuse_attestation_ref"), str) or not record["reuse_attestation_ref"]: + raise ReplayPlanError("reuse plans require only reuse_attestation_ref") + unsigned = dict(record) + actual_id = unsigned.pop("plan_id") + if actual_id != _plan_id(unsigned): + raise ReplayPlanError("plan_id does not match the frozen plan content") + return record + + +@dataclass(frozen=True, slots=True) +class ReplayPlan: + _canonical: bytes + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ReplayPlan": + return cls(canonical_json(_validate_record(value))) + + def as_dict(self) -> dict[str, Any]: + return json.loads(self._canonical) + + @property + def sha256(self) -> str: + return "sha256:" + hashlib.sha256(self._canonical).hexdigest() + + @property + def plan_id(self) -> str: + return self.as_dict()["plan_id"] + + @property + def mode(self) -> str: + return self.as_dict()["mode"] + + def verify_bindings(self, binding_inputs: Mapping[str, Any]) -> None: + if set(binding_inputs) != set(HASH_BINDING_NAMES): + raise ReplayPlanError("binding inputs must contain the frozen replay binding set") + expected = {name: sha256_json(binding_inputs[name]) for name in HASH_BINDING_NAMES} + if expected != self.as_dict()["hash_bindings"]: + raise ReplayPlanError("replay inputs changed after plan creation") + + +def build_replay_plan( + *, + lane_lock_sha256: str, + harness_lock_sha256: str, + binding_inputs: Mapping[str, Any], + deadlines_ms: Mapping[str, int], + budgets: Mapping[str, int | float], + cancellation_grace_ms: int, + mode: str = "execute", + scenario_ref: str | None = None, + provider_route_ref: str | None = None, + operation_policy_ref: str | None = None, + host_ref: str | None = None, + toolset_lock_ref: str | None = None, + reuse_attestation_ref: str | None = None, +) -> ReplayPlan: + if set(binding_inputs) != set(HASH_BINDING_NAMES): + raise ReplayPlanError("binding inputs must contain the frozen replay binding set") + unsigned: dict[str, Any] = { + "schema_version": PLAN_SCHEMA_VERSION, + "lane_lock_sha256": lane_lock_sha256, + "harness_lock_sha256": harness_lock_sha256, + "hash_bindings": {name: sha256_json(binding_inputs[name]) for name in HASH_BINDING_NAMES}, + "deadlines_ms": dict(deadlines_ms), + "budgets": dict(budgets), + "cancellation_grace_ms": cancellation_grace_ms, + "mode": mode, + "scenario_ref": scenario_ref, + "provider_route_ref": provider_route_ref, + "operation_policy_ref": operation_policy_ref, + "host_ref": host_ref, + "toolset_lock_ref": toolset_lock_ref, + "reuse_attestation_ref": reuse_attestation_ref, + } + return ReplayPlan.from_dict({"plan_id": _plan_id(unsigned), **unsigned}) diff --git a/breadboard/product/evidence/replay_runner.py b/breadboard/product/evidence/replay_runner.py new file mode 100644 index 00000000..cbf0e66f --- /dev/null +++ b/breadboard/product/evidence/replay_runner.py @@ -0,0 +1,2955 @@ +from __future__ import annotations + +import datetime +import enum +import encodings.idna +import hashlib +import functools +import importlib +import importlib.machinery +import importlib.util +import inspect +import json +import os +import platform +import re +import signal +import socket +import stat +import subprocess +import tempfile +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +import sys +import uuid +from pathlib import Path +from typing import Any +from jsonschema import Draft202012Validator, SchemaError, ValidationError +from referencing import Registry, Resource + +from agentic_coder_prototype.logging.provider_dump import provider_dump_logger +from agentic_coder_prototype.provider.runtime import ( + ProviderMessage, + ProviderRuntimeContext, + ProviderResult, + ProviderToolCall, + normalized_provider_usage, + provider_result_evidence, +) +from breadboard.product.harness.lock import EffectiveHarnessLock +from breadboard.product.runtime.artifacts import AnchoredStorage, ArtifactRef, ArtifactStore +from breadboard.product.runtime.events import Clock, IdSource, JsonlEventSink, Session, SystemClock, UUIDSource +from breadboard.product.integrations.provider import ProviderRuntimeAdapter + +from .replay_execution import ReplayArtifactManifest, ReplayExecution, _portable_id, problem, validate_provider_exchange +from .replay_plan import HASH_BINDING_NAMES, ReplayPlan, ReplayPlanError, canonical_json, sha256_json +from .workspace import BreadBoardWorkspace + + +class ReplayRunError(RuntimeError): + pass + + +@dataclass(frozen=True, slots=True) +class ReplayScenario: + task: str + initial_messages: tuple[Mapping[str, Any], ...] + initial_files: Mapping[str, bytes | str] + tool_schemas: tuple[Mapping[str, Any], ...] + interaction_script: tuple[Mapping[str, Any], ...] = () + + def __post_init__(self) -> None: + if not isinstance(self.task, str) or not self.task.strip(): + raise ReplayRunError("scenario task must be populated") + if any(not isinstance(row, Mapping) for row in (*self.initial_messages, *self.interaction_script)): + raise ReplayRunError("scenario messages and interaction script must contain objects") + _initial_file_rows(self.initial_files) + canonical_json(list(self.initial_messages)); canonical_json(list(self.tool_schemas)); canonical_json(list(self.interaction_script)) + _declared_tool_names(self.tool_schemas) + _tool_argument_validators(self.tool_schemas) + + def binding_inputs(self, frozen_inputs: Mapping[str, Any]) -> dict[str, Any]: + result = dict(frozen_inputs) + result.update({ + "scenario_sha256": {"task": self.task}, + "interaction_script_sha256": list(self.interaction_script), + "initial_workspace_sha256": _initial_file_rows(self.initial_files), + "initial_messages_sha256": list(self.initial_messages), + "tool_schema_lock_sha256": list(self.tool_schemas), + }) + if set(result) != set(HASH_BINDING_NAMES): + raise ReplayRunError("scenario bindings must complete the frozen replay binding set") + return result + + +@dataclass(frozen=True, slots=True) +class ReplayRunResult: + execution: ReplayExecution + execution_path: Path + manifest: ReplayArtifactManifest + + +class _RuntimeFailure(Exception): + def __init__(self, status: str, error_code: str, detail: str) -> None: + super().__init__(detail); self.status, self.error_code, self.detail = status, error_code, detail + +class _InvalidProviderResponse(ReplayRunError): + pass +class _RemoteCallError(RuntimeError): + pass +class _InvalidWorkerValue: + pass + + +def _worker_envelope(status: str, value: Any = None, *, kind: str | None = None) -> bytes: + return canonical_json({"status": status, "kind": kind, "value": value}) + + +def _decode_canonical_json(payload: bytes) -> Any: + def unique_object(rows: list[tuple[str, Any]]) -> dict[str, Any]: + result = {} + for name, value in rows: + if name in result: + raise ValueError("duplicate JSON key") + result[name] = value + return result + try: + record = json.loads(payload.decode("utf-8"), parse_constant=lambda value: (_ for _ in ()).throw(ValueError(value)), object_pairs_hook=unique_object) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: + raise _RemoteCallError("isolated replay IPC contained invalid JSON") from exc + if canonical_json(record) != payload: + raise _RemoteCallError("isolated replay IPC contained non-canonical JSON") + return record + + +def _decode_worker_envelope(payload: bytes) -> tuple[str, str | None, Any]: + record = _decode_canonical_json(payload) + if not isinstance(record, dict) or set(record) != {"status", "kind", "value"}: + raise _RemoteCallError("isolated replay call returned a malformed envelope") + status, kind = record["status"], record["kind"] + if status not in {"ready", "ok", "error", "invalid"} or (kind is not None and kind not in {"provider", "policy", "host", "tool"}): + raise _RemoteCallError("isolated replay call returned an invalid envelope status") + return status, kind, record["value"] + + +def _provider_result_from_evidence(value: Any) -> ProviderResult: + if not isinstance(value, dict) or set(value) != {"messages", "usage", "encrypted_reasoning", "reasoning_summaries", "model", "metadata"} or not isinstance(value["messages"], list): + raise _RemoteCallError("isolated provider returned malformed normalized evidence") + messages = [] + for row in value["messages"]: + if not isinstance(row, dict) or set(row) != {"role", "content", "tool_calls", "finish_reason", "index", "annotations"} or not isinstance(row["tool_calls"], list): + raise _RemoteCallError("isolated provider returned a malformed normalized message") + calls = [] + for call in row["tool_calls"]: + if not isinstance(call, dict) or set(call) != {"id", "name", "arguments", "type"}: + raise _RemoteCallError("isolated provider returned a malformed normalized tool call") + calls.append(ProviderToolCall(call["id"], call["name"], call["arguments"], call["type"])) + messages.append(ProviderMessage(row["role"], row["content"], calls, row["finish_reason"], row["index"], annotations=row["annotations"])) + return ProviderResult(messages, None, value["usage"], encrypted_reasoning=value["encrypted_reasoning"], reasoning_summaries=value["reasoning_summaries"], model=value["model"], metadata=value["metadata"]) + + +def _redirect_worker_stdio() -> None: + descriptor = os.open(os.devnull, os.O_RDWR) + try: + os.dup2(descriptor, 1) + os.dup2(descriptor, 2) + sys.stdout = open(1, "w", buffering=1, encoding="utf-8", errors="backslashreplace", closefd=False) + sys.stderr = open(2, "w", buffering=1, encoding="utf-8", errors="backslashreplace", closefd=False) + finally: + if descriptor not in (1, 2): + os.close(descriptor) + + +def _close_inherited_descriptors(keep: set[int]) -> None: + descriptor_root = "/proc/self/fd" if os.path.isdir("/proc/self/fd") else "/dev/fd" + try: + descriptors = tuple(int(name) for name in os.listdir(descriptor_root) if name.isdigit()) + except OSError: + import resource + soft_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + descriptors = tuple(range(min(int(soft_limit), 65_536))) + for descriptor in descriptors: + if descriptor in keep or descriptor in (1, 2): + continue + try: + os.close(descriptor) + except OSError: + continue + + +def _enforce_worker_sandbox( + workspace: str, + *, + allow_network: bool, + allow_process: bool, + allow_write: bool, + allow_workspace_read: bool, + module_read_paths: Sequence[str] = (), +) -> None: + import ctypes + import ctypes.util + import errno + import ssl + containment_root = os.path.realpath(workspace) + candidates = [ + sys.base_prefix, + sys.prefix, + "/System/Library", + "/Library/Apple", + "/usr/lib", + "/usr/lib64", + "/lib", + "/bin", + "/usr/bin", + "/lib64", + "/etc/hosts", + "/etc/resolv.conf", + "/etc/nsswitch.conf", + "/etc/gai.conf", + "/etc/services", + "/etc/ssl", + "/dev/null", + "/dev/random", + "/dev/urandom", + ] + if allow_workspace_read: + candidates.append(containment_root) + verify_paths = ssl.get_default_verify_paths() + candidates.extend((verify_paths.cafile, verify_paths.capath)) + candidates.extend(path for path in module_read_paths if os.path.isdir(path)) + read_roots = tuple(sorted({ + resolved + for path in candidates + if path and os.path.exists(path) + for resolved in (os.path.abspath(path), os.path.realpath(path)) + })) + module_files = tuple(sorted({ + resolved + for path in module_read_paths + if os.path.isfile(path) + for resolved in (os.path.abspath(path), os.path.realpath(path)) + })) + module_directories = tuple(sorted({ + str(parent) + for path in module_files + for parent in Path(path).parents + if str(parent) != os.path.sep + })) + if sys.platform == "darwin": + library = ctypes.CDLL("/usr/lib/libsandbox.dylib") + error = ctypes.c_char_p() + library.sandbox_init.argtypes = [ctypes.c_char_p, ctypes.c_uint64, ctypes.POINTER(ctypes.c_char_p)] + library.sandbox_init.restype = ctypes.c_int + readable = ( + "(literal \"/\") " + + " ".join(f"(subpath {json.dumps(path)})" for path in read_roots) + + " " + + " ".join(f"(literal {json.dumps(path)})" for path in module_files) + ) + profile = ( + "(version 1)(allow default)" + + ("" if allow_process else "(deny process-fork)(deny process-exec)") + + ("" if allow_network else "(deny network*)") + + ("" if allow_write else "(deny file-write* (require-not (literal " + json.dumps(os.devnull) + ")))") + + (f"(deny file-write* (require-not (require-any (subpath {json.dumps(containment_root)}) (literal {json.dumps(os.devnull)}))))" if allow_write else "") + + f"(deny file-read-data (require-not (require-any {readable})))" + ) + if library.sandbox_init(profile.encode("utf-8"), 0, ctypes.byref(error)) != 0: + raise ReplayRunError("could not enforce the replay worker sandbox") + return + if sys.platform.startswith("linux"): + library_name = ctypes.util.find_library("seccomp") + if not library_name: + raise ReplayRunError("libseccomp is required for replay process containment") + library = ctypes.CDLL(library_name) + library.seccomp_init.argtypes = [ctypes.c_uint32] + library.seccomp_init.restype = ctypes.c_void_p + library.seccomp_syscall_resolve_name.argtypes = [ctypes.c_char_p] + library.seccomp_syscall_resolve_name.restype = ctypes.c_int + library.seccomp_rule_add.argtypes = [ctypes.c_void_p, ctypes.c_uint32, ctypes.c_int, ctypes.c_uint] + library.seccomp_rule_add.restype = ctypes.c_int + library.seccomp_load.argtypes = [ctypes.c_void_p] + library.seccomp_load.restype = ctypes.c_int + library.seccomp_release.argtypes = [ctypes.c_void_p] + context = library.seccomp_init(0x7FFF0000) + if not context: + raise ReplayRunError("could not initialize replay process containment") + try: + deny = 0x00050000 | errno.EPERM + denied_syscalls = [] if allow_process else [b"fork", b"vfork", b"clone", b"clone3", b"execve", b"execveat"] + if not allow_network: + denied_syscalls.extend((b"socket", b"socketpair", b"connect", b"bind", b"listen", b"accept", b"accept4", b"sendto", b"sendmsg", b"recvfrom", b"recvmsg", b"shutdown", b"getsockopt", b"setsockopt", b"getpeername", b"getsockname")) + for syscall_name in denied_syscalls: + syscall = library.seccomp_syscall_resolve_name(syscall_name) + if syscall < 0: + raise ReplayRunError(f"libseccomp cannot resolve required replay syscall {syscall_name.decode('ascii')}") + if library.seccomp_rule_add(context, deny, syscall, 0) != 0: + raise ReplayRunError("could not configure replay process containment") + if library.seccomp_load(context) != 0: + raise ReplayRunError("could not enforce replay process containment") + finally: + library.seccomp_release(context) + + class _RulesetAttr(ctypes.Structure): + _fields_ = [("handled_access_fs", ctypes.c_uint64)] + + class _PathBeneathAttr(ctypes.Structure): + _fields_ = [("allowed_access", ctypes.c_uint64), ("parent_fd", ctypes.c_int32)] + + libc = ctypes.CDLL(None, use_errno=True) + libc.syscall.restype = ctypes.c_long + abi = libc.syscall(444, ctypes.c_void_p(), 0, 1) + if abi < 1: + raise ReplayRunError("Linux Landlock is required for replay filesystem containment") + read_file, read_dir = 1 << 2, 1 << 3 + write_access = sum(1 << bit for bit in range(1, 13) if bit not in (2, 3)) + if abi >= 2: + write_access |= 1 << 13 + if abi >= 3: + write_access |= 1 << 14 + handled_access = read_file | read_dir | write_access + ruleset_attr = _RulesetAttr(handled_access) + ruleset_fd = libc.syscall(444, ctypes.byref(ruleset_attr), ctypes.sizeof(ruleset_attr), 0) + if ruleset_fd < 0: + raise ReplayRunError(f"could not create replay filesystem ruleset: {os.strerror(ctypes.get_errno())}") + opened: list[int] = [] + try: + for path in module_directories: + path_fd = os.open(path, getattr(os, "O_PATH", os.O_RDONLY) | getattr(os, "O_CLOEXEC", 0)) + opened.append(path_fd) + path_rule = _PathBeneathAttr(read_dir, path_fd) + if libc.syscall(445, ruleset_fd, 1, ctypes.byref(path_rule), 0) < 0: + raise ReplayRunError(f"could not configure replay module search boundary: {os.strerror(ctypes.get_errno())}") + for path in module_files: + path_fd = os.open(path, getattr(os, "O_PATH", os.O_RDONLY) | getattr(os, "O_CLOEXEC", 0)) + opened.append(path_fd) + path_rule = _PathBeneathAttr(read_file, path_fd) + if libc.syscall(445, ruleset_fd, 1, ctypes.byref(path_rule), 0) < 0: + raise ReplayRunError(f"could not configure replay module file boundary: {os.strerror(ctypes.get_errno())}") + for path in read_roots: + path_fd = os.open(path, getattr(os, "O_PATH", os.O_RDONLY) | getattr(os, "O_CLOEXEC", 0)) + opened.append(path_fd) + allowed_access = read_file | (read_dir if os.path.isdir(path) else 0) + if path == containment_root and allow_write: + allowed_access |= write_access + path_rule = _PathBeneathAttr(allowed_access, path_fd) + if libc.syscall(445, ruleset_fd, 1, ctypes.byref(path_rule), 0) < 0: + raise ReplayRunError(f"could not configure replay filesystem boundary: {os.strerror(ctypes.get_errno())}") + libc.prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong, ctypes.c_ulong] + libc.prctl.restype = ctypes.c_int + if libc.prctl(38, 1, 0, 0, 0) != 0: + raise ReplayRunError(f"could not lock replay worker privileges: {os.strerror(ctypes.get_errno())}") + if libc.syscall(446, ruleset_fd, 0) < 0: + raise ReplayRunError(f"could not enforce replay filesystem boundary: {os.strerror(ctypes.get_errno())}") + finally: + for path_fd in opened: + os.close(path_fd) + os.close(ruleset_fd) + return + raise ReplayRunError("the current platform has no replay containment backend") + + +class _ProviderCapability: + def __init__(self, provider: Any, client: Any, client_spec: Mapping[str, Any] | None, model: str, context: Any, environment: Mapping[str, str], secret_bindings: Mapping[str, str], expected_client_binding: Mapping[str, Any]) -> None: + self.provider = provider + self.client = client + self.client_spec = dict(client_spec) if client_spec is not None else None + self.model = model + self.context = context + self.environment = dict(environment) + self.secret_bindings = dict(secret_bindings) + self.expected_client_binding = dict(expected_client_binding) + + +class _BoundPolicyCapability: + def __init__(self, target: Any, method_name: str) -> None: + self.target = target + self.method_name = method_name + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return getattr(self.target, self.method_name)(*args, **kwargs) + + +class _PartialPolicyCapability: + def __init__(self, function: Any, args: Sequence[Any], keywords: Mapping[str, Any]) -> None: + self.function = function + self.args = tuple(args) + self.keywords = dict(keywords) + + def __call__(self, name: str, arguments: Mapping[str, Any]) -> Any: + return self.function(*self.args, name, arguments, **self.keywords) + + +def _policy_capability(authorize: Any) -> Any: + if isinstance(authorize, functools.partial): + return _PartialPolicyCapability(_policy_capability(authorize.func), authorize.args, authorize.keywords or {}) + if inspect.ismethod(authorize) and authorize.__self__ is not None: + return _BoundPolicyCapability(authorize.__self__, authorize.__name__) + return authorize + + +def _isolated_provider(provider: Any) -> Any: + if type(provider).__module__ != "breadboard.product.integrations.provider" or type(provider).__qualname__ != "ProviderRuntimeAdapter": + return provider + isolated_provider = type(provider).__new__(type(provider)) + for name, value in _plain_object_state(provider).items(): + object.__setattr__(isolated_provider, name, [] if name in {"_client_identities", "_client_replay_specs"} else value) + return isolated_provider + + +def _provider_capability(provider: Any, client: Any, model: str, context: Any, environment: Mapping[str, str], secret_bindings: Mapping[str, str], expected_client_binding: Mapping[str, Any]) -> _ProviderCapability: + spec_method = getattr(provider, "replay_worker_client_spec", None) + client_spec = spec_method(client, secret_bindings) if callable(spec_method) else None + if client_spec is not None and not callable(getattr(provider, "replay_worker_client", None)): + raise ReplayRunError("provider client spec requires replay_worker_client reconstruction") + isolated_provider = _isolated_provider(provider) + return _ProviderCapability(isolated_provider, None if client_spec is not None else client, client_spec, model, context, environment, secret_bindings, expected_client_binding) + + +def _state_capability_kind(value: Any) -> str | None: + if callable(getattr(value, "invoke", None)): + return "provider" + if callable(getattr(value, "execute", None)): + if callable(getattr(value, "replay_process_containment", None)) and (callable(getattr(value, "workspace", None)) or callable(getattr(value, "get_workspace", None))): + return "host" + return "tool" + return None +def _host_network_access(host: Any) -> bool: + descriptor = getattr(host, "descriptor", None) + capabilities = getattr(descriptor, "capabilities", ()) + effects = getattr(descriptor, "effects", ()) + return "network" in capabilities or "network" in effects + + + + +def _fixed_timezone_state(value: datetime.tzinfo | None) -> dict[str, Any] | None: + if value is None: + return None + if type(value) is not datetime.timezone: + raise ReplayRunError(f"timezone type {type(value).__module__}.{type(value).__qualname__} cannot be encoded losslessly") + offset = value.utcoffset(None) + if offset is None: + raise ReplayRunError("fixed timezone does not expose an offset") + offset_microseconds = ((offset.days * 86_400 + offset.seconds) * 1_000_000) + offset.microseconds + return {"offset_microseconds": offset_microseconds, "name": value.tzname(None)} + + +def _scalar_state(value: Any) -> dict[str, Any] | None: + if type(value) is socket.socket: + return {"$closed_socket": True} + if type(value) is object: + return {"$object_sentinel": True} + if type(value) is Decimal: + return {"$decimal": str(value)} + if type(value) is datetime.datetime: + return {"$datetime": { + "value": value.replace(tzinfo=None).isoformat(timespec="microseconds"), + "fold": value.fold, + "timezone": _fixed_timezone_state(value.tzinfo), + }} + if type(value) is datetime.date: + return {"$date": value.isoformat()} + if type(value) is datetime.time: + return {"$time": { + "value": value.replace(tzinfo=None).isoformat(timespec="microseconds"), + "fold": value.fold, + "timezone": _fixed_timezone_state(value.tzinfo), + }} + if type(value) is datetime.timedelta: + return {"$timedelta": {"days": value.days, "seconds": value.seconds, "microseconds": value.microseconds}} + if type(value) is datetime.timezone: + return {"$timezone": _fixed_timezone_state(value)} + if type(value) is uuid.UUID: + return {"$uuid": {"hex": value.hex, "is_safe": value.is_safe.name}} + if isinstance(value, enum.Enum): + value_type = type(value) + if value_type.__module__ in {"__main__", "__mp_main__"} or "" in value_type.__qualname__: + raise ReplayRunError("capability enum types must be importable outside the entry-point module") + common = {"module": value_type.__module__, "qualname": value_type.__qualname__} + if isinstance(value.name, str) and value_type.__members__.get(value.name) is value: + return {"$enum": {**common, "name": value.name}} + if isinstance(value, enum.Flag) and type(value.value) is int: + return {"$enum": {**common, "value": value.value}} + raise ReplayRunError(f"enum value {value_type.__module__}.{value_type.__qualname__} cannot be encoded losslessly") + return None + + +def _restore_fixed_timezone(value: Mapping[str, Any] | None) -> datetime.tzinfo | None: + if value is None: + return None + return datetime.timezone(datetime.timedelta(microseconds=value["offset_microseconds"]), value["name"]) + + +def _tool_state_value(value: Any, capability_kind: str, seen: frozenset[int] = frozenset(), *, allow_callable: bool = False) -> Any: + if value is None or type(value) in (bool, int, float, str): + return value + if isinstance(value, Path): + return {"$path": str(value)} + if type(value) is bytes: + return {"$bytes": value.hex()} + scalar_state = _scalar_state(value) + if scalar_state is not None: + return scalar_state + nested_kind = _state_capability_kind(value) + if nested_kind is not None and nested_kind != capability_kind: + raise ReplayRunError(f"{capability_kind} capability state contains a forbidden {nested_kind} capability") + if id(value) in seen: + raise ReplayRunError("capability executor state must not contain cycles") + next_seen = seen | {id(value)} + if allow_callable and isinstance(value, functools.partial): + return {"$partial": { + "function": _tool_state_value(value.func, capability_kind, next_seen, allow_callable=True), + "args": _tool_state_value(value.args, capability_kind, next_seen), + "keywords": _tool_state_value(value.keywords or {}, capability_kind, next_seen), + }} + if allow_callable and inspect.ismethod(value) and value.__self__ is not None: + return {"$bound_method": { + "target": _tool_state_value(value.__self__, capability_kind, next_seen), + "name": value.__name__, + }} + if allow_callable and inspect.isbuiltin(value): + module, qualname = getattr(value, "__module__", None), getattr(value, "__qualname__", None) + if not isinstance(module, str) or not isinstance(qualname, str) or module in {"__main__", "__mp_main__"}: + raise ReplayRunError("built-in capability callables must expose an importable identity outside the entry-point module") + return {"$callable": {"module": module, "qualname": qualname}} + if type(value) is dict: + if any(type(name) is not str for name in value): + raise ReplayRunError("capability executor state mappings require string keys") + return {"$mapping": {name: _tool_state_value(item, capability_kind, next_seen) for name, item in sorted(value.items())}} + if type(value) in (list, tuple): + return {"$sequence": [_tool_state_value(item, capability_kind, next_seen) for item in value], "$tuple": type(value) is tuple} + if type(value) in (set, frozenset): + items = [_tool_state_value(item, capability_kind, next_seen) for item in value] + return {"$set": sorted(items, key=canonical_json), "$frozen": type(value) is frozenset} + if inspect.isfunction(value): + if capability_kind != "policy" and not allow_callable: + raise ReplayRunError(f"{capability_kind} capability state contains a forbidden policy callable") + if value.__module__ in {"__main__", "__mp_main__"} or "" in value.__qualname__ or "" in value.__qualname__: + raise ReplayRunError("capability callables must be importable outside the entry-point module") + return {"$callable": {"module": value.__module__, "qualname": value.__qualname__}} + value_type = type(value) + if value_type.__module__ in {"__main__", "__mp_main__"} or "" in value_type.__qualname__: + raise ReplayRunError("capability state types must be importable outside the entry-point module") + state = _plain_object_state(value) + if value_type.__new__ is not object.__new__: + raise ReplayRunError(f"capability value type {value_type.__module__}.{value_type.__qualname__} cannot be encoded losslessly") + return {"$object": { + "module": value_type.__module__, + "qualname": value_type.__qualname__, + "state": {name: _tool_state_value(item, capability_kind, next_seen) for name, item in sorted(state.items())}, + }} + + +def _plain_object_state(value: Any) -> dict[str, Any]: + try: + namespace = object.__getattribute__(value, "__dict__") + except AttributeError: + state: dict[str, Any] = {} + else: + state = dict(namespace) if isinstance(namespace, Mapping) else {} + for owner_type in type(value).__mro__: + slots = vars(owner_type).get("__slots__", ()) + for name in slots if isinstance(slots, (tuple, list)) else (slots,): + if not isinstance(name, str) or name in {"__dict__", "__weakref__"}: + continue + owner_name = owner_type.__name__.lstrip("_") + storage_name = f"_{owner_name}{name}" if owner_name and name.startswith("__") and not name.endswith("__") else name + try: + item = object.__getattribute__(value, storage_name) + except AttributeError: + continue + state.setdefault(storage_name, item) + return state + + +def _tool_executor_envelope(executor: Any, capability_kind: str) -> bytes: + if inspect.isfunction(executor): + if executor.__module__ in {"__main__", "__mp_main__"} or "" in executor.__qualname__ or "" in executor.__qualname__: + raise ReplayRunError("capability callables must be importable outside the entry-point module") + return canonical_json({"kind": "symbol", "module": executor.__module__, "qualname": executor.__qualname__}) + executor_type = type(executor) + qualname = executor_type.__qualname__ + if executor_type.__module__ in {"__main__", "__mp_main__"} or "" in qualname: + raise ReplayRunError("capability executor types must be importable outside the entry-point module") + callable_tool_adapter = capability_kind == "tool" and executor_type.__module__ == "breadboard.product.integrations.tool" and qualname == "ToolIntegrationAdapter" + return canonical_json({ + "kind": "object", + "module": executor_type.__module__, + "qualname": qualname, + "state": {name: _tool_state_value(value, capability_kind, allow_callable=callable_tool_adapter and name == "executor") for name, value in sorted(_plain_object_state(executor).items())}, + }) + + +def _path_is_within(path: str, roots: Sequence[str]) -> bool: + resolved = os.path.realpath(path) + for root in roots: + try: + if os.path.commonpath((resolved, root)) == root: + return True + except ValueError: + continue + return False + + +def _add_package_module_allowlist( + module_name: str, + runtime_roots: Sequence[str], + paths: set[str], + exact_specs: set[tuple[str, str, tuple[str, ...]]], + dependency_files: set[tuple[str, str]] | None = None, +) -> None: + package_name = module_name + package_locations: tuple[str, ...] = () + while package_name: + module = sys.modules.get(package_name) + spec = getattr(module, "__spec__", None) + if spec is None: + try: + spec = importlib.util.find_spec(package_name) + except (AttributeError, ImportError, ValueError): + spec = None + package_locations = tuple( + os.path.abspath(location) + for location in (() if spec is None or spec.submodule_search_locations is None else spec.submodule_search_locations) + if isinstance(location, str) and not _path_is_within(location, runtime_roots) + ) + if package_locations: + break + package_name = package_name.rpartition(".")[0] + if not package_locations: + return + parent_name = package_name.rpartition(".")[0] + while parent_name: + parent_module = sys.modules.get(parent_name) + parent_spec = getattr(parent_module, "__spec__", None) + if parent_spec is None: + try: + parent_spec = importlib.util.find_spec(parent_name) + except (AttributeError, ImportError, ValueError): + parent_spec = None + parent_location = getattr(parent_module, "__file__", None) or getattr(parent_spec, "origin", None) + parent_search_locations = tuple( + os.path.abspath(location) + for location in (() if parent_spec is None or parent_spec.submodule_search_locations is None else parent_spec.submodule_search_locations) + if isinstance(location, str) + ) + if isinstance(parent_location, str) and parent_location not in {"built-in", "frozen"}: + parent_location = os.path.abspath(parent_location) + paths.update((parent_location, os.path.realpath(parent_location))) + exact_specs.add((parent_name, parent_location, parent_search_locations)) + if dependency_files is not None: + dependency_files.add((f"{parent_name}:__init__", parent_location)) + parent_name = parent_name.rpartition(".")[0] + import_suffixes = tuple(sorted((".py", ".pyc", *importlib.machinery.EXTENSION_SUFFIXES), key=len, reverse=True)) + for package_location in package_locations: + package_root = Path(package_location) + for candidate in package_root.rglob("*"): + if candidate.is_symlink() or not candidate.is_file(): + continue + relative = candidate.relative_to(package_root) + if "__pycache__" in relative.parts: + continue + location = os.path.abspath(candidate) + paths.update((location, os.path.realpath(location))) + if dependency_files is not None: + dependency_files.add((f"{package_name}:{relative.as_posix()}", location)) + filename = relative.name + suffix = next((item for item in import_suffixes if filename.endswith(item)), None) + if suffix is None: + continue + stem = filename[:-len(suffix)] + if stem == "__init__": + components = relative.parts[:-1] + discovered_name = ".".join((package_name, *components)) + search_locations = (os.path.abspath(candidate.parent),) + else: + components = (*relative.parts[:-1], stem) + discovered_name = ".".join((package_name, *components)) + search_locations = () + exact_specs.add((discovered_name, location, search_locations)) + if suffix == ".py": + cached = importlib.util.cache_from_source(location) + if os.path.isfile(cached): + paths.update((os.path.abspath(cached), os.path.realpath(cached))) + + + + +def _add_top_level_sibling_allowlist( + module_name: str, + runtime_roots: Sequence[str], + paths: set[str], + exact_specs: set[tuple[str, str, tuple[str, ...]]], + dependency_files: set[tuple[str, str]] | None = None, +) -> None: + if "." in module_name: + return + module = sys.modules.get(module_name) + spec = getattr(module, "__spec__", None) + if spec is None: + try: + spec = importlib.util.find_spec(module_name) + except (AttributeError, ImportError, ValueError): + return + location = getattr(module, "__file__", None) or getattr(spec, "origin", None) + if not isinstance(location, str) or location in {"built-in", "frozen"}: + return + module_root = Path(os.path.realpath(location)).parent + if _path_is_within(str(module_root), runtime_roots): + return + import_suffixes = tuple(sorted((".py", ".pyc", *importlib.machinery.EXTENSION_SUFFIXES), key=len, reverse=True)) + for candidate in module_root.iterdir(): + if candidate.is_symlink(): + continue + if candidate.is_dir(): + package_name = candidate.name + package_init = next( + (candidate / f"__init__{suffix}" for suffix in import_suffixes if (candidate / f"__init__{suffix}").is_file()), + None, + ) if package_name.isidentifier() else None + if package_init is not None: + location = os.path.abspath(package_init) + exact_specs.add((package_name, location, (os.path.abspath(candidate),))) + paths.update((location, os.path.realpath(location))) + if dependency_files is not None: + dependency_files.add((f"{module_name}:{candidate.name}/{package_init.name}", location)) + _add_package_module_allowlist(package_name, runtime_roots, paths, exact_specs, dependency_files) + continue + continue + if not candidate.is_file(): + continue + suffix = next((item for item in import_suffixes if candidate.name.endswith(item)), None) + if suffix is None: + continue + location = os.path.abspath(candidate) + paths.update((location, os.path.realpath(location))) + if dependency_files is not None: + dependency_files.add((f"{module_name}:{candidate.name}", location)) + sibling_name = candidate.name[:-len(suffix)] + if sibling_name == "__init__" or not sibling_name.isidentifier(): + continue + exact_specs.add((sibling_name, location, ())) + if suffix == ".py": + cached = importlib.util.cache_from_source(location) + if os.path.isfile(cached): + paths.update((os.path.abspath(cached), os.path.realpath(cached))) + + +def _payload_module_names(payload: bytes) -> set[str]: + envelope = json.loads(payload) + modules: set[str] = set() + pending = [envelope] + while pending: + value = pending.pop() + if isinstance(value, dict): + module = value.get("module") + qualname = value.get("qualname") + if isinstance(module, str) and isinstance(qualname, str): + modules.add(module) + pending.extend(value.values()) + elif isinstance(value, list): + pending.extend(value) + return modules +_WORKER_BOOTSTRAP_MODULES = frozenset({ + "agentic_coder_prototype.logging.provider_dump", + "agentic_coder_prototype.provider.runtime", + "breadboard.product.evidence.replay_execution", + "breadboard.product.evidence.replay_plan", + "breadboard.product.evidence.replay_runner", + "breadboard.product.evidence.workspace", + "breadboard.product.harness.lock", + "breadboard.product.integrations.provider", + "breadboard.product.runtime.artifacts", + "breadboard.product.runtime.events", + "jsonschema", + "referencing", +}) + + +def _worker_module_names(payload: bytes) -> set[str]: + return {*_WORKER_BOOTSTRAP_MODULES, *_payload_module_names(payload)} +def _loaded_module_closure(module_names: Sequence[str]) -> set[str]: + pending = list(module_names) + discovered = set(pending) + while pending: + module = sys.modules.get(pending.pop()) + if module is None: + continue + for value in tuple(vars(module).values()): + referenced_name = ( + value.__name__ if inspect.ismodule(value) + else getattr(value, "__module__", None) if inspect.isclass(value) or inspect.isfunction(value) + else None + ) + if isinstance(referenced_name, str) and referenced_name not in discovered: + discovered.add(referenced_name) + pending.append(referenced_name) + return discovered + + + + + + +def _runtime_module_roots() -> tuple[str, ...]: + return tuple( + os.path.realpath(path) + for path in ( + sys.base_prefix, + sys.prefix, + "/System/Library", + "/Library/Apple", + "/usr/lib", + "/usr/lib64", + "/lib", + "/lib64", + ) + if os.path.isdir(path) + ) + + +def _executor_module_dependency_identity(payload: bytes) -> list[dict[str, Any]]: + runtime_roots = _runtime_module_roots() + paths: set[str] = set() + specs: set[tuple[str, str, tuple[str, ...]]] = set() + dependency_files: set[tuple[str, str]] = set() + payload_modules = _payload_module_names(payload) + modules = _loaded_module_closure(_worker_module_names(payload)) + for module_name in modules - payload_modules: + _add_package_module_allowlist(module_name, runtime_roots, paths, specs, dependency_files) + for module_name in payload_modules: + _add_package_module_allowlist(module_name, runtime_roots, paths, specs, dependency_files) + _add_top_level_sibling_allowlist(module_name, runtime_roots, paths, specs, dependency_files) + dependencies = [] + for logical_name, location in sorted(dependency_files): + try: + content = Path(location).read_bytes() + except OSError as exc: + raise ReplayRunError(f"capability dependency {logical_name} is not readable") from exc + dependencies.append({ + "name": logical_name, + "content_sha256": _digest(content), + }) + return dependencies + + + + +def _capability_runtime_identity(value: Any, capability_kind: str, *, payload: bytes | None = None) -> dict[str, Any]: + frozen_payload = _tool_executor_envelope(_isolated_provider(value) if capability_kind == "provider" else value, capability_kind) if payload is None else payload + identity = _runtime_type_identity(value) + identity["module_dependencies_sha256"] = sha256_json(_executor_module_dependency_identity(frozen_payload)) + return identity + + +def _executor_module_allowlist( + payload: bytes, +) -> tuple[tuple[str, ...], tuple[tuple[str, str, tuple[str, ...]], ...]]: + payload_modules = _payload_module_names(payload) + modules = _loaded_module_closure(_worker_module_names(payload)) + runtime_roots = _runtime_module_roots() + paths: set[str] = set(runtime_roots) + exact_specs: set[tuple[str, str, tuple[str, ...]]] = set() + for module_name in modules: + module = sys.modules.get(module_name) + spec = getattr(module, "__spec__", None) + if spec is None: + try: + spec = importlib.util.find_spec(module_name) + except (AttributeError, ImportError, ValueError): + spec = None + location = getattr(module, "__file__", None) + if not isinstance(location, str): + location = None if spec is None else spec.origin + search_locations = tuple( + os.path.abspath(path) + for path in (() if spec is None or spec.submodule_search_locations is None else spec.submodule_search_locations) + if isinstance(path, str) + ) + external_search_locations = tuple( + path for path in search_locations if not _path_is_within(path, runtime_roots) + ) + if not isinstance(location, str) or location in {"built-in", "frozen"}: + if external_search_locations: + exact_specs.add((module_name, "", external_search_locations)) + continue + resolved_location = os.path.realpath(location) + if _path_is_within(resolved_location, runtime_roots): + continue + exact_specs.add((module_name, os.path.abspath(location), external_search_locations)) + candidates = [location] + if location.endswith(".py"): + candidates.append(importlib.util.cache_from_source(location)) + for candidate in candidates: + if os.path.isfile(candidate): + paths.update((os.path.abspath(candidate), os.path.realpath(candidate))) + for module_name in modules - payload_modules: + _add_package_module_allowlist(module_name, runtime_roots, paths, exact_specs) + for module_name in payload_modules: + _add_package_module_allowlist(module_name, runtime_roots, paths, exact_specs) + _add_top_level_sibling_allowlist(module_name, runtime_roots, paths, exact_specs) + return tuple(sorted(paths)), tuple(sorted(exact_specs)) + + +class _ExactModuleFinder: + def __init__(self, module_specs: Sequence[tuple[str, str, tuple[str, ...]]]) -> None: + self._module_specs = { + name: (location, search_locations) + for name, location, search_locations in module_specs + } + + def find_spec(self, fullname: str, path: Any = None, target: Any = None) -> Any: + module_spec = self._module_specs.get(fullname) + if module_spec is None: + return None + location, search_locations = module_spec + if not location: + spec = importlib.machinery.ModuleSpec(fullname, loader=None, is_package=True) + spec.submodule_search_locations = list(search_locations) + return spec + return importlib.util.spec_from_file_location( + fullname, + location, + submodule_search_locations=list(search_locations) or None, + ) + + +def _restore_tool_state(value: Any) -> Any: + if not isinstance(value, dict) or not any(type(name) is str and name.startswith("$") for name in value): + return value + if set(value) == {"$path"}: + return Path(value["$path"]) + if set(value) == {"$bytes"}: + return bytes.fromhex(value["$bytes"]) + if set(value) == {"$closed_socket"} and value["$closed_socket"] is True: + return socket.socket.__new__(socket.socket) + if set(value) == {"$object_sentinel"} and value["$object_sentinel"] is True: + return object() + if set(value) == {"$decimal"}: + return Decimal(value["$decimal"]) + if set(value) == {"$datetime"}: + envelope = value["$datetime"] + restored = datetime.datetime.fromisoformat(envelope["value"]) + return restored.replace(tzinfo=_restore_fixed_timezone(envelope["timezone"]), fold=envelope["fold"]) + if set(value) == {"$date"}: + return datetime.date.fromisoformat(value["$date"]) + if set(value) == {"$time"}: + envelope = value["$time"] + restored = datetime.time.fromisoformat(envelope["value"]) + return restored.replace(tzinfo=_restore_fixed_timezone(envelope["timezone"]), fold=envelope["fold"]) + if set(value) == {"$timedelta"}: + return datetime.timedelta(**value["$timedelta"]) + if set(value) == {"$timezone"}: + return _restore_fixed_timezone(value["$timezone"]) + if set(value) == {"$uuid"}: + envelope = value["$uuid"] + return uuid.UUID(hex=envelope["hex"], is_safe=uuid.SafeUUID[envelope["is_safe"]]) + if set(value) == {"$enum"}: + envelope = value["$enum"] + enum_type: Any = importlib.import_module(envelope["module"]) + for component in envelope["qualname"].split("."): + enum_type = getattr(enum_type, component) + if set(envelope) == {"module", "qualname", "name"}: + return enum_type[envelope["name"]] + if set(envelope) == {"module", "qualname", "value"} and issubclass(enum_type, enum.Flag) and type(envelope["value"]) is int: + return enum_type(envelope["value"]) + raise ReplayRunError("isolated capability enum state envelope is invalid") + if set(value) == {"$mapping"}: + return {name: _restore_tool_state(item) for name, item in value["$mapping"].items()} + if set(value) == {"$sequence", "$tuple"}: + items = [_restore_tool_state(item) for item in value["$sequence"]] + return tuple(items) if value["$tuple"] else items + if set(value) == {"$set", "$frozen"}: + items = {_restore_tool_state(item) for item in value["$set"]} + return frozenset(items) if value["$frozen"] else items + if set(value) == {"$partial"}: + partial = value["$partial"] + return functools.partial( + _restore_tool_state(partial["function"]), + *_restore_tool_state(partial["args"]), + **_restore_tool_state(partial["keywords"]), + ) + if set(value) == {"$bound_method"}: + bound = value["$bound_method"] + return getattr(_restore_tool_state(bound["target"]), bound["name"]) + if set(value) == {"$callable"}: + target: Any = importlib.import_module(value["$callable"]["module"]) + for component in value["$callable"]["qualname"].split("."): + target = getattr(target, component) + return target + if set(value) == {"$object"}: + envelope = value["$object"] + object_type: Any = importlib.import_module(envelope["module"]) + for component in envelope["qualname"].split("."): + object_type = getattr(object_type, component) + restored = object_type.__new__(object_type) + for name, item in envelope["state"].items(): + object.__setattr__(restored, name, _restore_tool_state(item)) + return restored + raise ReplayRunError("isolated capability executor state envelope is invalid") +def _exec_tool_worker_bootstrap( + command_fd: int, + result_fd: int, + payload_fd: int, + workspace_descriptor: int, + capability_kind: str, + allow_network: bool, + module_read_paths: Sequence[str], + module_specs: Sequence[tuple[str, str, tuple[str, ...]]], +) -> None: + from multiprocessing.connection import Connection + command_recv = Connection(command_fd, readable=True, writable=False) + result_send = Connection(result_fd, readable=False, writable=True) + _redirect_worker_stdio() + os.environ.clear() + _close_inherited_descriptors({command_fd, result_fd, payload_fd, workspace_descriptor}) + workspace = str(_staging_descriptor_path(workspace_descriptor)) + _enforce_worker_sandbox( + workspace, + allow_network=allow_network, + allow_process=False, + allow_workspace_read=capability_kind in {"host", "tool"}, + allow_write=capability_kind in {"host", "tool"}, + module_read_paths=module_read_paths, + ) + runtime_search_roots = tuple( + os.path.realpath(path) for path in module_read_paths if os.path.isdir(path) + ) + sys.path[:] = [ + path + for path in sys.path + if isinstance(path, str) and path and _path_is_within(path, runtime_search_roots) + ] + sys.meta_path.insert(0, _ExactModuleFinder(module_specs)) + allowed_module_names = {name for name, _, _ in module_specs} + for loaded_name, loaded_module in tuple(sys.modules.items()): + loaded_location = getattr(loaded_module, "__file__", None) + if ( + isinstance(loaded_location, str) + and not _path_is_within(loaded_location, runtime_search_roots) + and loaded_name not in allowed_module_names + and not any(allowed_name.startswith(loaded_name + ".") for allowed_name in allowed_module_names) + ): + sys.modules.pop(loaded_name, None) + os.fchdir(workspace_descriptor) + payload = bytearray() + while True: + chunk = os.read(payload_fd, 65536) + if not chunk: + break + payload.extend(chunk) + os.close(payload_fd) + envelope = _decode_canonical_json(bytes(payload)) + if not isinstance(envelope, dict) or envelope.get("kind") not in {"symbol", "object"} or not isinstance(envelope.get("module"), str) or not isinstance(envelope.get("qualname"), str): + raise ReplayRunError("isolated capability executor envelope is invalid") + executor: Any = importlib.import_module(envelope["module"]) + for component in envelope["qualname"].split("."): + executor = getattr(executor, component) + if envelope["kind"] == "object": + if not isinstance(envelope.get("state"), dict): + raise ReplayRunError("isolated capability executor state is invalid") + executor_type = executor + executor = executor_type.__new__(executor_type) + for name, value in envelope["state"].items(): + object.__setattr__(executor, name, _restore_tool_state(value)) + if capability_kind not in {"tool", "host", "policy", "provider"}: + raise ReplayRunError("isolated capability kind is invalid") + if capability_kind == "policy" and not callable(executor) or capability_kind in {"tool", "host"} and not callable(getattr(executor, "execute", None)) or capability_kind == "provider" and not isinstance(executor, _ProviderCapability): + raise ReplayRunError("isolated capability does not expose its required entrypoint") + provider_client = None + def prepare_provider_environment() -> None: + os.environ.update(executor.environment) + for reserved_name in ("KC_PROVIDER_LOG_DIR", "KC_PROVIDER_WORKSPACE", "KC_PROVIDER_SESSION_ID"): + os.environ.pop(reserved_name, None) + provider_dump_logger.log_dir = None + provider_dump_logger.workspace_override = None + provider_dump_logger.session_override = None + provider_dump_logger.enabled = False + if capability_kind == "provider": + prepare_provider_environment() + try: + client_factory = getattr(executor.provider, "replay_worker_client", None) + provider_client = client_factory(executor.client_spec) if executor.client_spec is not None else executor.client + finally: + os.environ.clear() + result_send.send_bytes(_worker_envelope("ready")) + while True: + try: + command = _decode_canonical_json(command_recv.recv_bytes()) + except EOFError: + return + try: + if not isinstance(command, dict): + raise ReplayRunError("isolated capability command is invalid") + os.fchdir(workspace_descriptor) + if capability_kind == "provider" and set(command) == {"messages", "tools"}: + prepare_provider_environment() + try: + identity_method = getattr(executor.provider, "replay_client_identity", None) + if not callable(identity_method) or json.loads(canonical_json(identity_method(provider_client, executor.secret_bindings)).decode("utf-8")) != executor.expected_client_binding: + raise ReplayRunError("provider client state drifted before invocation") + result = executor.provider.invoke(client=provider_client, model=executor.model, messages=command["messages"], tools=command["tools"], stream=False, context=executor.context) + if json.loads(canonical_json(identity_method(provider_client, executor.secret_bindings)).decode("utf-8")) != executor.expected_client_binding: + raise ReplayRunError("provider client state drifted during invocation") + value = provider_result_evidence(result) + metadata_setter = getattr(getattr(executor.context, "session_state", None), "set_provider_metadata", None) + if callable(metadata_setter): + for metadata_name, metadata_value in value["metadata"].items(): + metadata_setter(metadata_name, metadata_value) + finally: + os.environ.clear() + elif capability_kind == "policy" and set(command) == {"name", "arguments"} and isinstance(command["name"], str) and isinstance(command["arguments"], dict): + value = executor(command["name"], command["arguments"]) + elif capability_kind == "tool" and set(command) == {"arguments"} and isinstance(command["arguments"], dict): + value = executor.execute(command["arguments"]) + elif capability_kind == "host" and isinstance(command.get("command"), str) and isinstance(command.get("cwd"), str): + host_arguments = dict(command) + command_text = host_arguments.pop("command") + command_cwd = host_arguments.pop("cwd") + value = executor.execute(command_text, cwd=command_cwd, **host_arguments) + else: + raise ReplayRunError("isolated capability command is invalid") + except BaseException as exc: + try: + result_send.send_bytes(_worker_envelope("error", type(exc).__name__, kind=capability_kind)) + except BaseException: + return + continue + try: + result_send.send_bytes(_worker_envelope("ok", value, kind=capability_kind)) + except BaseException: + try: + result_send.send_bytes(_worker_envelope("invalid", None, kind=capability_kind)) + except BaseException: + return + + +class _ExecToolWorker: + def __init__(self, executor: Any, workspace_descriptor: int, *, capability_kind: str = "tool", payload: bytes | None = None, startup_timeout_ms: int = 2_000, startup_timeout_code: str = "replay.tool_timeout", startup_timeout_detail: str = "capability worker startup exceeded its deadline", cancelled: Callable[[], bool] | None = None) -> None: + from multiprocessing.connection import Connection + self._capability_kind = capability_kind + payload = _tool_executor_envelope(executor, capability_kind) if payload is None else payload + allow_network = capability_kind == "provider" or capability_kind == "host" and _host_network_access(executor) + module_read_paths, module_specs = _executor_module_allowlist(payload) + command_read, command_write = os.pipe() + result_read, result_write = os.pipe() + payload_stream = tempfile.TemporaryFile() + payload_stream.write(payload) + payload_stream.flush() + payload_stream.seek(0) + payload_read = payload_stream.fileno() + self._terminated = False + self._command_send = Connection(command_write, readable=False, writable=True) + self._result_recv = Connection(result_read, readable=True, writable=False) + module_paths = [os.path.abspath(path or os.getcwd()) for path in sys.path if isinstance(path, str)] + environment = {"PATH": os.defpath, "PYTHONPATH": os.pathsep.join(dict.fromkeys(module_paths))} + bootstrap = ( + "from breadboard.product.evidence.replay_runner import _exec_tool_worker_bootstrap as b;" + f"b({command_read},{result_write},{payload_read},{workspace_descriptor},{capability_kind!r},{allow_network!r},{module_read_paths!r},{module_specs!r})" + ) + try: + self._process = subprocess.Popen( + [sys.executable, "-S", "-c", bootstrap], + close_fds=True, + pass_fds=(command_read, result_write, payload_read, workspace_descriptor), + env=environment, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except BaseException: + for descriptor in (command_read, result_write): + os.close(descriptor) + payload_stream.close() + self._command_send.close() + self._result_recv.close() + raise + self._descendants: set[int] = set() + self._closed = False + os.close(command_read) + os.close(result_write) + payload_stream.close() + startup_deadline = time.monotonic() + max(1, startup_timeout_ms) / 1_000 + while not self._result_recv.poll(max(0, min(0.01, startup_deadline - time.monotonic()))): + if cancelled is not None and cancelled(): + self.close() + raise _RuntimeFailure("cancelled", "replay.cancelled", "replay was cancelled") + if time.monotonic() >= startup_deadline: + self.close() + raise _RuntimeFailure("timed_out", startup_timeout_code, startup_timeout_detail) + try: + status, _, _ = _decode_worker_envelope(self._result_recv.recv_bytes()) + except BaseException as exc: + self.close() + raise ReplayRunError("isolated capability worker failed during startup") from exc + if status != "ready": + self.close() + raise ReplayRunError("isolated capability worker failed during startup") + @staticmethod + def _children(pid: int) -> set[int]: + try: + result = subprocess.run(["/usr/bin/pgrep", "-P", str(pid)], check=False, capture_output=True, text=True, timeout=0.2) + except (FileNotFoundError, subprocess.SubprocessError): + return set() + return {int(value) for value in result.stdout.split() if value.isdigit()} + + def _refresh_descendants(self) -> None: + pending = [self._process.pid, *self._descendants] + seen = set(pending) + while pending: + for child in self._children(pending.pop()): + if child not in seen: + seen.add(child) + pending.append(child) + seen.discard(self._process.pid) + self._descendants.update(seen) + + def _signal_descendants(self, signum: int) -> None: + for pid in self._descendants: + try: + os.kill(pid, signum) + except ProcessLookupError: + pass + + + def invoke(self, command: Mapping[str, Any], *, timeout_ms: int, timeout_code: str, timeout_detail: str, cancelled: Callable[[], bool] | None, cancellation_grace_ms: int) -> Any: + if cancelled is not None and cancelled(): + raise _RuntimeFailure("cancelled", "replay.cancelled", "replay was cancelled") + if self._process.poll() is not None: + raise _RemoteCallError("isolated capability worker is unavailable") + self._command_send.send_bytes(canonical_json(dict(command))) + deadline = time.monotonic() + timeout_ms / 1000 + cancellation_deadline: float | None = None + next_descendant_scan = 0.0 + while True: + now = time.monotonic() + if now >= deadline: + self._terminate() + raise _RuntimeFailure("timed_out", timeout_code, timeout_detail) + if self._result_recv.poll(min(0.01, deadline - now)): + if time.monotonic() >= deadline: + self._terminate() + raise _RuntimeFailure("timed_out", timeout_code, timeout_detail) + self._refresh_descendants() + status, _, value = _decode_worker_envelope(self._result_recv.recv_bytes()) + if status == "ok": + return _provider_result_from_evidence(value) if self._capability_kind == "provider" else value + if status == "invalid": + return _InvalidWorkerValue() + if cancelled is not None and cancelled(): + raise _RuntimeFailure("cancelled", "replay.cancelled", "replay was cancelled") + raise _RemoteCallError(f"isolated capability execution failed with {value}") + now = time.monotonic() + if now >= next_descendant_scan: + self._refresh_descendants() + next_descendant_scan = now + 0.1 + if cancelled is not None and cancelled(): + cancellation_deadline = cancellation_deadline or now + cancellation_grace_ms / 1000 + if now >= cancellation_deadline: + self._terminate() + raise _RuntimeFailure("cancelled", "replay.cancellation_grace_exceeded", "replay cancellation exceeded its grace period") + elif self._process.poll() is not None: + raise _RemoteCallError("isolated capability worker exited without a result") + if now >= deadline: + self._terminate() + raise _RuntimeFailure("timed_out", timeout_code, timeout_detail) + + def _terminate(self) -> None: + if self._terminated: + return + self._refresh_descendants() + try: + os.killpg(self._process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + self._signal_descendants(signal.SIGTERM) + try: + self._process.wait(timeout=0.25) + except subprocess.TimeoutExpired: + pass + self._refresh_descendants() + try: + os.killpg(self._process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + self._signal_descendants(signal.SIGKILL) + try: + self._process.wait(timeout=1) + except subprocess.TimeoutExpired as exc: + raise ReplayRunError("isolated capability worker could not be terminated") from exc + for _ in range(10): + survivors: list[int] = [] + for pid in self._descendants: + try: + os.kill(pid, 0) + except ProcessLookupError: + continue + survivors.append(pid) + if not survivors: + self._terminated = True + return + for pid in survivors: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + time.sleep(0.01) + raise ReplayRunError("isolated capability descendants survived termination") + + def close(self) -> None: + if self._closed: + return + self._terminate() + self._command_send.close() + self._result_recv.close() + self._closed = True + + + + + +def _digest(content: bytes) -> str: + return "sha256:" + hashlib.sha256(content).hexdigest() + + +def _initial_file_rows(files: Mapping[str, bytes | str]) -> list[dict[str, Any]]: + if not isinstance(files, Mapping): + raise ReplayRunError("initial_files must be a mapping") + if any(not isinstance(name, str) for name in files): + raise ReplayRunError("initial workspace paths must be strings") + rows: list[dict[str, Any]] = [] + for name, value in sorted(files.items()): + path = Path(name) + if not name or path.is_absolute() or path.as_posix() != name or ".." in path.parts or "\\" in name or any(part in ("", ".") for part in path.parts): + raise ReplayRunError(f"unsafe initial workspace path: {name!r}") + content = value.encode("utf-8") if isinstance(value, str) else value + if not isinstance(content, bytes): + raise ReplayRunError(f"initial workspace content must be bytes or text: {name}") + rows.append({"path": path.as_posix(), "sha256": _digest(content), "size_bytes": len(content)}) + return rows + + +def _materialize_initial(parent_descriptor: int, name: str, files: Mapping[str, bytes | str]) -> int: + if os.open not in os.supports_dir_fd or os.mkdir not in os.supports_dir_fd or not getattr(os, "O_NOFOLLOW", 0) or not getattr(os, "O_DIRECTORY", 0): + raise ReplayRunError("secure anchored workspace materialization is unavailable on this platform") + root_descriptor: int | None = None + try: + os.mkdir(name, mode=0o700, dir_fd=parent_descriptor) + root_descriptor = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_descriptor) + for row in _initial_file_rows(files): + content = files[row["path"]] + payload = content.encode("utf-8") if isinstance(content, str) else content + parent_fd = os.dup(root_descriptor) + try: + parts = Path(row["path"]).parts + for directory in parts[:-1]: + try: + os.mkdir(directory, mode=0o700, dir_fd=parent_fd) + except FileExistsError: + pass + child_fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_fd) + os.close(parent_fd) + parent_fd = child_fd + file_fd = os.open(parts[-1], os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=parent_fd) + try: + remaining = memoryview(payload) + while remaining: + written = os.write(file_fd, remaining) + if written < 1: + raise OSError("short workspace write") + remaining = remaining[written:] + finally: + os.close(file_fd) + except OSError as exc: + raise ReplayRunError(f"failed to materialize initial workspace path: {row['path']}") from exc + finally: + os.close(parent_fd) + return root_descriptor + except BaseException: + if root_descriptor is not None: + os.close(root_descriptor) + raise +class _AnchoredJsonlEventSink: + def __init__(self, parent_descriptor: int, name: str) -> None: + self._parent_descriptor = parent_descriptor + self._name = name + + def append(self, event: object) -> None: + payload = (json.dumps(event.as_dict(), allow_nan=False, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") # type: ignore[attr-defined] + descriptor = os.open(self._name, os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600, dir_fd=self._parent_descriptor) + try: + remaining = memoryview(payload) + while remaining: + written = os.write(descriptor, remaining) + if written < 1: + raise OSError("short replay event write") + remaining = remaining[written:] + os.fsync(descriptor) + os.fsync(self._parent_descriptor) + finally: + os.close(descriptor) + + +def _remove_entry_at(parent_descriptor: int, name: str) -> None: + metadata = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + if not stat.S_ISDIR(metadata.st_mode): + os.unlink(name, dir_fd=parent_descriptor) + return + descriptor = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_descriptor) + opened = os.fstat(descriptor) + if (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino): + os.close(descriptor) + raise ReplayRunError("staging entry changed during anchored cleanup") + stack: list[tuple[int, str, int]] = [(parent_descriptor, name, descriptor)] + try: + while stack: + entry_parent, entry_name, entry_descriptor = stack[-1] + children = os.listdir(entry_descriptor) + if children: + child_name = children[0] + child_metadata = os.stat(child_name, dir_fd=entry_descriptor, follow_symlinks=False) + if stat.S_ISDIR(child_metadata.st_mode): + child_descriptor = os.open(child_name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=entry_descriptor) + child_opened = os.fstat(child_descriptor) + if (child_opened.st_dev, child_opened.st_ino) != (child_metadata.st_dev, child_metadata.st_ino): + os.close(child_descriptor) + raise ReplayRunError("staging entry changed during anchored cleanup") + stack.append((entry_descriptor, child_name, child_descriptor)) + else: + os.unlink(child_name, dir_fd=entry_descriptor) + continue + os.fsync(entry_descriptor) + os.close(entry_descriptor) + stack.pop() + os.rmdir(entry_name, dir_fd=entry_parent) + except BaseException: + for _, _, open_descriptor in reversed(stack): + try: + os.close(open_descriptor) + except OSError: + pass + raise + + + + + + +def _snapshot(root: Path, expected_identity: tuple[int, int], source_descriptor: int | None = None, *, check: Callable[[], None] | None = None) -> dict[str, Any]: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + root_descriptor: int | None = None + rows: list[dict[str, Any]] = [] + + def walk(directory_descriptor: int, prefix: tuple[str, ...]) -> None: + if check is not None: + check() + for name in sorted(os.listdir(directory_descriptor)): + if check is not None: + check() + metadata = os.stat(name, dir_fd=directory_descriptor, follow_symlinks=False) + relative = "/".join((*prefix, name)) + if stat.S_ISLNK(metadata.st_mode): + raise _RuntimeFailure("host_failed", "replay.workspace_symlink", f"workspace contains a symlink: {relative}") + if stat.S_ISDIR(metadata.st_mode): + child_descriptor = os.open(name, flags, dir_fd=directory_descriptor) + try: + opened = os.fstat(child_descriptor) + if (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino): + raise _RuntimeFailure("host_failed", "replay.workspace_replaced", "workspace directory changed during snapshot") + walk(child_descriptor, (*prefix, name)) + finally: + os.close(child_descriptor) + continue + if not stat.S_ISREG(metadata.st_mode): + raise _RuntimeFailure("host_failed", "replay.workspace_special_file", f"workspace contains an unsupported special file: {relative}") + if metadata.st_nlink != 1: + raise _RuntimeFailure("host_failed", "replay.workspace_hardlink", f"workspace contains a hard-linked file: {relative}") + file_descriptor = os.open(name, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOCTTY", 0), dir_fd=directory_descriptor) + try: + opened = os.fstat(file_descriptor) + if (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino) or not stat.S_ISREG(opened.st_mode) or opened.st_nlink != 1: + raise _RuntimeFailure("host_failed", "replay.workspace_replaced", "workspace file changed during snapshot") + digest = hashlib.sha256() + size_bytes = 0 + while True: + if check is not None: + check() + chunk = os.read(file_descriptor, 1024 * 1024) + if not chunk: + break + digest.update(chunk) + size_bytes += len(chunk) + completed = os.fstat(file_descriptor) + stable_fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns", "st_nlink") + if any(getattr(opened, field) != getattr(completed, field) for field in stable_fields): + raise _RuntimeFailure("host_failed", "replay.workspace_replaced", "workspace file changed during snapshot") + rows.append({"path": relative, "sha256": "sha256:" + digest.hexdigest(), "size_bytes": size_bytes}) + finally: + os.close(file_descriptor) + + try: + root_descriptor = os.dup(source_descriptor) if source_descriptor is not None else os.open(root, flags) + root_metadata = os.fstat(root_descriptor) + if (root_metadata.st_dev, root_metadata.st_ino) != expected_identity: + raise _RuntimeFailure("host_failed", "replay.workspace_replaced", "fresh workspace root identity changed during replay") + walk(root_descriptor, ()) + except _RuntimeFailure: + raise + except OSError as exc: + raise _RuntimeFailure("host_failed", "replay.workspace_replaced", "workspace changed during snapshot") from exc + except RecursionError as exc: + raise _RuntimeFailure("host_failed", "replay.workspace_depth", "workspace nesting exceeded the snapshot limit") from exc + finally: + if root_descriptor is not None: + os.close(root_descriptor) + return {"schema_version": "bb.replay_workspace_snapshot.v1", "files": rows, "snapshot_sha256": sha256_json(rows)} + + +def _workspace_diff(before: Mapping[str, Any], after: Mapping[str, Any]) -> dict[str, Any]: + left = {row["path"]: row for row in before["files"]}; right = {row["path"]: row for row in after["files"]} + added = sorted(set(right) - set(left)); removed = sorted(set(left) - set(right)); changed = sorted(path for path in set(left) & set(right) if left[path] != right[path]) + return {"schema_version": "bb.replay_workspace_diff.v1", "added": added, "removed": removed, "changed": changed} + + +@dataclass +class _UsageAccountant: + token_limit: int + cost_limit: Decimal + tokens: int = 0 + cost: Decimal = Decimal(0) + currency: str | None = None + + @classmethod + def from_budgets(cls, budgets: Mapping[str, Any]) -> "_UsageAccountant": + try: + cost_limit = Decimal(str(budgets["cost"])) + except (InvalidOperation, KeyError, ValueError) as exc: + raise ReplayRunError("cost budget is not exactly representable") from exc + if not cost_limit.is_finite() or cost_limit <= 0: + raise ReplayRunError("cost budget must be finite and positive") + return cls(token_limit=budgets["tokens"], cost_limit=cost_limit) + + def add(self, usage: Mapping[str, Any]) -> bool: + tokens = usage.get("total_tokens") + currency = usage.get("cost_currency") + try: + cost = Decimal(str(usage.get("cost_amount"))) + except (InvalidOperation, ValueError) as exc: + raise ReplayRunError("provider cost is not exactly representable") from exc + if type(tokens) is not int or tokens < 0 or not isinstance(currency, str) or not currency or not cost.is_finite() or cost < 0: + raise ReplayRunError("provider usage cannot be accounted exactly") + if self.currency is None: + self.currency = currency + elif self.currency != currency: + raise _RuntimeFailure("provider_failed", "replay.cost_currency_mismatch", "provider usage changed cost currency during replay") + self.tokens += tokens + self.cost += cost + return self.tokens > self.token_limit or self.cost > self.cost_limit + + +def _redact(value: Any, *, secrets: Sequence[str], workspace: Path, counter: list[int], seen: frozenset[int] = frozenset()) -> Any: + secret_values = tuple(sorted({secret for secret in secrets if isinstance(secret, str) and secret}, key=lambda secret: (-len(secret), secret))) + if value is None or type(value) in (bool, int, float): + return value + if isinstance(value, str): + result = value.replace(str(workspace), "") + counter[0] += result != value + for secret in secret_values: + if secret in result: + result = result.replace(secret, ""); counter[0] += 1 + return result + if isinstance(value, Mapping): + if id(value) in seen: + raise ReplayRunError("cannot redact cyclic evidence") + result: dict[str, Any] = {} + for key, item in value.items(): + if not isinstance(key, str): + raise ReplayRunError("evidence keys must be strings") + redacted_key = _redact(key, secrets=secret_values, workspace=workspace, counter=counter) + if redacted_key in result: + raise ReplayRunError("redacted evidence keys collide") + normalized_key = key.lower().replace("-", "_") + compact_key = "".join(character for character in normalized_key if character.isalnum()) + sensitive_names = {"apikey", "token", "password", "passphrase", "secret", "authorization", "proxyauthorization", "bearer", "cookie", "setcookie", "credential", "credentials", "privatekey", "accesskey", "secretkey", "sessionid", "clientsecret", "signingkey", "encryptionkey", "sshkey"} + if compact_key in sensitive_names or any(compact_key.endswith(name) for name in sensitive_names): + result[redacted_key] = ""; counter[0] += 1 + else: + result[redacted_key] = _redact(item, secrets=secret_values, workspace=workspace, counter=counter, seen=seen | {id(value)}) + return result + if isinstance(value, (list, tuple)): + if id(value) in seen: + raise ReplayRunError("cannot redact cyclic evidence") + return [_redact(item, secrets=secret_values, workspace=workspace, counter=counter, seen=seen | {id(value)}) for item in value] + raise ReplayRunError(f"unsupported evidence value type: {type(value).__name__}") + + +@dataclass +class _ProviderSessionSnapshot: + metadata: dict[str, Any] + + def get_provider_metadata(self, key: str, default: Any = None) -> Any: + return self.metadata.get(key, default) + + def set_provider_metadata(self, key: str, value: Any) -> None: + self.metadata[key] = value + + +def _provider_context_snapshot(context: Any) -> ProviderRuntimeContext: + agent_config = getattr(context, "agent_config", {}) + extra = getattr(context, "extra", {}) + if not isinstance(agent_config, Mapping) or not isinstance(extra, Mapping): + raise ReplayRunError("provider context configuration must be a mapping") + try: + safe_agent_config = json.loads(canonical_json(agent_config)) + safe_extra_source = {name: extra[name] for name in ("phase16_phase_label", "turn_index", "responses_extra") if name in extra} + safe_extra = json.loads(canonical_json(safe_extra_source)) + session_state = getattr(context, "session_state", None) + metadata_source: dict[str, Any] = {} + snapshot = getattr(session_state, "provider_metadata_snapshot", None) + if callable(snapshot): + candidate = snapshot() + if not isinstance(candidate, Mapping): + raise ReplayRunError("provider metadata snapshot must be a mapping") + metadata_source.update(candidate) + else: + getter = getattr(session_state, "get_provider_metadata", None) + if callable(getter): + for name in ("conversation_id", "previous_response_id"): + value = getter(name) + if value is not None: + metadata_source[name] = value + safe_metadata = json.loads(canonical_json(metadata_source)) + except (TypeError, ValueError) as exc: + raise ReplayRunError("provider context must contain only canonical provider data") from exc + return ProviderRuntimeContext(session_state=_ProviderSessionSnapshot(safe_metadata), agent_config=safe_agent_config, stream=False, extra=safe_extra) + + +def _provider_context_request_payload(context: ProviderRuntimeContext) -> dict[str, Any]: + return json.loads(canonical_json({ + "agent_config": context.agent_config, + "extra": context.extra, + "session_metadata": context.session_state.metadata, + })) +def _model_policy_binding(model_policy: Any, provider_context: ProviderRuntimeContext) -> dict[str, Any]: + return { + "model_policy": json.loads(canonical_json(model_policy)), + "provider_context": _provider_context_request_payload(provider_context), + } + + + + + + + + +def _host_identity(host: Any) -> str: + method = getattr(host, "workspace", None) + if not callable(method): + method = getattr(host, "get_workspace", None) + if not callable(method): + raise _RuntimeFailure("host_failed", "replay.host_unavailable", "host does not expose a workspace identity") + value = method() + if not isinstance(value, str) or not value: + raise _RuntimeFailure("host_failed", "replay.host_unavailable", "host returned no workspace identity") + return value + + +def _host_containment_identity(host: Any) -> dict[str, Any]: + method = getattr(host, "replay_process_containment", None) + if not callable(method): + raise ReplayRunError("host does not attest detached-descendant containment") + value = json.loads(canonical_json(method()).decode("utf-8")) + if not isinstance(value, dict) or value.get("detached_descendants") != "contained": + raise ReplayRunError("host detached-descendant containment attestation is invalid") + return {**value, "worker_process_creation": "denied"} + + +def _provider_identity(provider: Any) -> tuple[str, str, str]: + descriptor = getattr(provider, "port_descriptor", None) or getattr(provider, "descriptor", None) + provider_id = getattr(provider, "provider_id", None) or getattr(descriptor, "provider_id", None) + runtime_id = getattr(provider, "runtime_id", None) or getattr(descriptor, "runtime_id", None) + endpoint = getattr(descriptor, "default_api_variant", None) or "completion" + if not all(isinstance(value, str) and value for value in (provider_id, runtime_id, endpoint)): + raise _RuntimeFailure("provider_failed", "replay.provider_identity", "provider does not expose a stable runtime identity") + return provider_id, runtime_id, endpoint + +def _validate_replay_provider_result(result: Any, request_id: str) -> ProviderResult: + if not isinstance(result, ProviderResult) or not isinstance(result.messages, list) or not result.messages: + raise ReplayRunError("provider must return at least one normalized message") + if not isinstance(result.metadata, Mapping): + raise ReplayRunError("provider metadata must be a mapping") + used_call_ids: set[str] = set() + calls_without_ids: list[tuple[int, int, ProviderToolCall]] = [] + for message_index, message in enumerate(result.messages): + if not isinstance(message, ProviderMessage) or message.role != "assistant": + raise ReplayRunError(f"provider message {message_index} must have assistant role") + if message.content is not None and not isinstance(message.content, str): + raise ReplayRunError(f"provider message {message_index} has invalid content") + if not isinstance(message.tool_calls, list): + raise ReplayRunError(f"provider message {message_index} has invalid tool_calls") + if message.finish_reason is not None and (not isinstance(message.finish_reason, str) or not message.finish_reason): + raise ReplayRunError(f"provider message {message_index} has an invalid finish_reason") + if message.index is not None and (type(message.index) is not int or message.index < 0): + raise ReplayRunError(f"provider message {message_index} has an invalid index") + for call_index, call in enumerate(message.tool_calls): + if not isinstance(call, ProviderToolCall): + raise ReplayRunError(f"provider tool call {message_index}:{call_index} is invalid") + valid_id = call.id is None or isinstance(call.id, str) and bool(call.id) + if call.type != "function" or not valid_id or not all(isinstance(getattr(call, name, None), str) and getattr(call, name) for name in ("name", "arguments")): + raise ReplayRunError(f"provider tool call {message_index}:{call_index} is invalid") + if call.id is None: + calls_without_ids.append((message_index, call_index, call)) + elif call.id in used_call_ids: + raise ReplayRunError(f"provider tool call {message_index}:{call_index} has a duplicate id") + else: + used_call_ids.add(call.id) + for message_index, call_index, call in calls_without_ids: + candidate = f"{request_id}:message:{message_index + 1}:call:{call_index + 1}" + suffix = 1 + while candidate in used_call_ids: + candidate = f"{request_id}:message:{message_index + 1}:call:{call_index + 1}:{suffix}" + suffix += 1 + call.id = candidate + used_call_ids.add(candidate) + return result + +def _declared_tool_names(tool_schemas: Sequence[Mapping[str, Any]]) -> frozenset[str]: + names: list[str] = [] + for index, schema in enumerate(tool_schemas): + function = schema.get("function") if isinstance(schema, Mapping) else None + name = function.get("name") if isinstance(function, Mapping) else None + if not isinstance(schema, Mapping) or schema.get("type") != "function" or not isinstance(name, str) or not name: + raise ReplayRunError(f"tool_schemas[{index}] must declare a populated function name") + names.append(name) + if len(names) != len(set(names)): + raise ReplayRunError("tool_schemas must declare unique function names") + return frozenset(names) +def _provider_tool_wire_schemas(tool_schemas: Sequence[Mapping[str, Any]]) -> tuple[list[dict[str, Any]], dict[str, str]]: + wire_schemas = json.loads(canonical_json(list(tool_schemas))) + used_names: set[str] = set() + reverse_aliases: dict[str, str] = {} + for index, schema in enumerate(wire_schemas): + function = schema["function"] + canonical_name = function["name"] + candidate = canonical_name + if re.fullmatch(r"[A-Za-z0-9_-]{1,64}", candidate) is None: + candidate = re.sub(r"[^A-Za-z0-9_-]", "_", canonical_name)[:48] or "tool" + if candidate in used_names: + suffix = hashlib.sha256(f"{canonical_name}:{index}".encode("utf-8")).hexdigest()[:8] + candidate = f"{candidate[:55]}_{suffix}" + counter = 1 + while candidate in used_names: + suffix = hashlib.sha256(f"{canonical_name}:{index}:{counter}".encode("utf-8")).hexdigest()[:8] + candidate = f"{candidate[:55]}_{suffix}" + counter += 1 + used_names.add(candidate) + function["name"] = candidate + if candidate != canonical_name: + reverse_aliases[candidate] = canonical_name + return wire_schemas, reverse_aliases +def _provider_wire_messages(messages: Sequence[Mapping[str, Any]], reverse_aliases: Mapping[str, str]) -> list[dict[str, Any]]: + aliases = {canonical: wire for wire, canonical in reverse_aliases.items()} + wire_messages = json.loads(canonical_json(list(messages))) + for message in wire_messages: + if message.get("name") in aliases and (message.get("role") == "tool" or message.get("type") in {"function_call", "tool_use", "tool_result"}): + message["name"] = aliases[message["name"]] + function_call = message.get("function_call") + if isinstance(function_call, dict) and function_call.get("name") in aliases: + function_call["name"] = aliases[function_call["name"]] + tool_calls = message.get("tool_calls") + if isinstance(tool_calls, list): + for call in tool_calls: + if isinstance(call, dict) and call.get("name") in aliases: + call["name"] = aliases[call["name"]] + function = call.get("function") if isinstance(call, dict) else None + if isinstance(function, dict) and function.get("name") in aliases: + function["name"] = aliases[function["name"]] + content = message.get("content") + if isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") in {"function_call", "tool_use", "tool_result"} and block.get("name") in aliases: + block["name"] = aliases[block["name"]] + return wire_messages + + +def _wire_tool_choice(value: Any, aliases: Mapping[str, str]) -> Any: + if isinstance(value, str): + return aliases.get(value, value) + if isinstance(value, dict): + return {name: _wire_tool_choice(item, aliases) for name, item in value.items()} + if isinstance(value, list): + return [_wire_tool_choice(item, aliases) for item in value] + return value + + +def _provider_context_with_wire_tool_aliases(context: ProviderRuntimeContext, reverse_aliases: Mapping[str, str]) -> ProviderRuntimeContext: + aliases = {canonical: wire for wire, canonical in reverse_aliases.items()} + provider_tools = context.agent_config.get("provider_tools") + if not aliases or not isinstance(provider_tools, dict): + return context + if "tool_choice" in provider_tools: + provider_tools["tool_choice"] = _wire_tool_choice(provider_tools["tool_choice"], aliases) + for provider_config in provider_tools.values(): + if isinstance(provider_config, dict) and "tool_choice" in provider_config: + provider_config["tool_choice"] = _wire_tool_choice(provider_config["tool_choice"], aliases) + return context + + + + + + +_REPLAY_SCHEMA_FILES = { + "plan": "bb.replay_plan.v1.schema.json", + "execution": "bb.replay_execution.v1.schema.json", + "manifest": "bb.replay_artifact_manifest.v1.schema.json", + "exchange": "bb.provider_exchange.v2.schema.json", + "problem": "bb.problem.v1.schema.json", +} + + +def _replay_schema_registry() -> tuple[dict[str, Any], dict[str, Draft202012Validator]]: + root = Path(__file__).resolve().parents[3] / "contracts/public/schemas" + schemas = {name: json.loads((root / filename).read_text(encoding="utf-8")) for name, filename in _REPLAY_SCHEMA_FILES.items()} + registry = Registry() + for schema in schemas.values(): + registry = registry.with_resource(schema["$id"], Resource.from_contents(schema)) + binding = {"schemas": {schema["$id"]: sha256_json(schema) for schema in schemas.values()}} + validators = {name: Draft202012Validator(schema, registry=registry) for name, schema in schemas.items() if name != "problem"} + return binding, validators + + +def _tool_argument_validators(tool_schemas: Sequence[Mapping[str, Any]]) -> dict[str, Draft202012Validator]: + validators: dict[str, Draft202012Validator] = {} + for index, schema in enumerate(tool_schemas): + function = schema.get("function") if isinstance(schema, Mapping) else None + name = function.get("name") if isinstance(function, Mapping) else None + parameters = function.get("parameters") if isinstance(function, Mapping) else None + if not isinstance(name, str) or not isinstance(parameters, Mapping): + raise ReplayRunError(f"tool_schemas[{index}] must contain an object parameters schema") + try: + Draft202012Validator.check_schema(parameters) + except SchemaError as exc: + raise ReplayRunError(f"tool_schemas[{index}] contains an invalid parameters schema") from exc + validators[name] = Draft202012Validator(parameters) + return validators + + +def _validated_arguments(call: ProviderToolCall, validators: Mapping[str, Draft202012Validator]) -> dict[str, Any]: + arguments = _arguments(call.arguments) + if call.name not in validators: + return arguments + try: + validators[call.name].validate(arguments) + except ValidationError as exc: + raise _InvalidProviderResponse("provider tool arguments violate the frozen schema") from exc + return arguments + + +def _provider_client_binding(provider: Any, provider_client: Any, secret_bindings: Mapping[str, str]) -> dict[str, Any]: + method = getattr(provider, "replay_client_identity", None) + if not callable(method): + raise ReplayRunError("provider does not expose replay_client_identity") + value = method(provider_client, secret_bindings) + try: + binding = json.loads(canonical_json(value)) + except (TypeError, ValueError) as exc: + raise ReplayRunError("provider client identity must be canonical") from exc + if not isinstance(binding, dict) or binding.get("verified") is not True: + raise ReplayRunError("provider client identity must attest verified configuration") + return binding + +def _provider_route_binding(provider_id: str, runtime_id: str, endpoint_class: str, provider_model: str, model_revision: str | None, runtime_version: str, provider: Any, provider_client_identity: Mapping[str, Any]) -> dict[str, Any]: + return { + "provider_id": provider_id, + "runtime_id": runtime_id, + "runtime_version": runtime_version, + "runtime_implementation": _capability_runtime_identity(provider, "provider"), + "client_identity": dict(provider_client_identity), + "endpoint_class": endpoint_class, + "model_id": provider_model, + "model_revision": model_revision, + } + +def _identity_value(value: Any, seen: frozenset[int] = frozenset()) -> Any: + if value is None or type(value) in (bool, int, float, str): + return value + if isinstance(value, Path): + return {"path_sha256": _digest(str(value).encode("utf-8"))} + if type(value) is bytes: + return {"bytes_sha256": _digest(value)} + scalar_state = _scalar_state(value) + if scalar_state is not None: + return scalar_state + if inspect.ismodule(value): + source_path = getattr(value, "__file__", None) + source_sha = _digest(Path(source_path).read_bytes()) if isinstance(source_path, str) and Path(source_path).is_file() else _digest(value.__name__.encode("utf-8")) + module_state = {name: item for name, item in vars(value).items() if not name.startswith("_") and (item is None or type(item) in (bool, int, float, str) or isinstance(item, (Mapping, list, tuple, set, frozenset)))} + return {"module": value.__name__, "version": str(getattr(value, "__version__", "")), "source_sha256": source_sha, "state_sha256": sha256_json(_identity_value(module_state, seen | {id(value)}))} + if isinstance(value, type): + return {"type": f"{value.__module__}.{value.__qualname__}"} + if isinstance(value, functools.partial) or inspect.isroutine(value): + return {"callable": _callable_identity(value)} + if id(value) in seen: + return {"cycle": f"{type(value).__module__}.{type(value).__qualname__}"} + next_seen = seen | {id(value)} + if type(value) is dict: + rows = [{"key": _identity_value(key, next_seen), "value": _identity_value(item, next_seen)} for key, item in value.items()] + return {"mapping": sorted(rows, key=canonical_json)} + if type(value) in (list, tuple, set, frozenset): + rows = [_identity_value(item, next_seen) for item in value] + return sorted(rows, key=canonical_json) if type(value) in (set, frozenset) else rows + transient = {"_client_identities", "_client_replay_specs"} if type(value) is ProviderRuntimeAdapter else set() + state = _plain_object_state(value) + if state: + return { + "type": f"{type(value).__module__}.{type(value).__qualname__}", + "state": {name: _identity_value(item, next_seen) for name, item in sorted(state.items()) if name not in transient}, + } + return {"type": f"{type(value).__module__}.{type(value).__qualname__}"} + + +def _runtime_type_identity(value: Any, seen: frozenset[int] = frozenset()) -> dict[str, Any]: + if inspect.isfunction(value): + symbol = f"{value.__module__}.{value.__qualname__}" + if value.__module__ in {"__main__", "__mp_main__"}: + raise ReplayRunError("runtime implementations must be importable outside the entry-point module") + try: + source = inspect.getsource(value).encode("utf-8") + except (OSError, TypeError) as exc: + raise ReplayRunError(f"runtime implementation {symbol} has no inspectable source") from exc + return { + "symbol": symbol, + "source_sha256": _digest(source), + "configuration_sha256": sha256_json(_identity_value({"args": value.__defaults__, "kwargs": value.__kwdefaults__})), + "methods_sha256": sha256_json({"call": _callable_identity(value)}), + } + if isinstance(value, functools.partial) or inspect.ismethod(value) or inspect.isbuiltin(value): + callable_identity = _callable_identity(value) + return { + "symbol": callable_identity["symbol"], + "source_sha256": sha256_json(callable_identity), + "configuration_sha256": sha256_json(_identity_value(value)), + "methods_sha256": sha256_json({"call": callable_identity}), + } + runtime_type = type(value) + symbol = f"{runtime_type.__module__}.{runtime_type.__qualname__}" + if runtime_type.__module__ in {"__main__", "__mp_main__"}: + raise ReplayRunError("runtime implementation types must be importable outside the entry-point module") + try: + source = inspect.getsource(runtime_type).encode("utf-8") + except (OSError, TypeError) as exc: + raise ReplayRunError(f"runtime implementation {symbol} has no inspectable source") from exc + methods: dict[str, Any] = {} + for owner_type in runtime_type.__mro__: + if owner_type is object: + continue + for method_name, raw_method in vars(owner_type).items(): + candidates = () + if inspect.isfunction(raw_method): + candidates = (raw_method,) + elif isinstance(raw_method, (staticmethod, classmethod)): + candidates = (raw_method.__func__,) + elif isinstance(raw_method, property): + candidates = tuple(method for method in (raw_method.fget, raw_method.fset, raw_method.fdel) if method is not None) + for index, method in enumerate(candidates): + methods[f"{owner_type.__module__}.{owner_type.__qualname__}.{method_name}:{index}"] = _callable_identity(method) + identity: dict[str, Any] = { + "symbol": symbol, + "source_sha256": _digest(source), + "configuration_sha256": sha256_json(_identity_value(value)), + "methods_sha256": sha256_json(methods), + } + if id(value) in seen: + return identity + delegates = {} + for name in ("runtime", "sandbox", "executor"): + delegate = getattr(value, name, None) + if delegate is not None and delegate is not value: + delegates[name] = _runtime_type_identity(delegate, seen | {id(value)}) + if delegates: + identity["delegates"] = delegates + return identity + + +def _callable_identity(value: Callable[..., Any]) -> dict[str, Any]: + if isinstance(value, functools.partial): + return { + "symbol": "functools.partial", + "callable": _callable_identity(value.func), + "args_sha256": sha256_json(_identity_value(value.args)), + "keywords_sha256": sha256_json(_identity_value(value.keywords or {})), + } + if inspect.isbuiltin(value): + module, qualname = getattr(value, "__module__", None), getattr(value, "__qualname__", None) + if not isinstance(module, str) or not isinstance(qualname, str): + raise ReplayRunError("built-in policy gate does not expose a stable callable identity") + symbol = f"{module}.{qualname}" + implementation = {"symbol": symbol, "python": platform.python_version(), "implementation": platform.python_implementation()} + return { + "symbol": symbol, + "source_sha256": sha256_json(implementation), + "nonlocals_sha256": sha256_json({}), + "globals_sha256": sha256_json(implementation), + "defaults_sha256": sha256_json({}), + } + module = getattr(value, "__module__", None) + qualname = getattr(value, "__qualname__", None) + if not isinstance(module, str) or not module or not isinstance(qualname, str) or not qualname: + raise ReplayRunError("policy gate does not expose a stable callable identity") + if module in {"__main__", "__mp_main__"}: + raise ReplayRunError("policy callables must be importable outside the entry-point module") + symbol = f"{module}.{qualname}" + try: + source = inspect.getsource(value).encode("utf-8") + closure = inspect.getclosurevars(value) + except (OSError, TypeError) as exc: + raise ReplayRunError(f"policy implementation {symbol} has no inspectable source") from exc + identity: dict[str, Any] = { + "symbol": symbol, + "source_sha256": _digest(source), + "nonlocals_sha256": sha256_json(_identity_value(closure.nonlocals)), + "globals_sha256": sha256_json(_identity_value(closure.globals)), + "defaults_sha256": sha256_json(_identity_value({"args": getattr(value, "__defaults__", None), "kwargs": getattr(value, "__kwdefaults__", None)})), + } + owner = getattr(value, "__self__", None) + if owner is not None: + identity["owner"] = _runtime_type_identity(owner) + return identity + + +def _host_platform_binding() -> dict[str, str]: + return { + "system": platform.system(), + "release": platform.release(), + "machine": platform.machine(), + "python_implementation": platform.python_implementation(), + "python_version": platform.python_version(), + "byteorder": sys.byteorder, + "replay_containment_backend": "seatbelt" if sys.platform == "darwin" else "landlock+seccomp" if sys.platform.startswith("linux") else "unsupported", + } + + +def _environment_binding(environment_allowlist: Sequence[str], secret_bindings: Mapping[str, str]) -> tuple[dict[str, Any], dict[str, str]]: + missing = [name for name in environment_allowlist if name not in secret_bindings and name not in os.environ] + if missing: + raise ReplayRunError("required replay environment values are absent: " + ", ".join(sorted(missing))) + values = {name: secret_bindings[name] if name in secret_bindings else os.environ[name] for name in environment_allowlist} + binding = {"names": sorted(environment_allowlist), "value_sha256": {name: _digest(values[name].encode("utf-8")) for name in sorted(values)}} + return binding, values + + +def _assistant_messages(result: ProviderResult, reverse_aliases: Mapping[str, str]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for message in result.messages: + row: dict[str, Any] = {"role": message.role, "content": message.content} + if message.tool_calls: + row["tool_calls"] = [{"id": call.id, "type": call.type, "function": {"name": reverse_aliases.get(call.name, call.name), "arguments": call.arguments}} for call in message.tool_calls] + rows.append(row) + return rows + + +def _arguments(value: str) -> dict[str, Any]: + def reject_constant(_value: str) -> Any: + raise _InvalidProviderResponse("provider tool arguments contain a non-JSON number") + + def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, item in pairs: + if key in result: + raise _InvalidProviderResponse(f"provider tool arguments contain duplicate key {key!r}") + result[key] = item + return result + + try: + parsed = json.loads(value, parse_constant=reject_constant, object_pairs_hook=unique_object) + canonical_json(parsed) + except _InvalidProviderResponse: + raise + except (TypeError, ValueError) as exc: + raise _InvalidProviderResponse("provider emitted invalid tool arguments") from exc + if not isinstance(parsed, dict): + raise _InvalidProviderResponse("provider tool arguments must be an object") + return parsed + + +def _open_directory_path(path: Path, *, create: bool) -> int: + absolute = path.absolute() + if not absolute.is_absolute(): + raise ReplayRunError("anchored storage root must be absolute") + descriptor = os.open(absolute.anchor, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)) + try: + for component in absolute.parts[1:]: + if create: + try: + os.mkdir(component, mode=0o700, dir_fd=descriptor) + except FileExistsError: + pass + child = os.open(component, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=descriptor) + os.close(descriptor) + descriptor = child + return descriptor + except BaseException: + os.close(descriptor) + raise + + +def _anchored_artifact_store(workspace: BreadBoardWorkspace) -> tuple[ArtifactStore, tuple[int, ...]]: + artifact_path = workspace.path(".breadboard/artifacts") + if os.name == "nt": + return ArtifactStore(artifact_path), () + descriptors: list[int] = [] + try: + descriptors.append(_open_directory_path(workspace.root, create=True)) + descriptors.append(AnchoredStorage.open_directory(descriptors[-1], ".breadboard")) + descriptors.append(AnchoredStorage.open_directory(descriptors[-1], "artifacts")) + return ArtifactStore(artifact_path, descriptor=descriptors[-1]), tuple(descriptors) + except BaseException as exc: + for descriptor in reversed(descriptors): + os.close(descriptor) + if isinstance(exc, OSError): + raise ReplayRunError("artifact storage namespace is not a secure directory") from exc + raise + + +def _anchored_replay_parent(workspace: BreadBoardWorkspace) -> tuple[Path, tuple[int, ...]]: + replay_path = workspace.path(".breadboard/replays") + if os.name == "nt": + replay_path.mkdir(parents=True, exist_ok=True) + return replay_path, () + descriptors: list[int] = [] + try: + descriptors.append(_open_directory_path(workspace.root, create=True)) + descriptors.append(AnchoredStorage.open_directory(descriptors[-1], ".breadboard")) + descriptors.append(AnchoredStorage.open_directory(descriptors[-1], "replays")) + return replay_path, tuple(descriptors) + except BaseException as exc: + for descriptor in reversed(descriptors): + os.close(descriptor) + if isinstance(exc, OSError): + raise ReplayRunError("replay storage namespace is not a secure directory") from exc + raise + + +def _close_descriptors(descriptors: Sequence[int]) -> None: + for descriptor in reversed(descriptors): + os.close(descriptor) + + +def _verify_staging_path(path: Path, expected_identity: tuple[int, int], descriptor: int) -> None: + descriptor_stat = os.fstat(descriptor) + if not path.exists() or path.is_symlink(): + raise ReplayRunError("replay staging path disappeared before finalization") + path_stat = path.lstat() + if (descriptor_stat.st_dev, descriptor_stat.st_ino) != expected_identity or (path_stat.st_dev, path_stat.st_ino) != expected_identity: + raise ReplayRunError("replay staging path changed before finalization") + + +def _rename_no_replace(source: Path, destination: Path, *, parent_descriptor: int | None = None) -> None: + import ctypes + import errno + libc = ctypes.CDLL(None, use_errno=True) + if parent_descriptor is None: + source_bytes, destination_bytes = os.fsencode(source), os.fsencode(destination) + source_parent = destination_parent = -2 if sys.platform == "darwin" else -100 + else: + source_bytes, destination_bytes = os.fsencode(source.name), os.fsencode(destination.name) + source_parent = destination_parent = parent_descriptor + if sys.platform == "darwin" and hasattr(libc, "renameatx_np"): + result = libc.renameatx_np(source_parent, source_bytes, destination_parent, destination_bytes, 0x00000004) + elif hasattr(libc, "renameat2"): + result = libc.renameat2(source_parent, source_bytes, destination_parent, destination_bytes, 0x00000001) + else: + if parent_descriptor is None: + if destination.exists(): + raise FileExistsError(destination) + os.rename(source, destination) + else: + try: + os.stat(destination.name, dir_fd=parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + os.rename(source.name, destination.name, src_dir_fd=parent_descriptor, dst_dir_fd=parent_descriptor) + else: + raise FileExistsError(destination) + return + if result != 0: + error = ctypes.get_errno() + raise OSError(error or errno.EIO, os.strerror(error or errno.EIO), str(destination)) + + +def _duration_ms(start: float, end: float) -> int: + return max(0, int((end - start) * 1000)) + + +def _staging_descriptor_path(descriptor: int) -> Path: + proc_path = Path("/proc/self/fd") / str(descriptor) + if proc_path.exists(): + return Path(os.readlink(proc_path)) + import fcntl + return Path(fcntl.fcntl(descriptor, 50, b"\0" * 1024).split(b"\0", 1)[0].decode()) + + +def _remove_tree_verified(parent_descriptor: int, name: str, expected_identity: tuple[int, int], descriptor: int) -> None: + descriptor_stat = os.fstat(descriptor) + if (descriptor_stat.st_dev, descriptor_stat.st_ino) != expected_identity: + raise ReplayRunError("replay staging descriptor identity changed before cleanup") + for child_name in os.listdir(descriptor): + _remove_entry_at(descriptor, child_name) + os.fsync(descriptor) + identity_names: list[str] = [] + for candidate_name in os.listdir(parent_descriptor): + candidate = os.stat(candidate_name, dir_fd=parent_descriptor, follow_symlinks=False) + if (candidate.st_dev, candidate.st_ino) == expected_identity: + identity_names.append(candidate_name) + if not identity_names: + if os.fstat(descriptor).st_nlink == 0: + return + raise ReplayRunError("replay staging inode remains linked at an unresolved path") + if len(identity_names) != 1: + raise ReplayRunError("replay staging inode has multiple namespace links") + os.rmdir(identity_names[0], dir_fd=parent_descriptor) + os.fsync(parent_descriptor) + if any((metadata.st_dev, metadata.st_ino) == expected_identity for candidate_name in os.listdir(parent_descriptor) for metadata in (os.stat(candidate_name, dir_fd=parent_descriptor, follow_symlinks=False),)): + raise ReplayRunError("replay staging cleanup did not unlink the verified inode") + try: + replacement = os.stat(name, dir_fd=parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + return + if stat.S_ISDIR(replacement.st_mode): + os.rmdir(name, dir_fd=parent_descriptor) + else: + os.unlink(name, dir_fd=parent_descriptor) + os.fsync(parent_descriptor) + + +def run_replay( + plan: ReplayPlan, + *, + binding_inputs: Mapping[str, Any], + runtime_bindings: Mapping[str, Any], + lane_lock: Mapping[str, Any], + harness_lock: EffectiveHarnessLock, + workspace: BreadBoardWorkspace, + scenario: ReplayScenario, + provider: Any, + provider_client: Any, + provider_model: str, + provider_context: Any, + tools: Mapping[str, Any], + host: Any, + authorize: Callable[[str, Mapping[str, Any]], bool], + secret_bindings: Mapping[str, str] | None = None, + environment_allowlist: Sequence[str] = (), + runtime_version: str = "unknown", + model_revision: str | None = None, + clock: Clock | None = None, + ids: IdSource | None = None, + monotonic: Callable[[], float] = time.monotonic, + cancelled: Callable[[], bool] | None = None, +) -> ReplayRunResult: + if not isinstance(plan, ReplayPlan) or plan.mode != "execute": + raise ReplayPlanError("run_replay requires an execute ReplayPlan") + if sys.platform != "darwin" and not sys.platform.startswith("linux"): + raise ReplayRunError("replay execution supports Linux and macOS containment backends only") + if not isinstance(harness_lock, EffectiveHarnessLock): + raise ReplayRunError("harness_lock must be an EffectiveHarnessLock") + plan.verify_bindings(binding_inputs) + expected_bindings = scenario.binding_inputs({name: binding_inputs[name] for name in HASH_BINDING_NAMES if name not in {"scenario_sha256", "interaction_script_sha256", "initial_workspace_sha256", "initial_messages_sha256", "tool_schema_lock_sha256"}}) + if dict(binding_inputs) != expected_bindings: + raise ReplayPlanError("scenario content changed after plan creation") + plan_record = plan.as_dict(); lane_sha = lane_lock.get("lock_sha256"); harness_sha = harness_lock.as_dict().get("graph_hash") + runtime_binding_names = {"capability_probe_sha256", "model_policy_sha256", "normalizer_config_sha256", "comparator_config_sha256"} + if not isinstance(runtime_bindings, Mapping) or set(runtime_bindings) != runtime_binding_names: + raise ReplayRunError("runtime_bindings must provide the active capability, model policy, normalizer, and comparator configurations") + for binding_name in sorted(runtime_binding_names - {"model_policy_sha256"}): + if sha256_json(runtime_bindings[binding_name]) != plan_record["hash_bindings"][binding_name]: + raise ReplayPlanError(f"{binding_name} does not match the frozen replay plan") + schema_binding, evidence_validators = _replay_schema_registry() + if binding_inputs["schema_registry_sha256"] != schema_binding: + raise ReplayPlanError("schema registry does not match the frozen replay plan") + evidence_validators["plan"].validate(plan_record) + if lane_sha != plan_record["lane_lock_sha256"] or harness_sha != plan_record["harness_lock_sha256"]: + raise ReplayPlanError("lane or harness lock changed after plan creation") + if not isinstance(provider_model, str) or not provider_model or not isinstance(runtime_version, str) or not runtime_version: + raise ReplayRunError("provider model and runtime version must be populated") + if not callable(authorize): + raise ReplayRunError("authorize must be a callable policy gate") + if binding_inputs["host_platform_sha256"] != _host_platform_binding(): + raise ReplayPlanError("host platform does not match the frozen replay plan") + if not isinstance(tools, Mapping): + raise ReplayRunError("tools must be a mapping") + secret_bindings = {} if secret_bindings is None else secret_bindings + if not isinstance(secret_bindings, Mapping) or any(type(name) is not str or not name or type(value) is not str or not value for name, value in secret_bindings.items()): + raise ReplayRunError("secret_bindings must map plain populated reference names to plain populated values") + if not isinstance(environment_allowlist, Sequence) or isinstance(environment_allowlist, (str, bytes)) or any(type(name) is not str or not name for name in environment_allowlist) or len(environment_allowlist) != len(set(environment_allowlist)): + raise ReplayRunError("environment_allowlist must contain unique plain populated names") + environment_binding, worker_environment = _environment_binding(environment_allowlist, secret_bindings) + secret_values = tuple(sorted({*secret_bindings.values(), *worker_environment.values()}, key=lambda secret: (-len(secret), secret))) + if binding_inputs["environment_allowlist_sha256"] != environment_binding: + raise ReplayPlanError("environment values do not match the frozen replay plan") + + provider_tools, reverse_tool_aliases = _provider_tool_wire_schemas(scenario.tool_schemas) + isolated_provider_context = _provider_context_with_wire_tool_aliases( + _provider_context_snapshot(provider_context), reverse_tool_aliases + ) + if sha256_json(_model_policy_binding(runtime_bindings["model_policy_sha256"], isolated_provider_context)) != plan_record["hash_bindings"]["model_policy_sha256"]: + raise ReplayPlanError("model_policy_sha256 does not match the frozen replay plan") + active_clock, active_ids = clock or SystemClock(), ids or UUIDSource() + execution_id = "replay_execution." + _portable_id(active_ids.new_id(), "id_source execution value") + fresh_nonce = _portable_id(active_ids.new_id(), "fresh_nonce") + started_at, overall_start = active_clock.now(), monotonic() + replay_descriptors: tuple[int, ...] = () + stage_descriptor: int | None = None + stage_created = False + try: + _, replay_descriptors = _anchored_replay_parent(workspace) + if not replay_descriptors: + raise ReplayRunError("descriptor-anchored replay storage is unavailable on this platform") + replay_parent_descriptor = replay_descriptors[-1] + replay_parent = _staging_descriptor_path(replay_parent_descriptor) + stage_name, final_name = f".{execution_id}.staging", execution_id + for candidate_name in (stage_name, final_name): + try: + os.stat(candidate_name, dir_fd=replay_parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + continue + raise ReplayRunError(f"replay execution identity already exists: {execution_id}") + os.mkdir(stage_name, mode=0o700, dir_fd=replay_parent_descriptor) + stage_created = True + stage_descriptor = os.open(stage_name, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=replay_parent_descriptor) + stage_metadata = os.fstat(stage_descriptor) + stage_identity = (stage_metadata.st_dev, stage_metadata.st_ino) + stage, final = _staging_descriptor_path(stage_descriptor), replay_parent / final_name + except BaseException: + try: + if stage_descriptor is not None: + failed_metadata = os.fstat(stage_descriptor) + _remove_tree_verified(replay_parent_descriptor, stage_name, (failed_metadata.st_dev, failed_metadata.st_ino), stage_descriptor) + os.close(stage_descriptor) + elif stage_created: + _remove_entry_at(replay_parent_descriptor, stage_name) + finally: + _close_descriptors(replay_descriptors) + raise + artifact_descriptors: tuple[int, ...] = () + fresh_workspace_descriptor: int | None = None + try: + fresh_workspace_descriptor = _materialize_initial(stage_descriptor, "workspace", scenario.initial_files) + fresh_workspace_metadata = os.fstat(fresh_workspace_descriptor) + fresh_workspace_identity = (fresh_workspace_metadata.st_dev, fresh_workspace_metadata.st_ino) + fresh_workspace = _staging_descriptor_path(fresh_workspace_descriptor) + before = _snapshot(fresh_workspace, fresh_workspace_identity, fresh_workspace_descriptor) + session_name = "session.events.jsonl" + session = Session.start(harness_lock, scenario.task, session_id=execution_id, clock=active_clock, sink=_AnchoredJsonlEventSink(stage_descriptor, session_name)) + store, artifact_descriptors = _anchored_artifact_store(workspace); created: set[ArtifactRef] = set(); artifacts: list[tuple[str, ArtifactRef, str | None, str]] = [] + provider_rows: list[dict[str, Any]] = []; tool_rows: list[dict[str, str]] = []; pending_provider_metadata: dict[str, Any] = {} + messages = [dict(row) for row in (*scenario.initial_messages, *scenario.interaction_script)] + redaction_count, terminal_status, completion_reason, failure = [0], "completed", "provider completed without tool calls", None + policy_state = {"capability": "allowed", "policy": "allowed", "approval": "not_required"}; provider_calls = tool_calls = 0 + integrity_verified = True + except BaseException: + try: + if fresh_workspace_descriptor is not None: + os.close(fresh_workspace_descriptor) + fresh_workspace_descriptor = None + _close_descriptors(artifact_descriptors) + _remove_tree_verified(replay_parent_descriptor, stage_name, stage_identity, stage_descriptor) + finally: + os.close(stage_descriptor) + _close_descriptors(replay_descriptors) + raise + + def put(role: str, value: Any, schema_id: str | None, sensitivity: str = "internal") -> ArtifactRef: + redacted = _redact(value, secrets=secret_values, workspace=workspace.root, counter=redaction_count) + ref = store.put_json(redacted, created=created); artifacts.append((role, ref, schema_id, sensitivity)); return ref + + try: + with store.transaction(rollback_created=created): + try: + host_identity = _host_identity(host); provider_id, runtime_id, endpoint_class = _provider_identity(provider) + declared_names = _declared_tool_names(scenario.tool_schemas) + expected_executor_names = declared_names - {"host.execute"} + if set(tools) != expected_executor_names: + raise ReplayPlanError("runtime tools do not match the frozen scenario toolset") + client_binding = _provider_client_binding(provider, provider_client, secret_bindings) + route_binding = _provider_route_binding(provider_id, runtime_id, endpoint_class, provider_model, model_revision, runtime_version, provider, client_binding) + if binding_inputs["provider_route_lock_sha256"] != route_binding: + raise ReplayPlanError("provider runtime does not match the frozen provider route") + host_worker_payload = _tool_executor_envelope(host, "host") + tool_worker_payloads = { + name: _tool_executor_envelope(tools[name], "tool") + for name in sorted(tools) + } + provider_capability = _provider_capability(provider, provider_client, provider_model, isolated_provider_context, worker_environment, secret_bindings, client_binding) + provider_worker_payload = _tool_executor_envelope(provider_capability, "provider") + policy_capability = _policy_capability(authorize) + policy_worker_payload = _tool_executor_envelope(policy_capability, "policy") + executor_binding = {"executors": {name: _capability_runtime_identity(tools[name], "tool", payload=tool_worker_payloads[name]) for name in sorted(tools)}} + if binding_inputs["tool_executor_identity_sha256"] != executor_binding: + raise ReplayPlanError("tool executors do not match the frozen runtime identities") + host_binding = {"driver_type": _capability_runtime_identity(host, "host", payload=host_worker_payload), "workspace_identity": host_identity, "process_containment": _host_containment_identity(host)} + if binding_inputs["host_driver_identity_sha256"] != host_binding: + raise ReplayPlanError("host driver does not match the frozen runtime identity") + policy_binding = {"callable": _callable_identity(authorize)} + if binding_inputs["operation_policy_sha256"] != policy_binding: + raise ReplayPlanError("policy gate does not match the frozen runtime identity") + secret_binding = {"references": sorted(secret_bindings)} + if binding_inputs["secret_references_sha256"] != secret_binding: + raise ReplayPlanError("resolved secrets do not match the frozen secret references") + tool_argument_validators = _tool_argument_validators(scenario.tool_schemas) + usage_accountant = _UsageAccountant.from_budgets(plan_record["budgets"]) + before_ref = put("workspace_before", before, None) + provider_worker = None + host_worker = None + policy_worker = None + tool_workers: dict[str, _ExecToolWorker] = {} + budgets, deadlines = plan_record["budgets"], plan_record["deadlines_ms"] + last_activity = overall_start + + def cancellation_state(_elapsed_ms: int = 0) -> tuple[str, str] | None: + if cancelled is None or not cancelled(): + return None + return "replay.cancelled", "replay was cancelled" + + while True: + turn_start = monotonic() + cancellation = cancellation_state(_duration_ms(last_activity, turn_start)) + if cancellation is not None: + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) + if _duration_ms(overall_start, turn_start) > deadlines["total"]: + raise _RuntimeFailure("timed_out", "replay.total_timeout", "replay exceeded its total deadline") + if _duration_ms(last_activity, turn_start) > deadlines["idle"]: + raise _RuntimeFailure("timed_out", "replay.idle_timeout", "replay exceeded its idle deadline") + if provider_calls >= budgets["provider_calls"] or provider_calls >= budgets["turns"]: + raise _RuntimeFailure("budget_exhausted", "replay.provider_budget", "replay exhausted its provider or turn budget") + provider_calls += 1; request_id = f"{execution_id}:request:{provider_calls}"; attempt_id = f"{request_id}:attempt:1" + provider_messages = _provider_wire_messages(messages, reverse_tool_aliases) + request = {"model": provider_model, "messages": provider_messages, "tools": provider_tools, "stream": False, "context": _provider_context_request_payload(isolated_provider_context)} + request_payload_sha256 = sha256_json(request) + request_ref = put("provider_request", request, None, "secret_redacted") + call_start, call_started = monotonic(), active_clock.now() + usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cost_amount": 0, "cost_currency": "USD"} + prepared_calls: list[tuple[ProviderToolCall, dict[str, Any]]] = [] + response_ref: ArtifactRef | None = None + response_payload_sha256: str | None = None + try: + total_remaining = max(1, deadlines["total"] - _duration_ms(overall_start, call_start)) + idle_remaining = max(1, deadlines["idle"] - _duration_ms(last_activity, call_start)) + call_limit = min(deadlines["provider_call"], total_remaining, idle_remaining) + if idle_remaining == call_limit: + timeout_code, timeout_detail = "replay.idle_timeout", "provider call exceeded the idle deadline" + elif total_remaining == call_limit: + timeout_code, timeout_detail = "replay.total_timeout", "replay exceeded its total deadline" + else: + timeout_code, timeout_detail = "replay.provider_timeout", "provider call exceeded its deadline" + if provider_worker is None: + provider_worker = _ExecToolWorker( + provider_capability, + fresh_workspace_descriptor, + capability_kind="provider", + payload=provider_worker_payload, + startup_timeout_ms=call_limit, + startup_timeout_code=timeout_code, + startup_timeout_detail=timeout_detail, + cancelled=cancelled, + ) + provider_ready = monotonic() + cancellation = cancellation_state(_duration_ms(last_activity, provider_ready)) + if cancellation is not None: + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) + if _duration_ms(overall_start, provider_ready) > deadlines["total"]: + raise _RuntimeFailure("timed_out", "replay.total_timeout", "replay exceeded its total deadline") + if _duration_ms(last_activity, provider_ready) > deadlines["idle"]: + raise _RuntimeFailure("timed_out", "replay.idle_timeout", "provider call exceeded the idle deadline") + provider_elapsed = _duration_ms(call_start, provider_ready) + if provider_elapsed > deadlines["provider_call"]: + raise _RuntimeFailure("timed_out", "replay.provider_timeout", "provider call exceeded its deadline") + total_remaining = max(1, deadlines["total"] - _duration_ms(overall_start, provider_ready)) + idle_remaining = max(1, deadlines["idle"] - _duration_ms(last_activity, provider_ready)) + provider_remaining = max(1, deadlines["provider_call"] - provider_elapsed) + call_limit = min(provider_remaining, total_remaining, idle_remaining) + if idle_remaining == call_limit: + timeout_code, timeout_detail = "replay.idle_timeout", "provider call exceeded the idle deadline" + elif total_remaining == call_limit: + timeout_code, timeout_detail = "replay.total_timeout", "replay exceeded its total deadline" + else: + timeout_code, timeout_detail = "replay.provider_timeout", "provider call exceeded its deadline" + raw_result = provider_worker.invoke({"messages": provider_messages, "tools": provider_tools}, timeout_ms=call_limit, timeout_code=timeout_code, timeout_detail=timeout_detail, cancelled=cancelled, cancellation_grace_ms=plan_record["cancellation_grace_ms"]) + result = _validate_replay_provider_result(raw_result, request_id) + evidence = provider_result_evidence(result) + response_payload_sha256 = sha256_json(evidence) + usage = normalized_provider_usage(result) + response_ref = put("provider_response", evidence, None, "secret_redacted") + prepared_calls = [] + for message in result.messages: + for call in message.tool_calls: + canonical_call = ProviderToolCall(call.id, reverse_tool_aliases.get(call.name, call.name), call.arguments, call.type, call.raw) + prepared_calls.append((canonical_call, _validated_arguments(canonical_call, tool_argument_validators))) + finish_reason = next((message.finish_reason for message in result.messages if message.finish_reason), "completed") + exchange_problem, exchange_status = None, "completed" + except _RuntimeFailure as exc: + response_ref = put("provider_error", {"error_type": type(exc).__name__, "message": exc.detail}, None, "secret_redacted") + finish_reason, exchange_status = None, exc.status + exchange_problem = problem(exc.error_code, exc.detail, record_refs=(response_ref.digest,)) + except _InvalidProviderResponse as exc: + response_ref = response_ref if response_ref is not None else put("provider_invalid_response", {"error_type": type(exc).__name__, "message": "provider returned invalid tool arguments"}, None, "secret_redacted") + finish_reason, exchange_status = None, "invalid_response"; exchange_problem = problem("replay.invalid_provider_response", "provider returned invalid tool arguments", record_refs=(response_ref.digest,)) + except Exception as exc: + response_ref = put("provider_error", {"error_type": type(exc).__name__, "message": "provider invocation failed"}, None, "secret_redacted") + finish_reason, exchange_status = None, "provider_error"; exchange_problem = problem("replay.provider_failed", "provider invocation failed", record_refs=(response_ref.digest,)) + call_end = monotonic(); call_duration = _duration_ms(call_start, call_end); last_activity = call_end + cancellation = None + if exchange_status == "completed": + cancellation = cancellation_state(call_duration) + if cancellation is not None: + finish_reason, exchange_status = None, "cancelled" + exchange_problem = problem(cancellation[0], cancellation[1], record_refs=(response_ref.digest,)) + elif _duration_ms(overall_start, call_end) > deadlines["total"]: + finish_reason, exchange_status = None, "timed_out" + exchange_problem = problem("replay.total_timeout", "replay exceeded its total deadline", record_refs=(response_ref.digest,)) + elif call_duration > deadlines["provider_call"]: + finish_reason, exchange_status = None, "timed_out" + exchange_problem = problem("replay.provider_timeout", "provider call exceeded its deadline", record_refs=(response_ref.digest,)) + exchange = { + "schema_version": "bb.provider_exchange.v2", "exchange_id": f"{execution_id}:exchange:{provider_calls}", "execution_id": execution_id, + "attempt_id": attempt_id, "request_id": request_id, "route_lock_sha256": plan_record["hash_bindings"]["provider_route_lock_sha256"], + "provider_family": provider_id, "runtime_id": runtime_id, "runtime_version": runtime_version, "endpoint_class": endpoint_class, + "model_id": provider_model, "model_revision": model_revision, "started_at_utc": call_started, "completed_at_utc": active_clock.now(), + "duration_ms": call_duration, "status": exchange_status, "request_payload_sha256": request_payload_sha256, + "response_payload_sha256": response_payload_sha256 if exchange_status == "completed" else None, "finish_reason": finish_reason, + "usage": usage, "evidence_refs": [request_ref.digest, response_ref.digest], "fallback_used": False, "problem": exchange_problem, + } + exchange = validate_provider_exchange(exchange) + evidence_validators["exchange"].validate(exchange) + exchange_ref = put("provider_exchange", exchange, "bb.provider_exchange.v2", "secret_redacted") + provider_rows.append({"ref": exchange_ref.digest, "sha256": exchange_ref.digest, "request_id": request_id, "attempt_id": attempt_id, "normalized_usage": usage, "status": exchange_status}) + if exchange_status != "completed": + if exchange_status == "cancelled": + if prepared_calls: + policy_state["policy"] = "denied" + raise _RuntimeFailure("cancelled", exchange_problem["error_code"], exchange_problem["message"]) + if exchange_status == "timed_out": + error_code = exchange_problem["error_code"] if exchange_problem is not None else "replay.provider_timeout" + raise _RuntimeFailure("timed_out", error_code, exchange_problem["message"] if exchange_problem is not None else "provider call exceeded its deadline") + if exchange_status == "invalid_response": + raise _RuntimeFailure("provider_failed", "replay.invalid_provider_response", "provider returned invalid tool arguments") + raise _RuntimeFailure("provider_failed", "replay.provider_failed", "provider invocation failed") + usage_budget_exceeded = usage_accountant.add(usage) + pending_provider_metadata.update(evidence["metadata"]) + session_state = isolated_provider_context.session_state + for metadata_name, metadata_value in evidence["metadata"].items(): + session_state.set_provider_metadata(metadata_name, metadata_value) + if usage_budget_exceeded: + raise _RuntimeFailure("budget_exhausted", "replay.usage_budget", "replay exceeded its token or cost budget") + session.input(f"provider turn {provider_calls}", attachments=(response_ref,)); messages.extend(_assistant_messages(result, reverse_tool_aliases)) + calls = prepared_calls + if not calls: + break + for call, arguments in calls: + if tool_calls >= budgets["tool_calls"]: + raise _RuntimeFailure("budget_exhausted", "replay.tool_budget", "replay exhausted its tool budget") + tool_calls += 1 + if not isinstance(call.name, str) or not call.name: + raise _RuntimeFailure("provider_failed", "replay.invalid_tool_call", "provider emitted a tool call without a name") + if call.name not in declared_names: + policy_state["capability"] = "denied" + raise _RuntimeFailure("policy_denied", "replay.undeclared_tool", "provider requested a tool outside the frozen toolset") + policy_start = monotonic() + cancellation = cancellation_state(_duration_ms(last_activity, policy_start)) + if cancellation is not None: + policy_state["policy"] = "denied" + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) + if _duration_ms(overall_start, policy_start) > deadlines["total"]: + policy_state["policy"] = "denied" + raise _RuntimeFailure("timed_out", "replay.total_timeout", "replay exceeded its total deadline") + if _duration_ms(last_activity, policy_start) > deadlines["idle"]: + policy_state["policy"] = "denied" + raise _RuntimeFailure("timed_out", "replay.idle_timeout", "replay exceeded its idle deadline") + policy_total_remaining = max(1, deadlines["total"] - _duration_ms(overall_start, policy_start)) + policy_idle_remaining = max(1, deadlines["idle"] - _duration_ms(last_activity, policy_start)) + policy_limit = min(deadlines["tool_call"], policy_total_remaining, policy_idle_remaining) + if policy_idle_remaining == policy_limit: + policy_timeout_code, policy_timeout_detail = "replay.idle_timeout", "policy evaluation exceeded the idle deadline" + elif policy_total_remaining == policy_limit: + policy_timeout_code, policy_timeout_detail = "replay.total_timeout", "replay exceeded its total deadline" + else: + policy_timeout_code, policy_timeout_detail = "replay.policy_timeout", "policy evaluation exceeded its deadline" + try: + if policy_worker is None: + policy_worker = _ExecToolWorker( + policy_capability, + fresh_workspace_descriptor, + capability_kind="policy", + payload=policy_worker_payload, + startup_timeout_ms=policy_limit, + startup_timeout_code=policy_timeout_code, + startup_timeout_detail=policy_timeout_detail, + cancelled=cancelled, + ) + policy_ready = monotonic() + cancellation = cancellation_state(_duration_ms(last_activity, policy_ready)) + if cancellation is not None: + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) + if _duration_ms(overall_start, policy_ready) > deadlines["total"]: + raise _RuntimeFailure("timed_out", "replay.total_timeout", "replay exceeded its total deadline") + if _duration_ms(last_activity, policy_ready) > deadlines["idle"]: + raise _RuntimeFailure("timed_out", "replay.idle_timeout", "policy evaluation exceeded the idle deadline") + policy_elapsed = _duration_ms(policy_start, policy_ready) + if policy_elapsed > deadlines["tool_call"]: + raise _RuntimeFailure("timed_out", "replay.policy_timeout", "policy evaluation exceeded its deadline") + policy_total_remaining = max(1, deadlines["total"] - _duration_ms(overall_start, policy_ready)) + policy_idle_remaining = max(1, deadlines["idle"] - _duration_ms(last_activity, policy_ready)) + policy_call_remaining = max(1, deadlines["tool_call"] - policy_elapsed) + policy_limit = min(policy_call_remaining, policy_total_remaining, policy_idle_remaining) + if policy_idle_remaining == policy_limit: + policy_timeout_code, policy_timeout_detail = "replay.idle_timeout", "policy evaluation exceeded the idle deadline" + elif policy_total_remaining == policy_limit: + policy_timeout_code, policy_timeout_detail = "replay.total_timeout", "replay exceeded its total deadline" + else: + policy_timeout_code, policy_timeout_detail = "replay.policy_timeout", "policy evaluation exceeded its deadline" + allowed = policy_worker.invoke({"name": call.name, "arguments": arguments}, timeout_ms=policy_limit, timeout_code=policy_timeout_code, timeout_detail=policy_timeout_detail, cancelled=cancelled, cancellation_grace_ms=plan_record["cancellation_grace_ms"]) + except _RuntimeFailure: + policy_state["policy"] = "denied" + raise + except Exception as exc: + policy_state["policy"] = "denied" + raise _RuntimeFailure("policy_denied", "replay.policy_error", "policy evaluation failed for a tool call") from exc + policy_end = monotonic() + cancellation = cancellation_state(_duration_ms(last_activity, policy_end)) + if cancellation is not None: + policy_state["policy"] = "denied" + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) + if _duration_ms(overall_start, policy_end) > deadlines["total"]: + policy_state["policy"] = "denied" + raise _RuntimeFailure("timed_out", "replay.total_timeout", "replay exceeded its total deadline") + if _duration_ms(last_activity, policy_end) > deadlines["idle"]: + policy_state["policy"] = "denied" + raise _RuntimeFailure("timed_out", "replay.idle_timeout", "replay exceeded its idle deadline") + if _duration_ms(policy_start, policy_end) > deadlines["tool_call"]: + policy_state["policy"] = "denied" + raise _RuntimeFailure("timed_out", "replay.policy_timeout", "policy evaluation exceeded its deadline") + if allowed is not True: + policy_state["policy"] = "denied" + raise _RuntimeFailure("policy_denied", "replay.policy_denied", "policy denied a tool call") + tool_start = monotonic() + cancellation = cancellation_state(_duration_ms(last_activity, tool_start)) + if cancellation is not None: + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) + if _duration_ms(overall_start, tool_start) > deadlines["total"]: + raise _RuntimeFailure("timed_out", "replay.total_timeout", "replay exceeded its total deadline") + if _duration_ms(last_activity, tool_start) > deadlines["idle"]: + raise _RuntimeFailure("timed_out", "replay.idle_timeout", "replay exceeded its idle deadline") + total_remaining = max(1, deadlines["total"] - _duration_ms(overall_start, tool_start)) + idle_remaining = max(1, deadlines["idle"] - _duration_ms(last_activity, tool_start)) + call_limit = min(deadlines["tool_call"], total_remaining, idle_remaining) + if idle_remaining == call_limit: + timeout_code, timeout_detail = "replay.idle_timeout", "tool call exceeded the idle deadline" + elif total_remaining == call_limit: + timeout_code, timeout_detail = "replay.total_timeout", "replay exceeded its total deadline" + else: + timeout_code, timeout_detail = "replay.tool_timeout", "tool call exceeded its deadline" + try: + started_worker = False + if call.name == "host.execute" and host_worker is None: + host_worker = _ExecToolWorker( + host, + fresh_workspace_descriptor, + capability_kind="host", + payload=host_worker_payload, + startup_timeout_ms=call_limit, + startup_timeout_code=timeout_code, + startup_timeout_detail=timeout_detail, + cancelled=cancelled, + ) + started_worker = True + elif call.name != "host.execute" and call.name not in tool_workers: + tool_workers[call.name] = _ExecToolWorker( + tools[call.name], + fresh_workspace_descriptor, + payload=tool_worker_payloads[call.name], + startup_timeout_ms=call_limit, + startup_timeout_code=timeout_code, + startup_timeout_detail=timeout_detail, + cancelled=cancelled, + ) + started_worker = True + if started_worker: + worker_ready = monotonic() + cancellation = cancellation_state(_duration_ms(last_activity, worker_ready)) + if cancellation is not None: + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) + if _duration_ms(overall_start, worker_ready) > deadlines["total"]: + raise _RuntimeFailure("timed_out", "replay.total_timeout", "replay exceeded its total deadline") + if _duration_ms(last_activity, worker_ready) > deadlines["idle"]: + raise _RuntimeFailure("timed_out", "replay.idle_timeout", "tool call exceeded the idle deadline") + tool_elapsed = _duration_ms(tool_start, worker_ready) + if tool_elapsed > deadlines["tool_call"]: + raise _RuntimeFailure("timed_out", "replay.tool_timeout", "tool call exceeded its deadline") + total_remaining = max(1, deadlines["total"] - _duration_ms(overall_start, worker_ready)) + idle_remaining = max(1, deadlines["idle"] - _duration_ms(last_activity, worker_ready)) + tool_remaining = max(1, deadlines["tool_call"] - tool_elapsed) + call_limit = min(tool_remaining, total_remaining, idle_remaining) + if idle_remaining == call_limit: + timeout_code, timeout_detail = "replay.idle_timeout", "tool call exceeded the idle deadline" + elif total_remaining == call_limit: + timeout_code, timeout_detail = "replay.total_timeout", "replay exceeded its total deadline" + else: + timeout_code, timeout_detail = "replay.tool_timeout", "tool call exceeded its deadline" + if call.name == "host.execute": + command = arguments.get("command") + if not isinstance(command, str) or not command: + raise ValueError("host.execute requires a command") + host_arguments = dict(arguments) + host_arguments["cwd"] = str(fresh_workspace) + outcome = host_worker.invoke(host_arguments, timeout_ms=call_limit, timeout_code=timeout_code, timeout_detail=timeout_detail, cancelled=cancelled, cancellation_grace_ms=plan_record["cancellation_grace_ms"]) + else: + executor = tools[call.name] + if not callable(getattr(executor, "execute", None)): + raise TypeError("authorized tool does not expose execute") + outcome = tool_workers[call.name].invoke({"arguments": arguments}, timeout_ms=call_limit, timeout_code=timeout_code, timeout_detail=timeout_detail, cancelled=cancelled, cancellation_grace_ms=plan_record["cancellation_grace_ms"]) + except Exception as exc: + tool_end = monotonic(); tool_duration = _duration_ms(tool_start, tool_end); last_activity = tool_end + cancellation = None; timed_out_code = None + if isinstance(exc, _RuntimeFailure) and exc.status == "cancelled": + cancellation = (exc.error_code, exc.detail); failure_status = "cancelled" + elif isinstance(exc, _RuntimeFailure) and exc.status == "timed_out": + failure_status, timed_out_code = "timed_out", exc.error_code + else: + cancellation = cancellation_state(tool_duration) + if cancellation is not None: + failure_status = "cancelled" + elif _duration_ms(overall_start, tool_end) > deadlines["total"]: + failure_status, timed_out_code = "timed_out", "replay.total_timeout" + elif tool_duration > deadlines["tool_call"]: + failure_status, timed_out_code = "timed_out", "replay.tool_timeout" + else: + failure_status = "failed" + error_ref = put("tool_outcome", {"schema_version": "bb.replay_tool_outcome.v1", "tool_call_id": call.id, "tool_name": call.name, "status": failure_status, "error_type": type(exc).__name__, "message": "tool execution failed"}, None, "secret_redacted") + tool_rows.append({"ref": error_ref.digest, "sha256": error_ref.digest}) + if cancellation is not None: + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) from exc + if timed_out_code is not None: + detail = "replay exceeded its total deadline" if timed_out_code == "replay.total_timeout" else "tool call exceeded its deadline" + raise _RuntimeFailure("timed_out", timed_out_code, detail) from exc + raise _RuntimeFailure("host_failed" if call.name == "host.execute" else "tool_failed", "replay.host_failed" if call.name == "host.execute" else "replay.tool_failed", "authorized tool call failed") from exc + try: + canonical_json(outcome) + except Exception as exc: + tool_end = monotonic(); tool_duration = _duration_ms(tool_start, tool_end); last_activity = tool_end + cancellation = cancellation_state(tool_duration) + timed_out_code = None + if cancellation is not None: + failure_status = "cancelled" + elif _duration_ms(overall_start, tool_end) > deadlines["total"]: + failure_status, timed_out_code = "timed_out", "replay.total_timeout" + elif tool_duration > deadlines["tool_call"]: + failure_status, timed_out_code = "timed_out", "replay.tool_timeout" + else: + failure_status = "failed" + error_ref = put("tool_outcome", {"schema_version": "bb.replay_tool_outcome.v1", "tool_call_id": call.id, "tool_name": call.name, "status": failure_status, "error_type": type(exc).__name__, "message": "tool returned a non-canonical result"}, None, "secret_redacted") + tool_rows.append({"ref": error_ref.digest, "sha256": error_ref.digest}) + if cancellation is not None: + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) from exc + if timed_out_code is not None: + detail = "replay exceeded its total deadline" if timed_out_code == "replay.total_timeout" else "tool call exceeded its deadline" + raise _RuntimeFailure("timed_out", timed_out_code, detail) from exc + raise _RuntimeFailure("host_failed" if call.name == "host.execute" else "tool_failed", "replay.host_invalid_result" if call.name == "host.execute" else "replay.tool_invalid_result", "authorized tool returned a non-canonical result") from exc + tool_end = monotonic(); tool_duration = _duration_ms(tool_start, tool_end); last_activity = tool_end + cancellation = cancellation_state(tool_duration) + timed_out_code = None + if cancellation is not None: + outcome_status = "cancelled" + elif _duration_ms(overall_start, tool_end) > deadlines["total"]: + outcome_status, timed_out_code = "timed_out", "replay.total_timeout" + elif tool_duration > deadlines["tool_call"]: + outcome_status, timed_out_code = "timed_out", "replay.tool_timeout" + else: + outcome_status = "completed" + outcome_record = {"schema_version": "bb.replay_tool_outcome.v1", "tool_call_id": call.id, "tool_name": call.name, "status": outcome_status} + if outcome_status == "completed": + outcome_record["result"] = outcome + outcome_ref = put("tool_outcome", outcome_record, None, "secret_redacted"); tool_rows.append({"ref": outcome_ref.digest, "sha256": outcome_ref.digest}) + if cancellation is not None: + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) + if timed_out_code is not None: + detail = "replay exceeded its total deadline" if timed_out_code == "replay.total_timeout" else "tool call exceeded its deadline" + raise _RuntimeFailure("timed_out", timed_out_code, detail) + messages.append({"role": "tool", "tool_call_id": call.id, "content": json.dumps(outcome, sort_keys=True)}) + except _RuntimeFailure as exc: + safe_detail = _redact(exc.detail, secrets=secret_values, workspace=workspace.root, counter=redaction_count) + terminal_status, completion_reason, failure = exc.status, safe_detail, problem(exc.error_code, safe_detail) + if session.read_model.status not in ("completed", "failed", "canceled"): + session.cancel(safe_detail) if exc.status == "cancelled" else session.fail(exc.error_code, safe_detail) + finally: + for worker_name in ("tool_workers", "host_worker", "policy_worker", "provider_worker"): + worker_value = locals().get(worker_name) + if isinstance(worker_value, dict): + for worker in worker_value.values(): + worker.close() + elif worker_value is not None: + worker_value.close() + + def check_snapshot_deadline() -> None: + snapshot_time = monotonic() + cancellation = cancellation_state(_duration_ms(last_activity, snapshot_time)) + if cancellation is not None: + raise _RuntimeFailure("cancelled", cancellation[0], cancellation[1]) + if _duration_ms(overall_start, snapshot_time) > deadlines["total"]: + raise _RuntimeFailure("timed_out", "replay.total_timeout", "replay exceeded its total deadline") + + try: + after = _snapshot(fresh_workspace, fresh_workspace_identity, fresh_workspace_descriptor, check=check_snapshot_deadline) + except _RuntimeFailure as exc: + integrity_verified = False + safe_detail = _redact(exc.detail, secrets=secret_values, workspace=workspace.root, counter=redaction_count) + terminal_status, completion_reason, failure = exc.status, safe_detail, problem(exc.error_code, safe_detail) + if session.read_model.status not in ("completed", "failed", "canceled"): + session.cancel(safe_detail) if exc.status == "cancelled" else session.fail(exc.error_code, safe_detail) + after = {"schema_version": "bb.replay_workspace_snapshot.v1", "files": [], "snapshot_sha256": sha256_json([])} + if fresh_workspace_descriptor is not None: + os.close(fresh_workspace_descriptor) + fresh_workspace_descriptor = None + if terminal_status == "completed" and session.read_model.status == "running": + finalization_time = monotonic() + cancellation = cancellation_state(_duration_ms(last_activity, finalization_time)) + if cancellation is not None: + terminal_status, completion_reason, failure = "cancelled", cancellation[1], problem(cancellation[0], cancellation[1]) + session.cancel(cancellation[1]) + elif _duration_ms(overall_start, finalization_time) > deadlines["total"]: + completion_reason = "replay exceeded its total deadline" + terminal_status, failure = "timed_out", problem("replay.total_timeout", completion_reason) + session.fail("replay.total_timeout", completion_reason) + if terminal_status == "completed" and session.read_model.status == "running": + session.complete(completion_reason) + after_ref = put("workspace_after", after, None); diff_ref = put("workspace_diff", _workspace_diff(before if "before" in locals() else {"files": []}, after), None) + _verify_staging_path(stage, stage_identity, stage_descriptor) + for child_name in os.listdir(stage_descriptor): + if child_name == session_name: + continue + _remove_entry_at(stage_descriptor, child_name) + cleanup = ["fresh_workspace_removed", "staging_siblings_removed"] + redacted_events = _redact(AnchoredStorage.read_at(stage_descriptor, session_name).decode("utf-8"), secrets=secret_values, workspace=workspace.root, counter=redaction_count) + event_ref = store.put(str(redacted_events).encode("utf-8"), media_type="application/x-ndjson", created=created); artifacts.append(("kernel_event_stream", event_ref, None, "secret_redacted")) + os.unlink(session_name, dir_fd=stage_descriptor) + os.fsync(stage_descriptor) + cleanup.append("raw_session_log_removed") + policy_rows: list[dict[str, str]] = [] + for kind in ("policy", "approval", "capability"): + decision_ref = put("policy_decision", {"schema_version": "bb.replay_policy_decision.v1", "kind": kind, "decision": policy_state[kind], "host_identity_sha256": _digest(host_identity.encode("utf-8")) if "host_identity" in locals() else None}, None) + policy_rows.append({"kind": kind, "decision": policy_state[kind], "ref": decision_ref.digest, "sha256": decision_ref.digest}) + redaction_ref = put("redaction_report", {"schema_version": "bb.replay_redaction_report.v1", "redaction_count": redaction_count[0], "secret_values_stored": False, "absolute_workspace_paths_stored": False}, None, "secret_redacted") + completed_at = active_clock.now(); duration = _duration_ms(overall_start, monotonic()) + entries = [{ + "artifact_id": f"artifact:{index}", "role": role, "location_kind": "object_ref", "location": ref.digest, "media_type": ref.media_type, + "schema_id": schema_id, "size_bytes": ref.size_bytes, "sha256": ref.digest, "producer": "breadboard.product.evidence.replay_runner", + "sensitivity": sensitivity, "created_at_utc": completed_at, + } for index, (role, ref, schema_id, sensitivity) in enumerate(artifacts, 1)] + manifest_unsigned = {"schema_version": "bb.replay_artifact_manifest.v1", "source_record_id": execution_id, "publish_status": "complete" if integrity_verified else "quarantined", "created_at_utc": completed_at, "entries": entries, "integrity_verified": integrity_verified} + manifest_id = "replay_manifest:" + hashlib.sha256(canonical_json(manifest_unsigned)).hexdigest(); manifest = ReplayArtifactManifest.from_dict({"manifest_id": manifest_id, **manifest_unsigned}) + evidence_validators["manifest"].validate(manifest.as_dict()) + execution_record = { + "schema_version": "bb.replay_execution.v1", "execution_id": execution_id, "fresh_nonce": fresh_nonce, "mode": "execute", "plan_sha256": plan.sha256, + "lane_lock_sha256": plan_record["lane_lock_sha256"], "harness_lock_sha256": plan_record["harness_lock_sha256"], "started_at_utc": started_at, + "completed_at_utc": completed_at, "duration_ms": duration, "terminal_status": terminal_status, "completion_reason": completion_reason, + "kernel_event_stream": {"ref": event_ref.digest, "sha256": event_ref.digest}, "provider_exchanges": provider_rows, "tool_outcomes": tool_rows, + "workspace_before": {"ref": before_ref.digest if "before_ref" in locals() else after_ref.digest, "sha256": before_ref.digest if "before_ref" in locals() else after_ref.digest}, + "workspace_after": {"ref": after_ref.digest, "sha256": after_ref.digest}, "workspace_diff": {"ref": diff_ref.digest, "sha256": diff_ref.digest}, + "artifact_manifest_id": manifest_id, "policy_decisions": policy_rows, "cleanup_ledger": cleanup, + "nondeterminism_disclosures": ["host_platform", "provider_runtime", "wall_clock"], "redaction_report": {"ref": redaction_ref.digest, "sha256": redaction_ref.digest}, + "schema_validation_passed": True, "integrity_verified": integrity_verified, "claimable": False, "reuse_attestation_id": None, + "normalization_evidence_ids": [], "comparison_report_id": None, "problem": failure, + } + evidence_validators["execution"].validate(execution_record) + execution = ReplayExecution.from_dict(execution_record); execution.verify_plan(plan) + AnchoredStorage.write_at(stage_descriptor, "execution.json", canonical_json(execution.as_dict()) + b"\n") + AnchoredStorage.write_at(stage_descriptor, "manifest.json", canonical_json(manifest.as_dict()) + b"\n") + _verify_staging_path(stage, stage_identity, stage_descriptor) + publication_succeeded = False + _rename_no_replace(replay_parent / stage_name, final, parent_descriptor=replay_parent_descriptor) + publication_succeeded = True + os.fsync(replay_parent_descriptor) + final_stat = os.stat(final_name, dir_fd=replay_parent_descriptor, follow_symlinks=False) + if (final_stat.st_dev, final_stat.st_ino) != stage_identity: + raise ReplayRunError("published replay identity does not match the staging inode") + if terminal_status == "completed" and integrity_verified: + metadata_setter = getattr(getattr(provider_context, "session_state", None), "set_provider_metadata", None) + if callable(metadata_setter): + for metadata_name, metadata_value in pending_provider_metadata.items(): + metadata_setter(metadata_name, metadata_value) + published_final = _staging_descriptor_path(replay_parent_descriptor) / final_name + _close_descriptors(artifact_descriptors) + os.close(stage_descriptor) + _close_descriptors(replay_descriptors) + return ReplayRunResult(execution, published_final / "execution.json", manifest) + except BaseException as cause: + cleanup_errors = [] + for worker_name in ("tool_workers", "host_worker", "policy_worker", "provider_worker"): + worker_value = locals().get(worker_name) + try: + if isinstance(worker_value, dict): + for worker in worker_value.values(): + worker.close() + elif worker_value is not None: + worker_value.close() + except BaseException as exc: + cleanup_errors.append(exc) + published_tree_removed = False + if "publication_succeeded" in locals() and publication_succeeded: + try: + final_metadata = os.stat(final_name, dir_fd=replay_parent_descriptor, follow_symlinks=False) + except FileNotFoundError: + pass + except BaseException as exc: + cleanup_errors.append(exc) + else: + if (final_metadata.st_dev, final_metadata.st_ino) == stage_identity: + try: + _remove_tree_verified(replay_parent_descriptor, final_name, stage_identity, stage_descriptor) + published_tree_removed = True + except BaseException as exc: + cleanup_errors.append(exc) + if (final_metadata.st_dev, final_metadata.st_ino) != stage_identity: + try: + if stat.S_ISDIR(final_metadata.st_mode): + os.rmdir(final_name, dir_fd=replay_parent_descriptor) + else: + os.unlink(final_name, dir_fd=replay_parent_descriptor) + except BaseException as exc: + cleanup_errors.append(exc) + if fresh_workspace_descriptor is not None: + try: + os.close(fresh_workspace_descriptor) + fresh_workspace_descriptor = None + except BaseException as exc: + cleanup_errors.append(exc) + if not published_tree_removed: + try: + _remove_tree_verified(replay_parent_descriptor, stage_name, stage_identity, stage_descriptor) + except BaseException as exc: + cleanup_errors.append(exc) + try: + os.close(stage_descriptor) + except BaseException as exc: + cleanup_errors.append(exc) + try: + with store.transaction(): + for ref in tuple(created): + store.discard(ref) + except BaseException as exc: + cleanup_errors.append(exc) + try: + _close_descriptors(artifact_descriptors) + except BaseException as exc: + cleanup_errors.append(exc) + try: + _close_descriptors(replay_descriptors) + except BaseException as exc: + cleanup_errors.append(exc) + if cleanup_errors: + raise ReplayRunError("replay rollback cleanup failed") from cleanup_errors[0] + raise cause diff --git a/breadboard/product/integrations/host.py b/breadboard/product/integrations/host.py index fe8b33bd..56d4c4ff 100644 --- a/breadboard/product/integrations/host.py +++ b/breadboard/product/integrations/host.py @@ -10,6 +10,7 @@ class HostPort(Protocol): def get_workspace(self) -> str: ... def execute(self, command: str, **kwargs: Any) -> Any: ... + def replay_process_containment(self) -> Mapping[str, str]: ... class SandboxHostAdapter: @@ -25,8 +26,9 @@ def __init__( effects: Iterable[str] = ("filesystem", "process"), permissions: Iterable[str] = ("host.execute",), ) -> None: - if not host_id or not callable(getattr(sandbox, "get_workspace", None)) or not callable(getattr(sandbox, "execute", None)): - raise TypeError("host adapter requires a sandbox with get_workspace() and execute()") + required_methods = ("get_workspace", "execute", "replay_process_containment") + if not host_id or any(not callable(getattr(sandbox, name, None)) for name in required_methods): + raise TypeError("host adapter requires a sandbox with get_workspace(), execute(), and replay_process_containment()") self.host_id = host_id self.sandbox = sandbox self.descriptor = IntegrationDescriptor( @@ -52,9 +54,19 @@ def execute(self, command: str, **kwargs: Any) -> Any: raise RuntimeError("sandbox does not expose the frozen execute port") return method(command, **kwargs) + def replay_process_containment(self) -> Mapping[str, str]: + method = getattr(self.sandbox, "replay_process_containment", None) + if not callable(method): + raise RuntimeError("sandbox does not attest detached-descendant containment") + identity = method() + if not isinstance(identity, Mapping) or identity.get("detached_descendants") != "contained": + raise RuntimeError("sandbox process containment attestation is invalid") + return dict(identity) + def probe(self) -> ProbeReport: try: self.workspace() + self.replay_process_containment() except Exception as exc: return probe_for(self.descriptor, error=type(exc).__name__) return probe_for(self.descriptor) diff --git a/breadboard/product/integrations/provider.py b/breadboard/product/integrations/provider.py index f56e2a8c..ddcfb88b 100644 --- a/breadboard/product/integrations/provider.py +++ b/breadboard/product/integrations/provider.py @@ -1,5 +1,7 @@ """Provider runtime adapter for the frozen integration catalog port.""" from __future__ import annotations +import hashlib +import json from typing import Any, Mapping, Protocol, Sequence, runtime_checkable @@ -26,6 +28,35 @@ def invoke( ) -> ProviderResult: ... +def _client_state_value(value: Any) -> Any: + if value is None or type(value) in (bool, int, float, str): + json.dumps(value, allow_nan=False) + return value + if isinstance(value, Mapping): + if any(type(key) is not str for key in value): + raise TypeError("provider client state mappings require plain string keys") + return {key: {"type": f"{type(item).__module__}.{type(item).__qualname__}", "sha256": "sha256:" + hashlib.sha256(str(item).encode("utf-8")).hexdigest()} for key, item in sorted(value.items())} + if isinstance(value, (list, tuple)): + return [_client_state_value(item) for item in value] + return {"type": f"{type(value).__module__}.{type(value).__qualname__}", "value_sha256": "sha256:" + hashlib.sha256(str(value).encode("utf-8")).hexdigest()} + + +def _client_live_state(client: Any) -> str: + state = {} + for name in ("base_url", "timeout", "max_retries", "default_headers", "headers"): + if isinstance(client, Mapping): + if name not in client: + continue + value = client[name] + else: + value = getattr(client, name, None) + if value is None: + continue + state[name] = _client_state_value(value) + payload = json.dumps(state, allow_nan=False, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return "sha256:" + hashlib.sha256(payload).hexdigest() + + class ProviderRuntimeAdapter: """Expose an existing ProviderRuntime without changing provider registries.""" @@ -33,6 +64,8 @@ def __init__(self, runtime: ProviderRuntime, descriptor: ProviderDescriptor | No if not callable(getattr(runtime, "invoke", None)) or not callable(getattr(runtime, "create_client", None)): raise TypeError("runtime must implement the frozen provider port") self.runtime = runtime + self._client_identities: list[tuple[Any, dict[str, Any]]] = [] + self._client_replay_specs: list[tuple[Any, dict[str, Any]]] = [] self.port_descriptor = descriptor or getattr(runtime, "descriptor", None) if not isinstance(self.port_descriptor, ProviderDescriptor): raise TypeError("runtime must expose a ProviderDescriptor") @@ -57,7 +90,51 @@ def runtime_id(self) -> str: return self.port_descriptor.runtime_id def create_client(self, api_key: str, *, base_url: str | None = None, default_headers: Mapping[str, str] | None = None) -> Any: - return self.runtime.create_client(api_key, base_url=base_url, default_headers=dict(default_headers or {})) + headers = dict(default_headers or {}) + client = self.runtime.create_client(api_key, base_url=base_url, default_headers=headers) + self._client_identities.append((client, { + "credential_sha256": "sha256:" + hashlib.sha256(api_key.encode("utf-8")).hexdigest(), + "base_url": base_url, + "default_headers": {name: "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest() for name, value in sorted(headers.items())}, + "timeout": str(getattr(client, "timeout", None)), + "max_retries": getattr(client, "max_retries", None), + "live_state_sha256": _client_live_state(client), + })) + self._client_replay_specs.append((client, { + "api_key": api_key, + "base_url": base_url, + "default_headers": headers, + })) + return client + def replay_client_identity(self, client: Any, secret_bindings: Mapping[str, str]) -> dict[str, Any]: + reference = self.port_descriptor.api_key_env + expected = secret_bindings.get(reference) + actual = client.get("api_key") if isinstance(client, Mapping) else getattr(client, "api_key", None) + if callable(getattr(actual, "get_secret_value", None)): + actual = actual.get_secret_value() + creation = next((record for candidate, record in self._client_identities if candidate is client), None) + expected_digest = "sha256:" + hashlib.sha256(expected.encode("utf-8")).hexdigest() if isinstance(expected, str) else None + exposed_credential_matches = actual is None or (isinstance(actual, str) and actual == expected) + return { + "verified": creation is not None and creation["credential_sha256"] == expected_digest and creation["live_state_sha256"] == _client_live_state(client) and exposed_credential_matches, + "client_type": f"{type(client).__module__}.{type(client).__qualname__}", + "credential_reference": reference, + "creation_options": creation, + } + + def replay_worker_client_spec(self, client: Any, secret_bindings: Mapping[str, str]) -> dict[str, Any]: + spec = next((record for candidate, record in self._client_replay_specs if candidate is client), None) + expected = secret_bindings.get(self.port_descriptor.api_key_env) + if spec is None or spec["api_key"] != expected: + raise ValueError("provider client cannot be reconstructed from the frozen secret binding") + return dict(spec) + + def replay_worker_client(self, spec: Mapping[str, Any]) -> Any: + return self.create_client( + spec["api_key"], + base_url=spec.get("base_url"), + default_headers=spec.get("default_headers"), + ) def invoke(self, **kwargs: Any) -> ProviderResult: return self.runtime.invoke(**kwargs) diff --git a/breadboard/product/runtime/artifacts.py b/breadboard/product/runtime/artifacts.py index cf5ec589..a64390c4 100644 --- a/breadboard/product/runtime/artifacts.py +++ b/breadboard/product/runtime/artifacts.py @@ -78,7 +78,7 @@ def __init__(self, root: str | Path, *, descriptor: int | None = None) -> None: self._root_path, self._descriptor, self._transaction_depth, self._transaction_stream, self._transaction_owner = Path(root), descriptor, 0, None, None if descriptor is None: self._root.mkdir(parents=True, exist_ok=True) @contextmanager - def transaction(self) -> Iterator[None]: + def transaction(self, *, rollback_created: set[ArtifactRef] | None = None) -> Iterator[None]: with _CAS_LOCK: outer, stream = self._transaction_owner != threading.get_ident(), self._transaction_stream if outer: @@ -95,7 +95,13 @@ def transaction(self) -> Iterator[None]: except BaseException: stream.close(); self._transaction_stream = None; raise self._transaction_owner = threading.get_ident() self._transaction_depth += 1 - try: yield + try: + yield + except BaseException: + if rollback_created is not None: + for ref in tuple(rollback_created): self._discard(ref) + rollback_created.clear() + raise finally: self._transaction_depth -= 1 if outer: diff --git a/tests/product/evidence/test_replay_core.py b/tests/product/evidence/test_replay_core.py new file mode 100644 index 00000000..9d5f6b41 --- /dev/null +++ b/tests/product/evidence/test_replay_core.py @@ -0,0 +1,2392 @@ +from __future__ import annotations + +import sys +import importlib +import datetime +import enum +import functools +import hashlib +import json +import time +import types +import uuid +from decimal import Decimal +from pathlib import Path +from typing import Any + +import pytest +import yaml +from jsonschema import Draft202012Validator +from referencing import Registry, Resource + +from agentic_coder_prototype.logging.provider_dump import provider_dump_logger +from agentic_coder_prototype.provider.runtime import ProviderMessage, ProviderResult, ProviderRuntimeContext, ProviderRuntimeError, ProviderToolCall, normalized_provider_usage +from agentic_coder_prototype.provider.routing import ProviderDescriptor +from breadboard.product.evidence import ( + BreadBoardWorkspace, + ReplayExecution, + ReplayArtifactManifest, + ReplayExecutionError, + ReplayPlan, + ReplayPlanError, + ReplayRunError, + ReplayScenario, + build_replay_plan, + run_replay, +) +from breadboard.product.evidence.replay_plan import HASH_BINDING_NAMES, canonical_json +from breadboard.product.evidence.replay_execution import _validate_usage +import breadboard.product.evidence.replay_runner as replay_runner_module +from breadboard.product.evidence.replay_runner import _callable_identity, _capability_runtime_identity, _environment_binding, _host_containment_identity, _host_platform_binding, _provider_client_binding, _provider_route_binding, _redact, _replay_schema_registry, _runtime_type_identity +from breadboard.product.evidence.workspace import WorkspacePathError +from breadboard.product.harness.compile import compile_harness_definition +from breadboard.product.runtime.artifacts import ArtifactRef, ArtifactStore +from breadboard.product.integrations.tool import ToolIntegrationAdapter +from breadboard.product.integrations.host import SandboxHostAdapter +from breadboard.product.integrations.provider import ProviderRuntimeAdapter + +ROOT = Path(__file__).resolve().parents[3] +HASH = "sha256:" + "1" * 64 +_SPECIAL_BINDINGS = {"scenario_sha256", "interaction_script_sha256", "initial_workspace_sha256", "initial_messages_sha256", "tool_schema_lock_sha256"} + + +class _Clock: + def __init__(self) -> None: self.value = 0 + def now(self) -> str: + self.value += 1 + return f"2026-07-21T10:00:{self.value:02d}Z" + + +class _Ids: + def __init__(self) -> None: self.value = 0 + def new_id(self) -> str: + self.value += 1 + return f"id-{self.value}" + + +class _Monotonic: + def __init__(self) -> None: self.value = 0.0 + def __call__(self) -> float: + self.value += 0.001 + return self.value + +class _SlowMonotonic: + def __init__(self) -> None: self.value = 0.0 + def __call__(self) -> float: + self.value += 1 + return self.value + +class _CancelAfterProvider: + def __init__(self) -> None: self.polls = 0 + def __call__(self) -> bool: + self.polls += 1 + return self.polls > 1 + + + + +class _AddTool: + def __init__(self, *, fail: bool = False, invalid: bool = False, hang: bool = False) -> None: self.fail, self.invalid, self.hang = fail, invalid, hang + def execute(self, arguments: dict[str, Any]) -> dict[str, Any]: + if self.hang: time.sleep(1) + if self.fail: raise RuntimeError("tool failed with Authorization: Bearer UNLISTED") + if self.invalid: return {"value": object()} + return {"sum": arguments["a"] + arguments["b"], "token": "SECRET"} + +class _BoundToolOwner: + def __init__(self, offset: int) -> None: + self.offset = offset + + def run(self, arguments: Mapping[str, Any]) -> dict[str, Any]: + return {"sum": arguments["a"] + arguments["b"] + self.offset} + + +class _PrivateSlotTool: + __slots__ = ("__offset",) + def __init__(self, offset: int) -> None: + self.__offset = offset + def execute(self, arguments: Mapping[str, Any]) -> dict[str, Any]: + return {"sum": arguments["a"] + arguments["b"] + self.__offset} +class _RelativeWriteTool: + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + Path("relative-output.txt").write_text("contained", encoding="utf-8") + return {"written": True} +class _AbsoluteWriteTool: + def __init__(self, outside: Path) -> None: + self.outside = outside + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + self.outside.write_text("escaped", encoding="utf-8") + return {"escaped": True} +class _OutsideReadTool: + def __init__(self, outside: Path) -> None: + self.outside = outside + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + return {"content": self.outside.read_text(encoding="utf-8")} +class _InheritedFdReadTool: + def __init__(self, outside: Path) -> None: + self.descriptor = __import__("os").open(outside, __import__("os").O_RDONLY) + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + return {"content": __import__("os").read(self.descriptor, 4096).decode("utf-8")} +class _InheritedSocketReadTool: + def __init__(self) -> None: + import socket + self.reader, self.writer = socket.socketpair() + self.writer.sendall(b"INHERITED_SOCKET_CONTENT") + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + return {"content": self.reader.recv(4096).decode("utf-8")} + def close(self) -> None: + self.reader.close() + self.writer.close() + + +class _EnvironmentReadTool: + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + return {"value": __import__("os").environ.get("BB_REPLAY_ENV_SECRET")} + + +class _SecretSymlinkTool: + def __init__(self, secret: str) -> None: + self.secret = secret + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + Path(self.secret).symlink_to("input.txt") + return {"created": True} + + + + + + +class _StageFifoTool: + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + __import__("os").mkfifo(Path.cwd().parent / "untracked.fifo") + return {"created": True} + + + +class _NetworkTool: + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + import socket + connection = socket.socket() + try: + connection.connect(("127.0.0.1", 9)) + finally: + connection.close() + return {"connected": True} + + +class _PrintingTool: + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + print("Authorization: Bearer SECRET", flush=True) + return {"printed": True} + + +class _WorkspaceFifoTool: + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + __import__("os").mkfifo(Path.cwd() / "untracked.fifo") + return {"created": True} + + + +class _ChangingCwdTool: + def __init__(self, outside: Path) -> None: + self.outside = outside + self.calls = 0 + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + self.calls += 1 + if self.calls == 1: + __import__("os").chdir(self.outside) + else: + Path("after-cwd-change.txt").write_text("contained", encoding="utf-8") + return {"call": self.calls} + + + + + + +class _FrameInspectionTool: + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + import inspect + exposed = set() + frame = inspect.currentframe() + while frame is not None: + exposed.update(name for name in ("host", "authorize", "secret_bindings", "provider_client") if frame.f_locals.get(name) is not None) + frame = frame.f_back + return {"exposed": sorted(exposed)} + + +class _Host: + def __init__(self, *, fail: bool = False, replace: bool = False) -> None: self.fail, self.replace = fail, replace + def get_workspace(self) -> str: return "sandbox:fixture" + def replay_process_containment(self) -> dict[str, str]: return {"mechanism": "fixture-supervisor", "detached_descendants": "contained"} + def execute(self, command: str, **kwargs: Any) -> dict[str, Any]: + if self.fail: raise RuntimeError("host failed with Authorization: Bearer UNLISTED") + cwd = Path(kwargs["cwd"]) + if self.replace: + cwd.rename(cwd.parent / "workspace-original") + cwd.mkdir() + (cwd / "host.txt").write_text(command, encoding="utf-8") + return {"exit_code": 0, "cwd": kwargs["cwd"], "token": "SECRET"} +class _ArgumentCheckingHost(_Host): + def execute(self, command: str, **kwargs: Any) -> dict[str, Any]: + if kwargs.get("timeout") != 7 or kwargs.get("stdin_data") != "payload": + raise RuntimeError("approved host arguments were not forwarded") + return super().execute(command, **kwargs) + + +class _SubprocessHost(_Host): + def execute(self, command: str, **kwargs: Any) -> dict[str, Any]: + import subprocess + completed = subprocess.run(["/usr/bin/touch", "host-subprocess.txt"], cwd=kwargs["cwd"], check=False, capture_output=True) + return {"exit_code": completed.returncode, "error": completed.stderr.decode("utf-8", "replace"), "command": command} + + +class _NetworkAttemptHost(_Host): + def execute(self, command: str, **kwargs: Any) -> dict[str, Any]: + connection = __import__("socket").socket() + connection.bind(("127.0.0.1", 0)) + connection.close() + return super().execute(command, **kwargs) + + +class _OutsideReadingSubprocessHost(_Host): + def __init__(self, outside: Path) -> None: + super().__init__() + self.outside = outside + + def execute(self, command: str, **kwargs: Any) -> dict[str, Any]: + import subprocess + completed = subprocess.run( + ["/bin/cat", str(self.outside)], + cwd=kwargs["cwd"], + check=False, + capture_output=True, + ) + return { + "exit_code": completed.returncode, + "output": completed.stdout.decode("utf-8", "replace"), + "error": completed.stderr.decode("utf-8", "replace"), + "command": command, + } + + + + + + +def _allow(_name: str, _arguments: Mapping[str, Any]) -> bool: + return True +def _deny(_name: str, _arguments: Mapping[str, Any]) -> bool: + return False +def _hang_policy(_name: str, _arguments: Mapping[str, Any]) -> bool: + time.sleep(1) + return True +def _add_callable(arguments: Mapping[str, Any]) -> dict[str, Any]: + return {"sum": arguments["a"] + arguments["b"], "token": "SECRET"} + + +def _prefixed_policy(expected: str, name: str, _arguments: Mapping[str, Any]) -> bool: + return expected == name + + +class _PolicyOwner: + def allow(self, _name: str, _arguments: Mapping[str, Any]) -> bool: + return True +class _PrefixedPolicyOwner: + def allow(self, expected: str, name: str, _arguments: Mapping[str, Any]) -> bool: + return expected == name + + + + +class _ForkTool: + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + __import__("os").fork() + return {"escaped": True} +class _ExecReplacementTool: + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + __import__("os").execve("/usr/bin/touch", ["touch", "exec-replacement.txt"], {}) + return {"escaped": True} + + + + +class _BadIds: + def new_id(self) -> str: + return "../../escaped" + + + + +class _IdentityProviderRuntime: + def create_client(self, api_key: str, *, base_url: str | None = None, default_headers: dict[str, str] | None = None) -> Any: + return types.SimpleNamespace(api_key=api_key, base_url=base_url, default_headers=default_headers or {}) + + def invoke(self, **kwargs: Any) -> ProviderResult: + raise AssertionError("not used") + + +class _Provider: + provider_id = "fixture" + runtime_id = "fixture_runtime" + descriptor = types.SimpleNamespace(provider_id="fixture", runtime_id="fixture_runtime", default_api_variant="fixture") + def __init__(self, sequence: tuple[str, ...]) -> None: self.sequence, self.calls, self.tools_seen, self.messages_seen = sequence, 0, None, None + def replay_client_identity(self, client: Any, secret_bindings: Mapping[str, str]) -> dict[str, Any]: + return {"verified": True, "client_type": f"{type(client).__module__}.{type(client).__qualname__}", "secret_references": sorted(secret_bindings)} + def invoke(self, **kwargs: Any) -> ProviderResult: + self.tools_seen = kwargs.get("tools") + self.messages_seen = [dict(row) for row in kwargs.get("messages", [])] + action = self.sequence[self.calls]; self.calls += 1 + if action == "provider_error": raise RuntimeError("provider failed with SECRET") + if action == "provider_secret_error": raise RuntimeError("Authorization: Bearer UNLISTED") + if action == "hang": time.sleep(1) + if action == "brief": time.sleep(0.005) + if action in {"add", "add_eur"}: message = ProviderMessage("assistant", None, [ProviderToolCall("call-add", "add", json.dumps({"a": 2, "b": 3}))], "tool_calls") + elif action == "add_without_id": message = ProviderMessage("assistant", None, [ProviderToolCall(None, "add", json.dumps({"a": 2, "b": 3}))], "tool_calls") + elif action == "two_adds": message = ProviderMessage("assistant", None, [ProviderToolCall("call-add-1", "add", '{"a":1,"b":2}'), ProviderToolCall("call-add-2", "add", '{"a":3,"b":4}')], "tool_calls") + elif action == "nan_args": message = ProviderMessage("assistant", None, [ProviderToolCall("call-nan", "add", '{"a":NaN,"b":2}')], "tool_calls") + elif action == "lazy": message = ProviderMessage("assistant", None, [ProviderToolCall("call-lazy", "lazy", "{}")], "tool_calls") + elif action in {"host", "host_args"}: + host_names = [tool["function"]["name"] for tool in kwargs["tools"] if tool["function"]["name"].startswith("host_execute")] + if len(host_names) != 1 or "." in host_names[0]: + raise RuntimeError("provider did not receive the canonical host tool through a valid wire alias") + arguments = {"command": "fixture", "timeout": 7, "stdin_data": "payload"} if action == "host_args" else {"command": "fixture"} + message = ProviderMessage("assistant", None, [ProviderToolCall("call-host", host_names[0], json.dumps(arguments))], "tool_calls") + elif action == "unknown": message = ProviderMessage("assistant", None, [ProviderToolCall("call-danger", "danger", "{}")], "tool_calls") + elif action == "invalid_message": message = ProviderMessage("assistant", "done", [], 7) # type: ignore[arg-type] + elif action == "environment_value": message = ProviderMessage("assistant", __import__("os").environ["BB_REPLAY_ENV_SECRET"], [], "stop") + elif action == "overlapping_environment_value": message = ProviderMessage("assistant", __import__("os").environ["BB_REPLAY_LONG_SECRET"], [], "stop") + elif action == "resolve_localhost": + __import__("socket").getaddrinfo("localhost", 443) + message = ProviderMessage("assistant", "resolved", [], "stop") + else: message = ProviderMessage("assistant", "done", [], "stop", annotations={"opaque": object()} if action == "opaque" else {}) + usage = {"input_tokens": 100, "output_tokens": 100, "total_tokens": 1} if action == "malformed_usage" else {"input_tokens": 1, "output_tokens": 1} + metadata = {"cost_currency": "EUR"} if action in {"eur", "add_eur"} else ({"Authorization": "Bearer UNLISTED", "Cookie": "sid=UNLISTED", "apiKey": "UNLISTED", "accessToken": "UNLISTED", "privateKey": "UNLISTED", "access_key": "UNLISTED", "session_id": "UNLISTED", "bearer": "UNLISTED"} if action == "credential_fields" else {"api_key": "SECRET"}) + return ProviderResult([message], {"raw": "must not be persisted"}, usage, model="fixture-model", metadata=metadata) +class _ReasoningProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + result = super().invoke(**kwargs) + result.encrypted_reasoning = [{"encrypted_content": "ciphertext", "metadata": {"response_id": "response-1"}}] + result.reasoning_summaries = ["reasoning summary"] + return result + + + + +class _LiveToolResultProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + if self.calls == 1: + live_result = json.loads(kwargs["messages"][-1]["content"]) + if live_result["token"] != "SECRET": + raise RuntimeError("live tool result was redacted") + return super().invoke(**kwargs) +class _WritingProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + Path("provider-bypass.txt").write_text("bypass", encoding="utf-8") + return super().invoke(**kwargs) +class _ReadingProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + Path("input.txt").read_text(encoding="utf-8") + return super().invoke(**kwargs) + + + + +class _DumpingProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + if provider_dump_logger.enabled or provider_dump_logger.log_dir is not None: + raise RuntimeError("provider dump logger remained enabled") + return super().invoke(**kwargs) + +class _StatefulProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + continuation = kwargs["context"].session_state.get_provider_metadata("custom_continuation") + if continuation is None: + message = ProviderMessage("assistant", None, [ProviderToolCall("call-add", "add", '{"a":2,"b":3}')], "tool_calls") + return ProviderResult([message], None, {"input_tokens": 1, "output_tokens": 1}, model="fixture-model", metadata={"custom_continuation": "response-1"}) + if continuation != "response-1": + raise RuntimeError("stale provider response state") + return ProviderResult([ProviderMessage("assistant", "done", [], "stop")], None, {"input_tokens": 1, "output_tokens": 1}, model="fixture-model", metadata={"custom_continuation": "response-2"}) +class _EnvironmentFactoryProvider(_Provider): + def replay_worker_client_spec(self, _client: Any, _secret_bindings: Mapping[str, str]) -> dict[str, Any]: + return {} + + def replay_worker_client(self, _spec: Mapping[str, Any]) -> Any: + if __import__("os").environ.get("BB_REPLAY_ENV_SECRET") != "ENVIRONMENT_ONLY_CREDENTIAL": + raise RuntimeError("provider client factory did not receive its allowlisted environment") + return object() +class _CollisionAliasProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + wire_names = [tool["function"]["name"] for tool in kwargs["tools"]] + if self.calls == 0: + if wire_names[0] != "a_b" or wire_names[1] == "a_b": + raise RuntimeError("collision aliases are not stable") + self.calls += 1 + return ProviderResult([ProviderMessage("assistant", None, [ProviderToolCall("collision-call", wire_names[0], '{"a":2,"b":3}')], "tool_calls")], None, {"input_tokens": 1, "output_tokens": 1}, model="fixture-model", metadata={}) + prior_name = kwargs["messages"][-2]["tool_calls"][0]["function"]["name"] + if prior_name != wire_names[0]: + raise RuntimeError("prior wire alias drifted across turns") + self.calls += 1 + return ProviderResult([ProviderMessage("assistant", "done", [], "stop")], None, {"input_tokens": 1, "output_tokens": 1}, model="fixture-model", metadata={}) + + +class _ForcedToolChoiceProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + tool_choice = kwargs["context"].agent_config["provider_tools"]["anthropic"]["tool_choice"] + if tool_choice != {"type": "tool", "name": "host_execute"}: + raise RuntimeError("forced tool choice did not follow the provider wire alias") + return super().invoke(**kwargs) + + + + +class _FlatForcedToolChoiceProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + tool_choice = kwargs["context"].agent_config["provider_tools"]["tool_choice"] + if tool_choice != {"type": "tool", "name": "host_execute"}: + raise RuntimeError("flat forced tool choice did not follow the provider wire alias") + return super().invoke(**kwargs) + + +class _MixedForcedToolChoiceProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + provider_tools = kwargs["context"].agent_config["provider_tools"] + expected = {"type": "tool", "name": "host_execute"} + if provider_tools["tool_choice"] != expected or provider_tools["anthropic"]["tool_choice"] != expected: + raise RuntimeError("mixed forced tool choices did not follow the provider wire alias") + return super().invoke(**kwargs) + + +class _SpecOnlyProvider(_Provider): + def replay_worker_client_spec(self, _client: Any, _secret_bindings: Mapping[str, str]) -> dict[str, Any]: + return {} + + +class _ToolWithProviderCapability: + def __init__(self) -> None: + self.provider = _Provider(("done",)) + + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + return {"should_not_run": True} +class _ToolWithPolicyCapability: + def __init__(self) -> None: + self.authorize = _allow + + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + return {"should_not_run": True} + + + + + + + + + +class _ExplosiveRaw: + def __init__(self, marker: Path) -> None: + self.marker = marker + def __reduce__(self): + return eval, (f"__import__('pathlib').Path({str(self.marker)!r}).write_text('owned')",) + + +class _RawProvider(_Provider): + def __init__(self, marker: Path) -> None: + super().__init__(("done",)) + self.marker = marker + def invoke(self, **kwargs: Any) -> ProviderResult: + result = super().invoke(**kwargs) + result.raw_response = _ExplosiveRaw(self.marker) + return result + + +class _ContextCapability: + def __init__(self, marker: Path) -> None: + self.marker = marker + + def execute(self) -> None: + self.marker.write_text("called", encoding="utf-8") + + +class _ContextInspectingProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + capability = kwargs["context"].extra.get("host") + if capability is not None: + capability.execute() + return super().invoke(**kwargs) + + +class _ProviderMetadataInspectingProvider(_Provider): + def invoke(self, **kwargs: Any) -> ProviderResult: + session_state = kwargs["context"].session_state + expected = { + "anthropic_rate_limits": {"tokens_remaining": 17}, + "conversation_id": "conversation-fixture", + "current_turn_index": 3, + "previous_response_id": "response-fixture", + } + if {name: session_state.get_provider_metadata(name) for name in expected} != expected: + raise RuntimeError("provider replay metadata was not preserved") + if session_state.get_provider_metadata("control_queue") is not None: + raise RuntimeError("runtime-only provider metadata escaped into replay") + return super().invoke(**kwargs) + + +class _CallableConfiguredTool: + def __init__(self, value: int) -> None: + self.value = value + + def __call__(self) -> int: + return self.value + + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, int]: + return {"value": self.value} + + +class _ScalarMode(enum.Flag): + READ = 1 + WRITE = 2 + + +class _ScalarConfiguredTool: + def __init__(self) -> None: + self.ratio = Decimal("0.700") + self.started_at = datetime.datetime(2026, 7, 22, 12, 34, 56, 789, tzinfo=datetime.timezone(datetime.timedelta(hours=-4), "EDT"), fold=1) + self.started_on = datetime.date(2026, 7, 22) + self.wake_at = datetime.time(8, 9, 10, 11, tzinfo=datetime.timezone.utc, fold=1) + self.duration = datetime.timedelta(days=2, seconds=3, microseconds=4) + self.zone = datetime.timezone(datetime.timedelta(hours=5, minutes=30), "IST") + self.run_id = uuid.UUID("12345678-1234-5678-1234-567812345678") + self.mode = _ScalarMode.READ | _ScalarMode.WRITE + + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + return { + "duration": [self.duration.days, self.duration.seconds, self.duration.microseconds], + "fold": self.started_at.fold, + "mode": self.mode.value, + "ratio": str(self.ratio), + "run_id": str(self.run_id), + "started_at": self.started_at.isoformat(), + "started_on": self.started_on.isoformat(), + "uuid_safety": self.run_id.is_safe.name, + "wake_at": self.wake_at.isoformat(), + "wake_fold": self.wake_at.fold, + "zone": self.zone.tzname(None), + } + + +class _UnsupportedScalarTool: + def __init__(self) -> None: + self.value = range(3) + + def execute(self, _arguments: Mapping[str, Any]) -> dict[str, Any]: + return {"value": list(self.value)} + + +class _MetadataSink: + def __init__(self, marker: Path) -> None: + self.marker = marker + + def set_provider_metadata(self, name: str, value: Any) -> None: + self.marker.write_text(json.dumps({name: value}, sort_keys=True), encoding="utf-8") + + +def _harness_lock(): + document = yaml.safe_load((ROOT / "agent_configs/templates/minimal_harness.v3.yaml").read_text(encoding="utf-8")) + return compile_harness_definition(document, source_ref="agent_configs/templates/minimal_harness.v3.yaml").lock + + +def _scenario(interaction_script: tuple[Mapping[str, Any], ...] = ()) -> ReplayScenario: + return ReplayScenario( + "fixture task", + ({"role": "user", "content": "go SECRET"},), + {"input.txt": "hello"}, + ( + {"type": "function", "function": {"name": "add", "parameters": {"type": "object"}}}, + {"type": "function", "function": {"name": "host.execute", "parameters": {"type": "object"}}}, + ), + interaction_script, + ) + + +def _plan(scenario: ReplayScenario, *, deadlines: dict[str, int] | None = None, provider_route: dict[str, Any] | None = None, budgets: dict[str, int | float] | None = None, authorize=_allow, provider_instance: _Provider | None = None, provider_client: Any = None, provider_context: Any = None, tool_instances: Mapping[str, Any] | None = None, host_instance: Any = None, environment_allowlist: tuple[str, ...] = ()): + lock = _harness_lock() + frozen = {name: {"binding": name} for name in HASH_BINDING_NAMES if name not in _SPECIAL_BINDINGS} + frozen["operation_policy_sha256"] = {"callable": _callable_identity(authorize)} + frozen["secret_references_sha256"] = {"references": ["fixture-secret"]} + frozen["environment_allowlist_sha256"] = _environment_binding(environment_allowlist, {"fixture-secret": "SECRET"})[0] + frozen["schema_registry_sha256"] = _replay_schema_registry()[0] + frozen["host_platform_sha256"] = _host_platform_binding() + active_provider = provider_instance or _Provider(("done",)) + active_client = object() if provider_client is None else provider_client + frozen["provider_route_lock_sha256"] = provider_route or _provider_route_binding("fixture", "fixture_runtime", "fixture", "fixture-model", None, "1.0", active_provider, _provider_client_binding(active_provider, active_client, {"fixture-secret": "SECRET"})) + declared_names = {row["function"]["name"] for row in scenario.tool_schemas} + active_tools = dict(tool_instances) if tool_instances is not None else ({"add": _AddTool()} if "add" in declared_names else {}) + active_host = host_instance or _Host() + frozen["tool_executor_identity_sha256"] = {"executors": {name: _capability_runtime_identity(active_tools[name], "tool") for name in sorted(active_tools)}} + frozen["host_driver_identity_sha256"] = {"driver_type": _capability_runtime_identity(active_host, "host"), "workspace_identity": "sandbox:fixture", "process_containment": _host_containment_identity(active_host)} + _, reverse_tool_aliases = replay_runner_module._provider_tool_wire_schemas(scenario.tool_schemas) + isolated_provider_context = replay_runner_module._provider_context_with_wire_tool_aliases( + replay_runner_module._provider_context_snapshot(provider_context if provider_context is not None else object()), + reverse_tool_aliases, + ) + frozen["model_policy_sha256"] = replay_runner_module._model_policy_binding( + {"binding": "model_policy_sha256"}, + isolated_provider_context, + ) + bindings = scenario.binding_inputs(frozen) + plan = build_replay_plan( + lane_lock_sha256=HASH, + harness_lock_sha256=lock.as_dict()["graph_hash"], + binding_inputs=bindings, + deadlines_ms=deadlines or {"total": 10_000, "idle": 1_000, "provider_call": 1_000, "tool_call": 1_000}, + budgets=budgets or {"turns": 5, "provider_calls": 5, "tool_calls": 5, "tokens": 100, "cost": 1}, + cancellation_grace_ms=100, + scenario_ref="scenario:fixture", + provider_route_ref="route:fixture", + operation_policy_ref="policy:fixture", + host_ref="host:fixture", + toolset_lock_ref="tools:fixture", + ) + return lock, bindings, plan + +def _run(tmp_path: Path, *, sequence: tuple[str, ...] = ("add", "host", "done"), tool_fail: bool = False, tool_invalid: bool = False, tool_hang: bool = False, host_fail: bool = False, host_replace: bool = False, host_instance: Any = None, authorize=None, deadlines: dict[str, int] | None = None, budgets: dict[str, int | float] | None = None, monotonic=None, cancelled=None, provider_route: dict[str, Any] | None = None, tool_schemas: tuple[dict[str, Any], ...] | None = None, interaction_script: tuple[Mapping[str, Any], ...] = (), provider_instance: _Provider | None = None, provider_context: Any = None, planned_provider_context: Any = None, tool_instances: Mapping[str, Any] | None = None, ids: Any = None, runtime_bindings: Mapping[str, Any] | None = None, environment_allowlist: tuple[str, ...] = ()): + workspace = BreadBoardWorkspace(tmp_path) + scenario = _scenario(interaction_script) if tool_schemas is None else ReplayScenario("fixture task", ({"role": "user", "content": "go SECRET"},), {"input.txt": "hello"}, tool_schemas, interaction_script) + active_authorize = authorize or _allow + active_provider = provider_instance or _Provider(sequence) + active_provider_context = provider_context if provider_context is not None else object() + active_client = object() + declared_names = {row["function"]["name"] for row in scenario.tool_schemas} + active_tools = dict(tool_instances) if tool_instances is not None else ({"add": _AddTool(fail=tool_fail, invalid=tool_invalid, hang=tool_hang)} if "add" in declared_names else {}) + active_host = host_instance or _Host(fail=host_fail, replace=host_replace) + lock, bindings, plan = _plan(scenario, deadlines=deadlines, provider_route=provider_route, budgets=budgets, authorize=active_authorize, provider_instance=active_provider, provider_client=active_client, provider_context=planned_provider_context if planned_provider_context is not None else active_provider_context, tool_instances=active_tools, host_instance=active_host, environment_allowlist=environment_allowlist) + result = run_replay( + plan, + binding_inputs=bindings, + runtime_bindings=runtime_bindings or {name: {"binding": name} for name in ("capability_probe_sha256", "model_policy_sha256", "normalizer_config_sha256", "comparator_config_sha256")}, + lane_lock={"lock_sha256": HASH}, + harness_lock=lock, + workspace=workspace, + scenario=scenario, + provider_client=active_client, + provider=active_provider, + provider_model="fixture-model", + provider_context=active_provider_context, + tools=active_tools, + host=active_host, + authorize=active_authorize, + secret_bindings={"fixture-secret": "SECRET"}, + environment_allowlist=environment_allowlist, + runtime_version="1.0", + ids=ids or _Ids(), + monotonic=monotonic or _Monotonic(), + cancelled=cancelled, + ) + return workspace, plan, result + + +def _validator(name: str) -> Draft202012Validator: + root = ROOT / "contracts/public/schemas"; schema = json.loads((root / name).read_text(encoding="utf-8")); problem = json.loads((root / "bb.problem.v1.schema.json").read_text(encoding="utf-8")) + registry = Registry().with_resource(problem["$id"], Resource.from_contents(problem)) + return Draft202012Validator(schema, registry=registry) + + +def test_deterministic_multiturn_replay_completes_with_full_redacted_artifact_graph(tmp_path: Path) -> None: + workspace, plan, result = _run(tmp_path) + execution = result.execution.as_dict(); manifest = result.manifest.as_dict() + assert execution["terminal_status"] == "completed" + assert execution["claimable"] is False + assert execution["comparison_report_id"] is None + assert execution["normalization_evidence_ids"] == [] + assert len(execution["provider_exchanges"]) == 3 + assert len(execution["tool_outcomes"]) == 2 + assert result.execution_path.is_file() + assert not (result.execution_path.parent / "workspace").exists() + _validator("bb.replay_plan.v1.schema.json").validate(plan.as_dict()) + _validator("bb.replay_execution.v1.schema.json").validate(execution) + _validator("bb.replay_artifact_manifest.v1.schema.json").validate(manifest) + store = ArtifactStore(workspace.path(".breadboard/artifacts")) + entries = {entry["artifact_id"]: entry for entry in manifest["entries"]} + stored_digests = {path.name for path in workspace.path(".breadboard/artifacts/sha256").glob("*/*") if path.is_file()} + assert stored_digests == {entry["sha256"].removeprefix("sha256:") for entry in entries.values()} + assert {entry["role"] for entry in entries.values()} >= {"kernel_event_stream", "provider_exchange", "tool_outcome", "workspace_before", "workspace_after", "workspace_diff", "redaction_report", "policy_decision"} + for entry in entries.values(): + payload = store.read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"])) + assert b"SECRET" not in payload + assert str(tmp_path).encode() not in payload + policy_entry = next(entry for entry in entries.values() if entry["role"] == "policy_decision") + policy = json.loads(store.read(ArtifactRef(policy_entry["sha256"], policy_entry["size_bytes"], policy_entry["media_type"]))) + assert "host_identity_sha256" in policy and "host_identity" not in policy + exchange_entry = next(entry for entry in entries.values() if entry["role"] == "provider_exchange") + exchange = json.loads(store.read(ArtifactRef(exchange_entry["sha256"], exchange_entry["size_bytes"], exchange_entry["media_type"]))) + _validator("bb.provider_exchange.v2.schema.json").validate(exchange) + + +def test_workspace_baseline_precedes_capability_worker_startup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import importlib + import sys + + module_root = tmp_path / "executor_module" + module_root.mkdir() + (module_root / "bootstrap_mutating_tool.py").write_text( + "from pathlib import Path\n" + "if Path.cwd().name == 'workspace':\n" + " Path('bootstrap-write.txt').write_text('startup mutation', encoding='utf-8')\n" + "class BootstrapMutatingTool:\n" + " def execute(self, arguments):\n" + " return {'unused': True}\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(module_root)) + importlib.invalidate_caches() + module = importlib.import_module("bootstrap_mutating_tool") + try: + tool_schemas = ({"type": "function", "function": {"name": "bootstrap", "parameters": {"type": "object"}}},) + workspace, _, result = _run( + tmp_path / "workspace-root", + sequence=("done",), + tool_schemas=tool_schemas, + tool_instances={"bootstrap": module.BootstrapMutatingTool()}, + ) + finally: + sys.modules.pop("bootstrap_mutating_tool", None) + assert result.execution.as_dict()["terminal_status"] == "completed" + entries = result.manifest.as_dict()["entries"] + store = ArtifactStore(workspace.path(".breadboard/artifacts")) + + def snapshot(role: str) -> dict[str, Any]: + entry = next(row for row in entries if row["role"] == role) + ref = ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]) + return json.loads(store.read(ref)) + + assert [row["path"] for row in snapshot("workspace_before")["files"]] == ["input.txt"] + assert [row["path"] for row in snapshot("workspace_after")["files"]] == ["input.txt"] + + +def test_replay_executes_tool_integration_adapter_in_its_capability_worker(tmp_path: Path) -> None: + adapter = ToolIntegrationAdapter("add", _AddTool()) + workspace, _, result = _run(tmp_path, sequence=("add", "done"), tool_instances={"add": adapter}) + assert result.execution.as_dict()["terminal_status"] == "completed" + entry = next(row for row in result.manifest.as_dict()["entries"] if row["role"] == "tool_outcome") + outcome = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]))) + assert outcome["result"]["sum"] == 5 + assert outcome["result"]["token"] == "" +def test_capability_worker_round_trips_scalar_value_configuration(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("add", "done"), tool_instances={"add": _ScalarConfiguredTool()}) + assert result.execution.as_dict()["terminal_status"] == "completed" + entry = next(row for row in result.manifest.as_dict()["entries"] if row["role"] == "tool_outcome") + outcome = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]))) + assert outcome["result"] == { + "duration": [2, 3, 4], + "fold": 1, + "mode": 3, + "ratio": "0.700", + "run_id": "12345678-1234-5678-1234-567812345678", + "started_at": "2026-07-22T12:34:56.000789-04:00", + "started_on": "2026-07-22", + "uuid_safety": "unknown", + "wake_at": "08:09:10.000011+00:00", + "wake_fold": 1, + "zone": "IST", + } + + +def test_capability_worker_rejects_unsupported_scalar_configuration(tmp_path: Path) -> None: + with pytest.raises(ReplayRunError, match="cannot be encoded losslessly"): + _run(tmp_path, sequence=("done",), tool_instances={"add": _UnsupportedScalarTool()}) + + +@pytest.mark.parametrize("nested", [False, True]) +def test_capability_worker_rejects_entrypoint_module_types(tmp_path: Path, nested: bool) -> None: + entrypoint_type = type( + "EntryPointCapability", + (), + {"__module__": "__main__", "execute": lambda self, _arguments: {"unexpected": True}}, + ) + if nested: + executor = _AddTool() + executor.entrypoint_state = entrypoint_type() + else: + executor = entrypoint_type() + with pytest.raises(ReplayRunError, match="entry-point module"): + _run(tmp_path, sequence=("done",), tool_instances={"add": executor}) + + + + +def test_live_provider_receives_unredacted_tool_result_while_artifact_is_redacted(tmp_path: Path) -> None: + provider = _LiveToolResultProvider(("add", "done")) + workspace, _, result = _run(tmp_path, provider_instance=provider) + assert result.execution.as_dict()["terminal_status"] == "completed" + entry = next(row for row in result.manifest.as_dict()["entries"] if row["role"] == "tool_outcome") + outcome = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]))) + assert outcome["result"]["token"] == "" + + +def test_replay_executes_callable_tool_integration_adapter_in_its_capability_worker(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("add", "done"), tool_instances={"add": ToolIntegrationAdapter("add", _add_callable)}) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_runtime_identity_binds_nested_bound_callable_owner_state() -> None: + owner = _BoundToolOwner(1) + adapter = ToolIntegrationAdapter("add", owner.run) + before = _runtime_type_identity(adapter) + owner.offset = 2 + assert _runtime_type_identity(adapter) != before + + +@pytest.mark.parametrize("authorize", [_PolicyOwner().allow, functools.partial(_prefixed_policy, "add"), functools.partial(_PrefixedPolicyOwner().allow, "add")]) +def test_replay_executes_bound_and_partial_policy_capabilities(tmp_path: Path, authorize: Any) -> None: + _, _, result = _run(tmp_path, sequence=("add", "done"), authorize=authorize) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +@pytest.mark.parametrize("executor", [functools.partial(_add_callable), len]) +def test_replay_executes_partial_and_builtin_tool_adapter_callables(tmp_path: Path, executor: Any) -> None: + _, _, result = _run(tmp_path, sequence=("add", "done"), tool_instances={"add": ToolIntegrationAdapter("add", executor)}) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_replay_preserves_private_slotted_tool_state(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("add", "done"), tool_instances={"add": _PrivateSlotTool(5)}) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_replay_forwards_validated_host_arguments(tmp_path: Path) -> None: + host_schema = ({ + "type": "function", + "function": { + "name": "host.execute", + "parameters": { + "type": "object", + "properties": { + "command": {"type": "string"}, + "timeout": {"type": "integer"}, + "stdin_data": {"type": "string"}, + }, + "required": ["command", "timeout", "stdin_data"], + "additionalProperties": False, + }, + }, + },) + _, _, result = _run(tmp_path, sequence=("host_args", "done"), tool_schemas=host_schema, host_instance=_ArgumentCheckingHost()) + assert result.execution.as_dict()["terminal_status"] == "completed" + + + + +def test_replay_executes_sandbox_host_adapter_in_its_capability_worker(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("host", "done"), host_instance=SandboxHostAdapter("fixture", _Host())) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_host_worker_denies_network_without_declared_capability(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("host", "done"), host_instance=_NetworkAttemptHost()) + assert result.execution.as_dict()["terminal_status"] == "host_failed" + + +def test_host_worker_allows_explicit_network_capability(tmp_path: Path) -> None: + host = SandboxHostAdapter("fixture", _NetworkAttemptHost(), effects=("filesystem", "network", "process")) + _, _, result = _run(tmp_path, sequence=("host", "done"), host_instance=host) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_tool_capability_rejects_reachable_provider_capability(tmp_path: Path) -> None: + with pytest.raises(ReplayRunError, match="tool capability state contains a forbidden provider capability"): + _run(tmp_path, sequence=("add", "done"), tool_instances={"add": _ToolWithProviderCapability()}) + + +def test_tool_capability_rejects_reachable_policy_callable(tmp_path: Path) -> None: + with pytest.raises(ReplayRunError, match="tool capability state contains a forbidden policy callable"): + _run(tmp_path, sequence=("add", "done"), tool_instances={"add": _ToolWithPolicyCapability()}) + + +def test_provider_wire_messages_alias_canonical_tool_calls_without_mutating_transcript() -> None: + schemas, reverse = replay_runner_module._provider_tool_wire_schemas(({"type": "function", "function": {"name": "host.execute", "parameters": {"type": "object"}}},)) + messages = [ + {"role": "assistant", "content": None, "tool_calls": [{"id": "call-1", "type": "function", "function": {"name": "host.execute", "arguments": "{}"}}]}, + {"role": "tool", "name": "host.execute", "tool_call_id": "call-1", "content": "{}"}, + {"type": "function_call", "name": "host.execute", "arguments": "{}"}, + {"role": "assistant", "content": [{"type": "tool_use", "name": "host.execute", "input": {}}]}, + {"role": "assistant", "content": None, "tool_calls": [{"id": "call-2", "type": "function", "name": "host.execute", "arguments": "{}"}]}, + ] + wire_messages = replay_runner_module._provider_wire_messages(messages, reverse) + assert schemas[0]["function"]["name"] == "host_execute" + assert wire_messages[0]["tool_calls"][0]["function"]["name"] == "host_execute" + assert wire_messages[1]["name"] == "host_execute" + assert wire_messages[2]["name"] == "host_execute" + assert wire_messages[3]["content"][0]["name"] == "host_execute" + assert wire_messages[4]["tool_calls"][0]["name"] == "host_execute" + assert messages[0]["tool_calls"][0]["function"]["name"] == "host.execute" + + +def test_colliding_provider_aliases_remain_stable_across_turns(tmp_path: Path) -> None: + schemas = ( + {"type": "function", "function": {"name": "a.b", "parameters": {"type": "object"}}}, + {"type": "function", "function": {"name": "a_b", "parameters": {"type": "object"}}}, + ) + _, _, result = _run( + tmp_path, + sequence=("unused",), + provider_instance=_CollisionAliasProvider(("unused",)), + tool_schemas=schemas, + tool_instances={"a.b": _AddTool(), "a_b": _AddTool()}, + ) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_forced_tool_choice_follows_provider_wire_alias(tmp_path: Path) -> None: + provider_context = types.SimpleNamespace( + agent_config={"provider_tools": {"anthropic": {"tool_choice": {"type": "tool", "name": "host.execute"}}}}, + extra={}, + ) + _, _, result = _run( + tmp_path, + sequence=("host", "done"), + provider_instance=_ForcedToolChoiceProvider(("host", "done")), + provider_context=provider_context, + ) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_flat_forced_tool_choice_follows_provider_wire_alias(tmp_path: Path) -> None: + provider_context = types.SimpleNamespace( + agent_config={"provider_tools": {"tool_choice": {"type": "tool", "name": "host.execute"}}}, + extra={}, + ) + _, _, result = _run( + tmp_path, + sequence=("host", "done"), + provider_instance=_FlatForcedToolChoiceProvider(("host", "done")), + provider_context=provider_context, + ) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_mixed_flat_and_scoped_tool_choices_follow_provider_wire_alias(tmp_path: Path) -> None: + canonical_choice = {"type": "tool", "name": "host.execute"} + provider_context = types.SimpleNamespace( + agent_config={"provider_tools": {"tool_choice": canonical_choice, "anthropic": {"tool_choice": canonical_choice}}}, + extra={}, + ) + _, _, result = _run( + tmp_path, + sequence=("host", "done"), + provider_instance=_MixedForcedToolChoiceProvider(("host", "done")), + provider_context=provider_context, + ) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_isolated_provider_context_advances_state_between_turns(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("unused",), provider_instance=_StatefulProvider(("unused",))) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_provider_client_factory_receives_allowlisted_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BB_REPLAY_ENV_SECRET", "ENVIRONMENT_ONLY_CREDENTIAL") + _, _, result = _run(tmp_path, sequence=("done",), provider_instance=_EnvironmentFactoryProvider(("done",)), environment_allowlist=("BB_REPLAY_ENV_SECRET",)) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_provider_client_spec_requires_reconstruction_factory(tmp_path: Path) -> None: + with pytest.raises(ReplayRunError, match="requires replay_worker_client"): + _run(tmp_path, sequence=("done",), provider_instance=_SpecOnlyProvider(("done",))) + + +def test_interaction_script_is_bound_into_provider_messages(tmp_path: Path) -> None: + script = ({"role": "system", "content": "frozen replay step"},) + workspace, _, result = _run(tmp_path, sequence=("done",), interaction_script=script) + request_entry = next(entry for entry in result.manifest.as_dict()["entries"] if entry["role"] == "provider_request") + request = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(request_entry["sha256"], request_entry["size_bytes"], request_entry["media_type"]))) + assert [row["role"] for row in request["messages"]] == ["user", "system"] + assert request["messages"][1]["content"] == "frozen replay step" + + +@pytest.mark.parametrize( + ("sequence", "tool_fail", "host_fail", "authorize", "status"), + [ + (("provider_error",), False, False, None, "provider_failed"), + (("add",), True, False, None, "tool_failed"), + (("host",), False, True, None, "host_failed"), + (("add",), False, False, _deny, "policy_denied"), + ], +) +def test_runtime_failures_are_persisted_but_never_claimable(tmp_path: Path, sequence, tool_fail, host_fail, authorize, status) -> None: + _, _, result = _run(tmp_path, sequence=sequence, tool_fail=tool_fail, host_fail=host_fail, authorize=authorize) + record = result.execution.as_dict() + assert record["terminal_status"] == status + assert record["claimable"] is False + assert record["comparison_report_id"] is None + assert record["normalization_evidence_ids"] == [] + assert record["problem"]["failed_stage"] == "replay" + _validator("bb.replay_execution.v1.schema.json").validate(record) + with pytest.raises(ReplayExecutionError, match="only completed"): + result.execution.require_comparable() + +def test_reuse_execution_is_never_comparable(tmp_path: Path) -> None: + _, plan, result = _run(tmp_path, sequence=("done",)) + record = result.execution.as_dict() + record.update( + mode="reuse", + fresh_nonce=None, + reuse_attestation_id="reuse-attestation.fixture", + provider_exchanges=[], + ) + execution = ReplayExecution.from_dict(record) + with pytest.raises(ReplayExecutionError, match="execute-mode"): + execution.require_comparable() + with pytest.raises(ReplayExecutionError, match="mode does not match"): + execution.verify_plan(plan) + + + +@pytest.mark.parametrize( + ("sequence", "deadlines", "provider_status", "tool_count"), + [ + (("done",), {"total": 10_000, "idle": 1_000, "provider_call": 1, "tool_call": 2_000}, "timed_out", 0), + (("add",), {"total": 10_000, "idle": 1_000, "provider_call": 2_000, "tool_call": 1}, "timed_out", 0), + ], +) +def test_provider_and_tool_deadlines_persist_timeout_evidence(tmp_path: Path, sequence, deadlines, provider_status, tool_count) -> None: + _, _, result = _run(tmp_path, sequence=sequence, deadlines=deadlines, monotonic=_SlowMonotonic()) + record = result.execution.as_dict() + assert record["terminal_status"] == "timed_out" + assert record["provider_exchanges"][0]["status"] == provider_status + assert len(record["tool_outcomes"]) == tool_count + assert record["claimable"] is False + _validator("bb.replay_execution.v1.schema.json").validate(record) + +def test_tool_worker_startup_obeys_tool_deadline( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module_root = tmp_path / "capabilities" + module_root.mkdir() + module_path = module_root / "slow_start_tool.py" + module_path.write_text( + "class SlowStartTool:\n" + " def __init__(self):\n" + " self.payload = 'x' * (512 * 1024)\n" + " def execute(self, _arguments):\n" + " return {'started': True}\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(module_root)) + module = importlib.import_module("slow_start_tool") + module_path.write_text( + "import time\n" + "time.sleep(10)\n" + "class SlowStartTool:\n" + " def __init__(self):\n" + " self.payload = 'x' * (512 * 1024)\n" + " def execute(self, _arguments):\n" + " return {'started': True}\n", + encoding="utf-8", + ) + importlib.invalidate_caches() + started = time.monotonic() + try: + tool_schemas = ({"type": "function", "function": {"name": "lazy", "parameters": {"type": "object"}}},) + _, _, result = _run( + tmp_path / "workspace", + sequence=("lazy",), + tool_schemas=tool_schemas, + tool_instances={"lazy": module.SlowStartTool()}, + deadlines={"total": 10_000, "idle": 3_000, "provider_call": 1_000, "tool_call": 1_200}, + ) + finally: + sys.modules.pop("slow_start_tool", None) + record = result.execution.as_dict() + assert time.monotonic() - started < 8 + assert record["terminal_status"] == "timed_out" + assert record["problem"]["error_code"] == "replay.tool_timeout" + + + +def test_policy_dispatch_rechecks_deadline_after_exchange_evidence_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + clock = [0.0] + original_put_json = ArtifactStore.put_json + + def put_json(store: ArtifactStore, value: Any, *, created: set[ArtifactRef] | None = None) -> ArtifactRef: + ref = original_put_json(store, value, created=created) + if isinstance(value, dict) and value.get("schema_version") == "bb.provider_exchange.v2": + clock[0] = 2.0 + return ref + + monkeypatch.setattr(ArtifactStore, "put_json", put_json) + deadlines = {"total": 1_000, "idle": 10_000, "provider_call": 1_000, "tool_call": 1_000} + _, _, result = _run( + tmp_path, + sequence=("add",), + authorize=_deny, + deadlines=deadlines, + monotonic=lambda: clock[0], + ) + record = result.execution.as_dict() + assert record["terminal_status"] == "timed_out" + assert record["problem"]["error_code"] == "replay.total_timeout" + + +def test_worker_result_arriving_after_call_deadline_is_timed_out(tmp_path: Path) -> None: + deadlines = {"total": 2_000, "idle": 1_000, "provider_call": 1, "tool_call": 1_000} + _, _, result = _run(tmp_path, sequence=("brief",), deadlines=deadlines) + record = result.execution.as_dict() + assert record["terminal_status"] == "timed_out" + assert record["problem"]["error_code"] == "replay.provider_timeout" + + +@pytest.mark.parametrize( + ("sequence", "tool_hang", "deadlines", "error_code"), + [ + (("hang",), False, {"total": 2_000, "idle": 1_000, "provider_call": 700, "tool_call": 1_000}, "replay.provider_timeout"), + (("add",), True, {"total": 5_000, "idle": 2_000, "provider_call": 1_000, "tool_call": 1_000}, "replay.tool_timeout"), + ], +) +def test_hung_external_calls_are_terminated_at_deadline(tmp_path: Path, sequence, tool_hang, deadlines, error_code) -> None: + _, _, result = _run(tmp_path, sequence=sequence, tool_hang=tool_hang, deadlines=deadlines) + record = result.execution.as_dict() + assert record["terminal_status"] == "timed_out" + assert record["problem"]["error_code"] == error_code + +@pytest.mark.parametrize(("sequence", "tool_hang"), [(("hang",), False), (("add",), True)]) +def test_idle_deadline_caps_hung_provider_and_tool_calls(tmp_path: Path, sequence, tool_hang) -> None: + deadlines = {"total": 2_000, "idle": 20, "provider_call": 100, "tool_call": 100} + _, _, result = _run(tmp_path, sequence=sequence, tool_hang=tool_hang, deadlines=deadlines) + record = result.execution.as_dict() + assert record["terminal_status"] == "timed_out" + assert record["problem"]["error_code"] == "replay.idle_timeout" + + +def test_tool_budget_counts_each_dispatch(tmp_path: Path) -> None: + budgets = {"turns": 5, "provider_calls": 5, "tool_calls": 1, "tokens": 100, "cost": 1} + _, _, result = _run(tmp_path, sequence=("two_adds",), budgets=budgets) + record = result.execution.as_dict() + assert record["terminal_status"] == "budget_exhausted" + assert record["problem"]["error_code"] == "replay.tool_budget" + assert len(record["tool_outcomes"]) == 1 + + +def test_total_and_idle_deadlines_are_enforced_independently(tmp_path: Path) -> None: + _, _, total_result = _run( + tmp_path / "total", + sequence=("done",), + deadlines={"total": 2_500, "idle": 10_000, "provider_call": 2_000, "tool_call": 2_000}, + monotonic=_SlowMonotonic(), + ) + total = total_result.execution.as_dict() + assert total["terminal_status"] == "timed_out" + assert total["provider_exchanges"][0]["status"] == "timed_out" + assert total["problem"]["error_code"] == "replay.total_timeout" + _, _, idle_result = _run( + tmp_path / "idle", + sequence=("done",), + deadlines={"total": 10_000, "idle": 1, "provider_call": 2_000, "tool_call": 2_000}, + monotonic=_SlowMonotonic(), + ) + idle = idle_result.execution.as_dict() + assert idle["terminal_status"] == "timed_out" + assert idle["provider_exchanges"] == [] + assert idle["problem"]["error_code"] == "replay.idle_timeout" + + +def test_cancellation_after_provider_call_is_not_misclassified_by_elapsed_work(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("done",), cancelled=_CancelAfterProvider()) + record = result.execution.as_dict() + assert record["terminal_status"] == "cancelled" + assert record["provider_exchanges"][0]["status"] == "cancelled" + assert record["problem"]["error_code"] == "replay.cancelled" + _, _, grace_result = _run( + tmp_path / "grace", + sequence=("done",), + deadlines={"total": 10_000, "idle": 1_000, "provider_call": 2_000, "tool_call": 2_000}, + monotonic=_SlowMonotonic(), + cancelled=_CancelAfterProvider(), + ) + assert grace_result.execution.as_dict()["problem"]["error_code"] == "replay.cancelled" + + +def test_cancellation_wins_when_provider_failure_arrives_concurrently(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("provider_error",), cancelled=_CancelAfterProvider()) + record = result.execution.as_dict() + assert record["terminal_status"] == "cancelled" + assert record["provider_exchanges"][0]["status"] == "cancelled" + assert record["problem"]["error_code"] == "replay.cancelled" + + +def test_runtime_rejects_route_and_toolset_drift(tmp_path: Path) -> None: + wrong_route = {"provider_id": "other", "runtime_id": "fixture_runtime", "endpoint_class": "fixture", "model_id": "fixture-model", "model_revision": None} + with pytest.raises(ReplayPlanError, match="provider runtime"): + _run(tmp_path / "route", sequence=("done",), provider_route=wrong_route) + _, _, result = _run(tmp_path / "tool", sequence=("unknown",)) + record = result.execution.as_dict() + assert record["terminal_status"] == "policy_denied" + assert record["problem"]["error_code"] == "replay.undeclared_tool" + assert {row["kind"]: row["decision"] for row in record["policy_decisions"]}["capability"] == "denied" + +def test_runtime_identity_binds_generic_counter_state() -> None: + provider = _Provider(("done",)) + planned = replay_runner_module._runtime_type_identity(provider) + provider.calls = 1 + assert replay_runner_module._runtime_type_identity(provider) != planned + +def test_runtime_identity_binds_scalar_value_state() -> None: + tool = _ScalarConfiguredTool() + planned = replay_runner_module._runtime_type_identity(tool) + tool.ratio = Decimal("0.701") + assert replay_runner_module._runtime_type_identity(tool) != planned + + + +def test_provider_route_identity_excludes_transient_replay_client_specs() -> None: + descriptor = ProviderDescriptor("fixture", "fixture_runtime", "chat", True, False, False, False, "openai", None, "FIXTURE_KEY", {}) + adapter = ProviderRuntimeAdapter(_IdentityProviderRuntime(), descriptor) + active_client = adapter.create_client("ACTIVE_SECRET", base_url="https://active.invalid") + active_identity = _provider_client_binding(adapter, active_client, {"FIXTURE_KEY": "ACTIVE_SECRET"}) + before = _provider_route_binding("fixture", "fixture_runtime", "chat", "fixture-model", None, "1.0", adapter, active_identity) + adapter.create_client("UNRELATED_SECRET", base_url="https://unrelated.invalid") + after = _provider_route_binding("fixture", "fixture_runtime", "chat", "fixture-model", None, "1.0", adapter, active_identity) + assert after == before + encoded = json.dumps(after, sort_keys=True) + assert "ACTIVE_SECRET" not in encoded + assert "UNRELATED_SECRET" not in encoded + + +def test_provider_cost_currency_cannot_change_mid_replay(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("add", "eur")) + record = result.execution.as_dict() + assert record["terminal_status"] == "provider_failed" + assert record["problem"]["error_code"] == "replay.cost_currency_mismatch" +def test_failed_provider_turn_preserves_failure_after_non_usd_usage(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("add_eur", "provider_error")) + record = result.execution.as_dict() + assert record["terminal_status"] == "provider_failed" + assert record["problem"]["error_code"] == "replay.provider_failed" + + + + +def test_usage_normalization_is_conservative_and_rejects_invalid_counts() -> None: + understated = ProviderResult([], None, {"input_tokens": 100, "output_tokens": 100, "total_tokens": 1}) + assert normalized_provider_usage(understated)["total_tokens"] == 200 + with pytest.raises(ProviderRuntimeError, match="nonnegative integer"): + normalized_provider_usage(ProviderResult([], None, {"input_tokens": -1})) + charged = normalized_provider_usage(ProviderResult([], None, {"cost_amount": 0.75}, metadata={})) + assert charged["cost_amount"] == 0.75 + + +def test_opaque_provider_values_fail_without_nondeterministic_evidence(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("opaque",)) + record = result.execution.as_dict() + assert record["terminal_status"] == "provider_failed" + assert record["provider_exchanges"][0]["status"] == "provider_error" + _, _, invalid_result = _run(tmp_path / "invalid", sequence=("invalid_message",)) + assert invalid_result.execution.as_dict()["terminal_status"] == "provider_failed" + +def test_noncanonical_tool_arguments_are_invalid_provider_responses(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("nan_args",)) + record = result.execution.as_dict() + assert record["terminal_status"] == "provider_failed" + assert record["provider_exchanges"][0]["status"] == "invalid_response" + assert record["problem"]["error_code"] == "replay.invalid_provider_response" + assert record["tool_outcomes"] == [] + + +def test_provider_request_hash_matches_empty_tool_payload(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("done",), tool_schemas=()) + request_entry = next(entry for entry in result.manifest.as_dict()["entries"] if entry["role"] == "provider_request") + request = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(request_entry["sha256"], request_entry["size_bytes"], request_entry["media_type"]))) + assert request["tools"] == [] + + +def test_provider_request_evidence_binds_replay_context_options(tmp_path: Path) -> None: + contexts = ( + ProviderRuntimeContext(session_state=None, agent_config={"provider_tools": {"anthropic": {"temperature": 0.1}}}, extra={"responses_extra": {"mode": "first"}}), + ProviderRuntimeContext(session_state=None, agent_config={"provider_tools": {"anthropic": {"temperature": 0.9}}}, extra={"responses_extra": {"mode": "second"}}), + ) + requests = [] + request_digests = [] + for index, context in enumerate(contexts): + workspace, _, result = _run(tmp_path / str(index), sequence=("done",), provider_context=context) + request_entry = next(entry for entry in result.manifest.as_dict()["entries"] if entry["role"] == "provider_request") + requests.append(json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(request_entry["sha256"], request_entry["size_bytes"], request_entry["media_type"])))) + request_digests.append(request_entry["sha256"]) + assert requests[0]["context"]["agent_config"]["provider_tools"]["anthropic"]["temperature"] == 0.1 + assert requests[0]["context"]["extra"]["responses_extra"] == {"mode": "first"} + assert request_digests[0] != request_digests[1] + + + + +def test_tool_arguments_must_match_frozen_parameter_schema(tmp_path: Path) -> None: + strict_schema = ( + { + "type": "function", + "function": { + "name": "add", + "parameters": { + "type": "object", + "properties": {"required_value": {"type": "integer"}}, + "required": ["required_value"], + "additionalProperties": False, + }, + }, + }, + ) + _, _, result = _run(tmp_path, sequence=("add",), tool_schemas=strict_schema) + record = result.execution.as_dict() + assert record["terminal_status"] == "provider_failed" + assert record["provider_exchanges"][0]["status"] == "invalid_response" + assert record["tool_outcomes"] == [] + + +def test_noncanonical_tool_results_persist_failure_evidence(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("add",), tool_invalid=True) + record = result.execution.as_dict() + assert record["terminal_status"] == "tool_failed" + assert record["problem"]["error_code"] == "replay.tool_invalid_result" + assert len(record["tool_outcomes"]) == 1 + _, _, timed_result = _run( + tmp_path / "timed", + sequence=("add",), + tool_invalid=True, + deadlines={"total": 10_000, "idle": 1_000, "provider_call": 2_000, "tool_call": 1}, + monotonic=_SlowMonotonic(), + ) + timed = timed_result.execution.as_dict() + assert timed["terminal_status"] == "timed_out" + assert timed["problem"]["error_code"] == "replay.idle_timeout" + + +def test_workspace_directory_replacement_is_blocked_by_worker_sandbox(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("host",), host_replace=True) + record = result.execution.as_dict() + assert record["terminal_status"] == "host_failed" + assert record["problem"]["error_code"] == "replay.host_failed" + assert not any(path.is_dir() for path in result.execution_path.parent.iterdir()) + assert record["integrity_verified"] is True + assert result.manifest.as_dict()["publish_status"] == "complete" + assert result.manifest.as_dict()["integrity_verified"] is True + + +def test_initial_workspace_paths_must_be_canonical() -> None: + with pytest.raises(ReplayRunError, match="unsafe initial workspace path"): + ReplayScenario("task", ({"role": "user", "content": "go"},), {"a//b": "content"}, ()) + with pytest.raises(ReplayRunError, match="tool_schemas"): + ReplayScenario("task", ({"role": "user", "content": "go"},), {}, (None,)) # type: ignore[arg-type] + + +def test_provider_exchange_hashes_bind_unredacted_payloads(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("add", "done")) + store = ArtifactStore(workspace.path(".breadboard/artifacts")) + entries = result.manifest.as_dict()["entries"] + request_entries = [entry for entry in entries if entry["role"] == "provider_request"] + response_entries = [entry for entry in entries if entry["role"] == "provider_response"] + exchange_entries = [entry for entry in entries if entry["role"] == "provider_exchange"] + + exchanges = [ + json.loads(store.read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]))) + for entry in exchange_entries + ] + assert len(exchanges) == len(request_entries) == len(response_entries) == 2 + assert all(exchange["request_payload_sha256"] != entry["sha256"] for exchange, entry in zip(exchanges, request_entries, strict=True)) + assert all(exchange["response_payload_sha256"] != entry["sha256"] for exchange, entry in zip(exchanges, response_entries, strict=True)) + request_digests = {entry["sha256"] for entry in request_entries} + assert all(len(request_digests.intersection(exchange["evidence_refs"])) == 1 for exchange in exchanges) + assert {ref for exchange in exchanges for ref in exchange["evidence_refs"] if ref in request_digests} == request_digests + + +def test_provider_response_evidence_preserves_reasoning_fields(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("done",), provider_instance=_ReasoningProvider(("done",))) + entry = next(entry for entry in result.manifest.as_dict()["entries"] if entry["role"] == "provider_response") + payload = json.loads( + ArtifactStore(workspace.path(".breadboard/artifacts")).read( + ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]) + ) + ) + assert payload["encrypted_reasoning"] == [{"encrypted_content": "ciphertext", "metadata": {"response_id": "response-1"}}] + assert payload["reasoning_summaries"] == ["reasoning summary"] + + +def test_authorization_and_cookie_fields_are_always_redacted(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("credential_fields",)) + store = ArtifactStore(workspace.path(".breadboard/artifacts")) + for entry in result.manifest.as_dict()["entries"]: + payload = store.read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"])) + assert b"UNLISTED" not in payload + for subdir, kwargs in ( + ("provider-error", {"sequence": ("provider_secret_error",)}), + ("tool-error", {"sequence": ("add",), "tool_fail": True}), + ): + failure_workspace, _, failure_result = _run(tmp_path / subdir, **kwargs) + failure_store = ArtifactStore(failure_workspace.path(".breadboard/artifacts")) + for entry in failure_result.manifest.as_dict()["entries"]: + payload = failure_store.read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"])) + assert b"UNLISTED" not in payload + + +def test_manifest_paths_and_content_address_are_revalidated(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("done",)) + unsafe = result.manifest.as_dict() + unsafe["entries"][0]["location_kind"] = "workspace_relative_path" + unsafe["entries"][0]["location"] = "../escape" + with pytest.raises(ReplayExecutionError, match="within the workspace"): + ReplayArtifactManifest.from_dict(unsafe) + tampered = result.manifest.as_dict() + tampered["entries"][0]["role"] = "changed" + with pytest.raises(ReplayExecutionError, match="manifest_id"): + ReplayArtifactManifest.from_dict(tampered) + noncanonical = result.manifest.as_dict() + noncanonical["entries"][0]["location_kind"] = "workspace_relative_path" + noncanonical["entries"][0]["location"] = "a//b" + with pytest.raises(ReplayExecutionError, match="canonical"): + ReplayArtifactManifest.from_dict(noncanonical) + + +def test_manifest_accepts_schema_valid_remote_object_reference(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("done",)) + remote = result.manifest.as_dict() + remote["entries"][0]["location"] = "s3://breadboard-evidence/replays/execution.json" + unsigned = dict(remote) + unsigned.pop("manifest_id") + remote["manifest_id"] = "replay_manifest:" + hashlib.sha256(canonical_json(unsigned)).hexdigest() + _validator("bb.replay_artifact_manifest.v1.schema.json").validate(remote) + assert ReplayArtifactManifest.from_dict(remote).as_dict() == remote + + +def test_plan_mutation_invalidates_identity_and_execution_reuse(tmp_path: Path) -> None: + _, plan, result = _run(tmp_path, sequence=("done",)) + changed = plan.as_dict(); changed["budgets"]["turns"] += 1 + with pytest.raises(ReplayPlanError, match="plan_id"): + ReplayPlan.from_dict(changed) + replacement = _scenario(); lock, bindings, other = _plan(replacement); other_record = other.as_dict(); other_record["budgets"]["turns"] += 1; other_record["plan_id"] = "replay_plan:" + "2" * 64 + with pytest.raises(ReplayPlanError): + ReplayPlan.from_dict(other_record) + assert lock.as_dict()["graph_hash"] == plan.as_dict()["harness_lock_sha256"] + result.execution.verify_plan(plan) + mismatched_locks = result.execution.as_dict() + mismatched_locks["lane_lock_sha256"] = "sha256:" + "2" * 64 + with pytest.raises(ReplayExecutionError, match="lock hashes"): + ReplayExecution.from_dict(mismatched_locks).verify_plan(plan) + + +def test_stored_execution_cannot_acquire_executed_pass_or_claimability(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("done",)) + injected = result.execution.as_dict(); injected["executed_pass"] = True + with pytest.raises(ReplayExecutionError, match="fields"): + ReplayExecution.from_dict(injected) + promoted = result.execution.as_dict() + promoted["claimable"] = True + promoted["normalization_evidence_ids"] = ["normalization:forged"] + promoted["comparison_report_id"] = "comparison:forged" + with pytest.raises(ReplayExecutionError, match="candidate replay"): + ReplayExecution.from_dict(promoted) + + +def test_persistence_failure_rolls_back_cas_and_staging(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + original = replay_runner_module.AnchoredStorage.write_at + def fail_execution(parent, name, content): + if name == "execution.json": raise OSError("injected persistence failure") + return original(parent, name, content) + monkeypatch.setattr(replay_runner_module.AnchoredStorage, "write_at", staticmethod(fail_execution)) + with pytest.raises(OSError, match="injected"): + _run(tmp_path, sequence=("done",)) + artifact_root = tmp_path / ".breadboard/artifacts" + digest_files = [path for path in artifact_root.rglob("*") if path.is_file() and len(path.name) == 64] + assert digest_files == [] + assert list((tmp_path / ".breadboard/replays").glob(".*.staging")) == [] + +def test_post_publication_metadata_failure_removes_replay_and_cas(tmp_path: Path) -> None: + class FailingMetadataSink: + def provider_metadata_snapshot(self) -> dict[str, Any]: + return {} + + def set_provider_metadata(self, _name: str, _value: Any) -> None: + raise RuntimeError("injected metadata persistence failure") + + context = ProviderRuntimeContext(session_state=FailingMetadataSink(), agent_config={}, extra={}) + with pytest.raises(RuntimeError, match="metadata persistence"): + _run(tmp_path, sequence=("done",), provider_context=context) + assert not any((tmp_path / ".breadboard/replays").iterdir()) + artifact_root = tmp_path / ".breadboard/artifacts" + assert not [path for path in artifact_root.rglob("*") if path.is_file() and len(path.name) == 64] + + +def test_replay_contracts_remain_candidate_until_lifecycle_promotion() -> None: + surface = json.loads((ROOT / "contracts/public/record_surface.v1.json").read_text(encoding="utf-8")) + roles = {row["role_id"]: row["status"] for row in surface["roles"]} + assert roles["replay_plan"] == "candidate" + assert roles["replay_execution"] == "candidate" + assert roles["provider_exchange"] == "candidate" + + +def test_worker_ipc_never_unpickles_raw_provider_objects(tmp_path: Path) -> None: + marker = tmp_path / "unpickled" + _, _, result = _run(tmp_path / "workspace", sequence=("done",), provider_instance=_RawProvider(marker)) + assert result.execution.as_dict()["terminal_status"] == "completed" + assert not marker.exists() + + +def test_runtime_and_environment_bindings_capture_configuration(monkeypatch: pytest.MonkeyPatch) -> None: + assert _runtime_type_identity(_AddTool()) != _runtime_type_identity(_AddTool(fail=True)) + numeric_key = _AddTool(); numeric_key.config = {1: "x"} + string_key = _AddTool(); string_key.config = {"1": "x"} + assert _runtime_type_identity(numeric_key) != _runtime_type_identity(string_key) + first_bytes = _AddTool(); first_bytes.config = {b"a": "x"} + second_bytes = _AddTool(); second_bytes.config = {b"b": "x"} + assert _runtime_type_identity(first_bytes) != _runtime_type_identity(second_bytes) + monkeypatch.setenv("BB_REPLAY_FIXTURE", "first") + first, _ = _environment_binding(("BB_REPLAY_FIXTURE",), {}) + monkeypatch.setenv("BB_REPLAY_FIXTURE", "second") + second, _ = _environment_binding(("BB_REPLAY_FIXTURE",), {}) + original_execute = _AddTool.execute + def replacement_execute(self, arguments): + return {"sum": 999} + before_runtime = _runtime_type_identity(_AddTool()) + monkeypatch.setattr(_AddTool, "execute", replacement_execute) + assert _runtime_type_identity(_AddTool()) != before_runtime + monkeypatch.setattr(_AddTool, "execute", original_execute) + assert first != second + monkeypatch.delenv("BB_REPLAY_FIXTURE") + with pytest.raises(ReplayRunError, match="absent"): + _environment_binding(("BB_REPLAY_FIXTURE",), {}) + + +def test_environment_only_values_are_redacted_from_all_artifacts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + value = "ENVIRONMENT_ONLY_CREDENTIAL" + monkeypatch.setenv("BB_REPLAY_ENV_SECRET", value) + workspace, _, result = _run(tmp_path, sequence=("environment_value",), environment_allowlist=("BB_REPLAY_ENV_SECRET",)) + execution = result.execution.as_dict() + assert execution["terminal_status"] == "completed" + assert execution["provider_exchanges"][0]["status"] == "completed" + store = ArtifactStore(workspace.path(".breadboard/artifacts")) + for entry in result.manifest.as_dict()["entries"]: + payload = store.read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"])) + assert value.encode() not in payload + response_entry = next(entry for entry in result.manifest.as_dict()["entries"] if entry["role"] == "provider_response") + response = json.loads(store.read(ArtifactRef(response_entry["sha256"], response_entry["size_bytes"], response_entry["media_type"]))) + assert response["messages"][0]["content"] == "" + + +def test_overlapping_environment_secrets_are_redacted_longest_first(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BB_REPLAY_SHORT_SECRET", "OVERLAP") + monkeypatch.setenv("BB_REPLAY_LONG_SECRET", "OVERLAP_SECRET") + workspace, _, result = _run( + tmp_path, + sequence=("overlapping_environment_value",), + environment_allowlist=("BB_REPLAY_SHORT_SECRET", "BB_REPLAY_LONG_SECRET"), + ) + response_entry = next(entry for entry in result.manifest.as_dict()["entries"] if entry["role"] == "provider_response") + response = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(response_entry["sha256"], response_entry["size_bytes"], response_entry["media_type"]))) + assert response["messages"][0]["content"] == "" + + + + +def test_manifest_rejects_workspace_root_location(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("done",)) + malformed = result.manifest.as_dict() + malformed["entries"][0]["location_kind"] = "workspace_relative_path" + malformed["entries"][0]["location"] = "." + with pytest.raises(ReplayExecutionError, match="workspace_relative_path"): + ReplayArtifactManifest.from_dict(malformed) + + +def test_policy_evaluation_obeys_deadline_and_persists_timeout(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("add",), authorize=_hang_policy, deadlines={"total": 2_000, "idle": 2_000, "provider_call": 1_000, "tool_call": 20}) + record = result.execution.as_dict() + assert record["terminal_status"] == "timed_out" + assert record["problem"]["error_code"] == "replay.policy_timeout" + + assert next(row for row in record["policy_decisions"] if row["kind"] == "policy")["decision"] == "denied" + +def test_cancellation_before_policy_dispatch_records_denial(tmp_path: Path) -> None: + ticks = [0] + + def monotonic() -> float: + ticks[0] += 1 + return ticks[0] / 1_000 + + def cancelled() -> bool: + return ticks[0] >= 6 + + _, _, result = _run(tmp_path, sequence=("add",), monotonic=monotonic, cancelled=cancelled) + record = result.execution.as_dict() + assert record["terminal_status"] == "cancelled" + assert next(row for row in record["policy_decisions"] if row["kind"] == "policy")["decision"] == "denied" + + +def test_cancellation_after_policy_result_precedes_policy_denial(tmp_path: Path) -> None: + ticks = [0] + + def monotonic() -> float: + ticks[0] += 1 + return ticks[0] / 1_000 + + def cancelled() -> bool: + return ticks[0] >= 8 + + _, _, result = _run( + tmp_path, + sequence=("add",), + authorize=_deny, + monotonic=monotonic, + cancelled=cancelled, + ) + record = result.execution.as_dict() + assert record["terminal_status"] == "cancelled" + assert record["problem"]["error_code"] == "replay.cancelled" + + +def test_tool_relative_filesystem_writes_are_workspace_anchored(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("add",), tool_instances={"add": _RelativeWriteTool()}) + entry = next(row for row in result.manifest.as_dict()["entries"] if row["role"] == "workspace_after") + snapshot = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]))) + assert "relative-output.txt" in {row["path"] for row in snapshot["files"]} + assert not (ROOT / "relative-output.txt").exists() + + +def test_worker_os_sandbox_denies_process_creation(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("add",), tool_instances={"add": _ForkTool()}) + record = result.execution.as_dict() + assert record["terminal_status"] == "tool_failed" + assert record["tool_outcomes"] + + +def test_worker_os_sandbox_denies_process_replacement(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("add",), tool_instances={"add": _ExecReplacementTool()}) + assert result.execution.as_dict()["terminal_status"] == "tool_failed" + assert not (workspace.root / "exec-replacement.txt").exists() + + +def test_worker_os_sandbox_denies_tool_network_egress(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("add",), tool_instances={"add": _NetworkTool()}) + record = result.execution.as_dict() + assert record["terminal_status"] == "tool_failed" + assert record["problem"]["error_code"] == "replay.tool_failed" + + +def test_worker_supports_lazy_sibling_package_imports(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + package_parent = tmp_path / "capabilities" + package = package_parent / "lazy_capability_fixture" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "helper.py").write_text("def result():\n return {'lazy_imported': True}\n", encoding="utf-8") + (package / "tool.py").write_text( + "class LazyTool:\n" + " def execute(self, _arguments):\n" + " from .helper import result\n" + " return result()\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(package_parent)) + module = importlib.import_module("lazy_capability_fixture.tool") + try: + tool_schemas = ({"type": "function", "function": {"name": "lazy", "parameters": {"type": "object"}}},) + _, _, result = _run( + tmp_path / "workspace", + sequence=("lazy", "done"), + tool_schemas=tool_schemas, + tool_instances={"lazy": module.LazyTool()}, + ) + finally: + sys.modules.pop("lazy_capability_fixture.helper", None) + sys.modules.pop("lazy_capability_fixture.tool", None) + sys.modules.pop("lazy_capability_fixture", None) + record = result.execution.as_dict() + assert record["terminal_status"] == "completed" + + +def test_worker_supports_and_binds_package_resource_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + package_parent = tmp_path / "capabilities" + package = package_parent / "resource_capability_fixture" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + resource = package / "data.json" + resource.write_text('{"version":1}', encoding="utf-8") + (package / "tool.py").write_text( + "import json\n" + "from pathlib import Path\n" + "class ResourceTool:\n" + " def execute(self, _arguments):\n" + " return json.loads(Path(__file__).with_name('data.json').read_text(encoding='utf-8'))\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(package_parent)) + module = importlib.import_module("resource_capability_fixture.tool") + try: + before = _capability_runtime_identity(module.ResourceTool(), "tool") + resource.write_text('{"version":2}', encoding="utf-8") + after = _capability_runtime_identity(module.ResourceTool(), "tool") + tool_schemas = ({"type": "function", "function": {"name": "resource", "parameters": {"type": "object"}}},) + _, _, result = _run( + tmp_path / "workspace", + sequence=("resource", "done"), + tool_schemas=tool_schemas, + tool_instances={"resource": module.ResourceTool()}, + ) + finally: + sys.modules.pop("resource_capability_fixture.tool", None) + sys.modules.pop("resource_capability_fixture", None) + assert before != after + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_top_level_capability_does_not_allowlist_unrelated_sibling_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module_root = tmp_path / "capabilities" + module_root.mkdir() + unrelated = module_root / ".env" + unrelated.write_text("PRIVATE=first", encoding="utf-8") + (module_root / "top_level_resource_tool.py").write_text( + "from pathlib import Path\n" + "class ResourceTool:\n" + " def execute(self, _arguments):\n" + " return {'content': Path(__file__).with_name('.env').read_text(encoding='utf-8')}\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(module_root)) + module = importlib.import_module("top_level_resource_tool") + try: + tool = module.ResourceTool() + before = _capability_runtime_identity(tool, "tool") + unrelated.write_text("PRIVATE=second", encoding="utf-8") + after = _capability_runtime_identity(tool, "tool") + payload = replay_runner_module._tool_executor_envelope(tool, "tool") + module_read_paths, _ = replay_runner_module._executor_module_allowlist(payload) + finally: + sys.modules.pop("top_level_resource_tool", None) + assert before == after + assert str(unrelated) not in module_read_paths + assert str(unrelated.resolve()) not in module_read_paths + +def test_loaded_unrelated_module_is_not_in_worker_allowlist( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module_root = tmp_path / "unrelated" + module_root.mkdir() + module_path = module_root / "unrelated_capability_module.py" + module_path.write_text("VALUE = 'unrelated'\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(module_root)) + importlib.import_module("unrelated_capability_module") + try: + payload = replay_runner_module._tool_executor_envelope(_AddTool(), "tool") + module_read_paths, module_specs = replay_runner_module._executor_module_allowlist(payload) + finally: + sys.modules.pop("unrelated_capability_module", None) + assert str(module_path) not in module_read_paths + assert "unrelated_capability_module" not in {name for name, _, _ in module_specs} + + + + +def test_worker_supports_lazy_top_level_sibling_imports(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + module_root = tmp_path / "capabilities" + module_root.mkdir() + (module_root / "lazy_top_level_helper.py").write_text("def result():\n return {'lazy_imported': True}\n", encoding="utf-8") + (module_root / "lazy_top_level_tool.py").write_text( + "class LazyTool:\n" + " def execute(self, _arguments):\n" + " from lazy_top_level_helper import result\n" + " return result()\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(module_root)) + module = importlib.import_module("lazy_top_level_tool") + try: + tool_schemas = ({"type": "function", "function": {"name": "lazy", "parameters": {"type": "object"}}},) + _, _, result = _run( + tmp_path / "workspace", + sequence=("lazy", "done"), + tool_schemas=tool_schemas, + tool_instances={"lazy": module.LazyTool()}, + ) + finally: + sys.modules.pop("lazy_top_level_helper", None) + sys.modules.pop("lazy_top_level_tool", None) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_unused_host_and_tool_capability_workers_are_not_started( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module_root = tmp_path / "capabilities" + module_root.mkdir() + (module_root / "unused_capabilities.py").write_text( + "import sys\n" + "from pathlib import Path\n" + "if sys.argv[0] == '-c':\n" + " Path('unused-worker-started.txt').write_text('started', encoding='utf-8')\n" + "class UnusedTool:\n" + " def execute(self, _arguments):\n" + " return {'used': True}\n" + "class UnusedHost:\n" + " def get_workspace(self):\n" + " return 'sandbox:fixture'\n" + " def replay_process_containment(self):\n" + " return {'mechanism': 'fixture-supervisor', 'detached_descendants': 'contained'}\n" + " def execute(self, command, **kwargs):\n" + " return {'exit_code': 0, 'cwd': kwargs['cwd']}\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(module_root)) + module = importlib.import_module("unused_capabilities") + try: + tool_schemas = ({"type": "function", "function": {"name": "unused", "parameters": {"type": "object"}}},) + workspace, _, result = _run( + tmp_path / "workspace", + sequence=("done",), + tool_schemas=tool_schemas, + tool_instances={"unused": module.UnusedTool()}, + host_instance=module.UnusedHost(), + ) + finally: + sys.modules.pop("unused_capabilities", None) + assert result.execution.as_dict()["terminal_status"] == "completed" + assert not (workspace.root / "unused-worker-started.txt").exists() + + +def test_lazy_sibling_module_content_is_bound_into_capability_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module_root = tmp_path / "capabilities" + module_root.mkdir() + helper = module_root / "identity_helper.py" + helper.write_text("def result():\n return {'version': 1}\n", encoding="utf-8") + (module_root / "identity_tool.py").write_text( + "class IdentityTool:\n" + " def execute(self, _arguments):\n" + " from identity_helper import result\n" + " return result()\n" + "class IdentityProvider:\n" + " def invoke(self, **_kwargs):\n" + " from identity_helper import result\n" + " return result()\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(module_root)) + module = importlib.import_module("identity_tool") + try: + before = _capability_runtime_identity(module.IdentityTool(), "tool") + provider_before = _capability_runtime_identity(module.IdentityProvider(), "provider") + helper.write_text("def result():\n return {'version': 2}\n", encoding="utf-8") + after = _capability_runtime_identity(module.IdentityTool(), "tool") + provider_after = _capability_runtime_identity(module.IdentityProvider(), "provider") + finally: + sys.modules.pop("identity_helper", None) + sys.modules.pop("identity_tool", None) + assert before != after + assert provider_before != provider_after + + +def test_provider_collaborator_modules_are_bound_into_route_identity( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + provider_root = tmp_path / "provider" + helper_root = tmp_path / "helper" + provider_root.mkdir() + helper_root.mkdir() + helper = helper_root / "external_state_helper.py" + helper.write_text("class Helper:\n def value(self):\n return 1\n", encoding="utf-8") + (provider_root / "stateful_provider.py").write_text( + "from external_state_helper import Helper\n" + "class StatefulProvider:\n" + " def __init__(self):\n" + " self.collaborator = Helper()\n" + " def invoke(self, **_kwargs):\n" + " return self.collaborator.value()\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(helper_root)) + monkeypatch.syspath_prepend(str(provider_root)) + module = importlib.import_module("stateful_provider") + try: + provider = module.StatefulProvider() + before = _capability_runtime_identity(provider, "provider") + helper.write_text("class Helper:\n def value(self):\n return 2\n", encoding="utf-8") + after = _capability_runtime_identity(provider, "provider") + finally: + sys.modules.pop("external_state_helper", None) + sys.modules.pop("stateful_provider", None) + assert before != after + + +def test_worker_stdio_cannot_bypass_evidence_redaction(tmp_path: Path, capfd: pytest.CaptureFixture[str]) -> None: + _, _, result = _run(tmp_path, sequence=("add", "done"), tool_instances={"add": _PrintingTool()}) + captured = capfd.readouterr() + assert result.execution.as_dict()["terminal_status"] == "completed" + assert "SECRET" not in captured.out + assert "SECRET" not in captured.err + + +def test_worker_os_sandbox_denies_writes_outside_workspace(tmp_path: Path) -> None: + outside = tmp_path / "escaped.txt" + _, _, result = _run(tmp_path / "workspace", sequence=("add",), tool_instances={"add": _AbsoluteWriteTool(outside)}) + record = result.execution.as_dict() + assert record["terminal_status"] == "tool_failed" + assert not outside.exists() + + +def test_worker_os_sandbox_denies_reads_outside_workspace(tmp_path: Path) -> None: + secret = "OUTSIDE_WORKSPACE_CONTENT" + outside = tmp_path / "outside.txt" + outside.write_text(secret, encoding="utf-8") + workspace, _, result = _run(tmp_path / "workspace", sequence=("add",), tool_instances={"add": _OutsideReadTool(outside)}) + assert result.execution.as_dict()["terminal_status"] == "tool_failed" + store = ArtifactStore(workspace.path(".breadboard/artifacts")) + for entry in result.manifest.as_dict()["entries"]: + assert secret.encode() not in store.read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"])) + + +def test_worker_closes_inherited_outside_file_descriptors(tmp_path: Path) -> None: + secret = "INHERITED_DESCRIPTOR_CONTENT" + outside = tmp_path / "inherited.txt" + outside.write_text(secret, encoding="utf-8") + tool = _InheritedFdReadTool(outside) + try: + workspace, _, result = _run(tmp_path / "workspace", sequence=("add",), tool_instances={"add": tool}) + finally: + __import__("os").close(tool.descriptor) + assert result.execution.as_dict()["terminal_status"] == "tool_failed" + store = ArtifactStore(workspace.path(".breadboard/artifacts")) + for entry in result.manifest.as_dict()["entries"]: + assert secret.encode() not in store.read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"])) +def test_worker_closes_inherited_socket_descriptors(tmp_path: Path) -> None: + tool = _InheritedSocketReadTool() + try: + workspace, _, result = _run(tmp_path, sequence=("add",), tool_instances={"add": tool}) + finally: + tool.close() + assert result.execution.as_dict()["terminal_status"] == "tool_failed" + store = ArtifactStore(workspace.path(".breadboard/artifacts")) + for entry in result.manifest.as_dict()["entries"]: + assert b"INHERITED_SOCKET_CONTENT" not in store.read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"])) + + +def test_workspace_special_file_is_rejected_from_snapshot(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("add",), tool_instances={"add": _WorkspaceFifoTool()}) + record = result.execution.as_dict() + assert record["terminal_status"] == "host_failed" + assert record["integrity_verified"] is False + assert record["problem"]["error_code"] == "replay.workspace_special_file" + assert result.manifest.as_dict()["publish_status"] == "quarantined" + +def test_workspace_snapshot_checks_deadline_between_file_chunks(tmp_path: Path) -> None: + workspace_file = tmp_path / "large.bin" + workspace_file.write_bytes(b"x" * (3 * 1024 * 1024)) + metadata = tmp_path.stat() + checks = [0] + + def check() -> None: + checks[0] += 1 + if checks[0] == 4: + raise replay_runner_module._RuntimeFailure("timed_out", "replay.total_timeout", "snapshot deadline") + + with pytest.raises(replay_runner_module._RuntimeFailure) as raised: + replay_runner_module._snapshot(tmp_path, (metadata.st_dev, metadata.st_ino), check=check) + assert raised.value.error_code == "replay.total_timeout" + assert checks == [4] + + +def test_exec_isolated_tool_cannot_reach_sibling_capabilities(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("add", "done"), tool_instances={"add": _FrameInspectionTool()}) + assert result.execution.as_dict()["terminal_status"] == "completed" + entry = next(row for row in result.manifest.as_dict()["entries"] if row["role"] == "tool_outcome") + outcome = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]))) + assert outcome["result"]["exposed"] == [] + + +def test_provider_worker_cannot_mutate_workspace(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("done",), provider_instance=_WritingProvider(("done",))) + assert result.execution.as_dict()["terminal_status"] == "provider_failed" + assert not list(tmp_path.rglob("provider-bypass.txt")) +def test_provider_worker_cannot_read_workspace(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("done",), provider_instance=_ReadingProvider(("done",))) + assert result.execution.as_dict()["terminal_status"] == "provider_failed" + + + + +def test_host_worker_denies_local_subprocess_execution(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("host",), host_instance=_SubprocessHost()) + assert result.execution.as_dict()["terminal_status"] == "host_failed" + assert not (workspace.root / "host-subprocess.txt").exists() + + +def test_host_worker_denies_subprocess_reads_outside_replay_workspace( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_checkout = tmp_path / "source-checkout" + source_checkout.mkdir() + outside = source_checkout / "outside-secret.txt" + outside.write_text("OUTSIDE_REPLAY_SECRET", encoding="utf-8") + monkeypatch.syspath_prepend(source_checkout) + workspace, _, result = _run( + source_checkout, + sequence=("host",), + host_instance=_OutsideReadingSubprocessHost(outside), + ) + assert result.execution.as_dict()["terminal_status"] == "host_failed" + payloads = [ + ArtifactStore(workspace.path(".breadboard/artifacts")).read( + ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]) + ) + for entry in result.manifest.as_dict()["entries"] + ] + assert all(b"OUTSIDE_REPLAY_SECRET" not in payload for payload in payloads) + + +def test_allowlisted_environment_is_provider_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("BB_REPLAY_ENV_SECRET", "PROVIDER_ONLY_SECRET") + workspace, _, result = _run(tmp_path, sequence=("add", "done"), tool_instances={"add": _EnvironmentReadTool()}, environment_allowlist=("BB_REPLAY_ENV_SECRET",)) + assert result.execution.as_dict()["terminal_status"] == "completed" + entry = next(row for row in result.manifest.as_dict()["entries"] if row["role"] == "tool_outcome") + outcome = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]))) + assert outcome["result"]["value"] is None + + +def test_provider_dump_logger_is_disabled_inside_replay_worker(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + outside = tmp_path / "provider-dumps" + monkeypatch.setenv("KC_PROVIDER_LOG_DIR", str(outside)) + previous = (provider_dump_logger.log_dir, provider_dump_logger.workspace_override, provider_dump_logger.session_override, provider_dump_logger.enabled) + provider_dump_logger.log_dir = str(outside) + provider_dump_logger.enabled = True + try: + _, _, result = _run(tmp_path / "workspace", sequence=("done",), provider_instance=_DumpingProvider(("done",)), environment_allowlist=("KC_PROVIDER_LOG_DIR",)) + finally: + provider_dump_logger.log_dir, provider_dump_logger.workspace_override, provider_dump_logger.session_override, provider_dump_logger.enabled = previous + assert result.execution.as_dict()["terminal_status"] == "completed" + assert not outside.exists() + + +def test_synthesizes_deterministic_missing_provider_call_ids(tmp_path: Path) -> None: + workspace, _, result = _run(tmp_path, sequence=("add_without_id", "done")) + assert result.execution.as_dict()["terminal_status"] == "completed" + entry = next(row for row in result.manifest.as_dict()["entries"] if row["role"] == "provider_response") + response = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]))) + assert response["messages"][0]["tool_calls"][0]["id"].endswith(":message:1:call:1") + + +def test_terminal_failure_details_are_redacted(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + secret = "SYMLINK_NAME_SECRET" + monkeypatch.setenv("BB_REPLAY_ENV_SECRET", secret) + _, _, result = _run(tmp_path, sequence=("add", "done"), tool_instances={"add": _SecretSymlinkTool(secret)}, environment_allowlist=("BB_REPLAY_ENV_SECRET",)) + execution = result.execution.as_dict() + assert execution["terminal_status"] == "host_failed" + assert secret not in json.dumps(execution, sort_keys=True) + assert "" in execution["completion_reason"] + + +def test_artifact_store_remains_anchored_during_namespace_swap(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + outside = tmp_path / "outside-artifacts" + outside.mkdir() + moved = tmp_path / "workspace" / ".breadboard" / "artifacts-original" + original = replay_runner_module._anchored_artifact_store + def swap_store(workspace: BreadBoardWorkspace): + store, descriptors = original(workspace) + artifact_path = workspace.root / ".breadboard" / "artifacts" + artifact_path.rename(moved) + artifact_path.symlink_to(outside, target_is_directory=True) + return store, descriptors + monkeypatch.setattr(replay_runner_module, "_anchored_artifact_store", swap_store) + _, _, result = _run(tmp_path / "workspace", sequence=("done",)) + assert result.execution.as_dict()["terminal_status"] == "completed" + assert not list(outside.iterdir()) + assert any(path.is_file() for path in moved.glob("sha256/*/*")) + + +def test_replay_parent_remains_anchored_during_namespace_swap(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + outside = tmp_path / "outside-replays" + outside.mkdir() + moved = tmp_path / "workspace" / ".breadboard" / "replays-original" + original = replay_runner_module._anchored_replay_parent + + def swap_parent(workspace: BreadBoardWorkspace): + replay_path, descriptors = original(workspace) + replay_path.rename(moved) + replay_path.symlink_to(outside, target_is_directory=True) + return replay_path, descriptors + + monkeypatch.setattr(replay_runner_module, "_anchored_replay_parent", swap_parent) + _, _, result = _run(tmp_path / "workspace", sequence=("done",)) + assert result.execution.as_dict()["terminal_status"] == "completed" + assert not list(outside.iterdir()) + assert any(path.name == "execution.json" for path in moved.glob("*/execution.json")) + + +def test_failed_publication_removes_swapped_final_name(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + outside = tmp_path / "outside-publication" + outside.mkdir() + original = replay_runner_module._rename_no_replace + moved: list[Path] = [] + def swap_stage(source: Path, destination: Path, *, parent_descriptor: int | None = None) -> None: + original_stage = source.with_name(source.name + ".original") + source.rename(original_stage) + moved.append(original_stage) + source.symlink_to(outside, target_is_directory=True) + original(source, destination, parent_descriptor=parent_descriptor) + monkeypatch.setattr(replay_runner_module, "_rename_no_replace", swap_stage) + with pytest.raises(ReplayRunError, match="published replay identity"): + _run(tmp_path / "workspace", sequence=("done",)) + assert moved and not moved[0].exists() + assert not list(outside.iterdir()) + assert not any((tmp_path / "workspace" / ".breadboard" / "replays").iterdir()) +def test_publication_fsyncs_parent_directory_after_rename(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + renamed = [False] + parent_synced = [False] + original_rename = replay_runner_module._rename_no_replace + original_fsync = replay_runner_module.os.fsync + + def track_rename(source: Path, destination: Path, *, parent_descriptor: int | None = None) -> None: + original_rename(source, destination, parent_descriptor=parent_descriptor) + renamed[0] = True + + def track_fsync(descriptor: int) -> None: + if renamed[0]: + parent_synced[0] = True + original_fsync(descriptor) + + monkeypatch.setattr(replay_runner_module, "_rename_no_replace", track_rename) + monkeypatch.setattr(replay_runner_module.os, "fsync", track_fsync) + _, _, result = _run(tmp_path, sequence=("done",)) + assert result.execution.as_dict()["terminal_status"] == "completed" + assert parent_synced == [True] + + + + + + +def test_worker_can_resolve_provider_hostnames(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("resolve_localhost",)) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_staging_cleanup_removes_special_filesystem_entries(tmp_path: Path) -> None: + _, _, result = _run(tmp_path, sequence=("add", "done"), tool_instances={"add": _StageFifoTool()}) + assert result.execution.as_dict()["terminal_status"] == "tool_failed" + assert not (result.execution_path.parent / "untracked.fifo").exists() + assert sorted(path.name for path in result.execution_path.parent.iterdir()) == ["execution.json", "manifest.json"] + + +def test_generated_execution_id_is_validated_before_path_creation(tmp_path: Path) -> None: + with pytest.raises(ReplayExecutionError, match="portable identifier"): + _run(tmp_path, sequence=("done",), ids=_BadIds()) + assert not (tmp_path / ".breadboard/replays").exists() + +def test_replay_rejects_unsupported_windows_backend_before_storage( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(replay_runner_module.sys, "platform", "win32") + with pytest.raises(ReplayRunError, match="Linux and macOS"): + _run(tmp_path, sequence=("done",)) + assert not (tmp_path / ".breadboard/replays").exists() + +def test_worker_resets_workspace_cwd_between_calls(tmp_path: Path) -> None: + outside = tmp_path / "outside" + outside.mkdir() + workspace, _, result = _run(tmp_path / "workspace", sequence=("add", "add", "done"), tool_instances={"add": _ChangingCwdTool(outside)}) + entry = next(row for row in result.manifest.as_dict()["entries"] if row["role"] == "workspace_after") + snapshot = json.loads(ArtifactStore(workspace.path(".breadboard/artifacts")).read(ArtifactRef(entry["sha256"], entry["size_bytes"], entry["media_type"]))) + assert "after-cwd-change.txt" in {row["path"] for row in snapshot["files"]} + assert not (outside / "after-cwd-change.txt").exists() + + +def test_active_runtime_bindings_must_match_frozen_plan(tmp_path: Path) -> None: + active = {name: {"binding": name} for name in ("capability_probe_sha256", "model_policy_sha256", "normalizer_config_sha256", "comparator_config_sha256")} + active["model_policy_sha256"] = {"binding": "changed"} + with pytest.raises(ReplayPlanError, match="model_policy_sha256"): + _run(tmp_path, sequence=("done",), runtime_bindings=active) + assert not (tmp_path / ".breadboard/replays").exists() + +def test_provider_context_must_match_frozen_model_policy_binding(tmp_path: Path) -> None: + planned_state = types.SimpleNamespace(provider_metadata_snapshot=lambda: {"previous_response_id": "response-1"}) + active_state = types.SimpleNamespace(provider_metadata_snapshot=lambda: {"previous_response_id": "response-2"}) + planned = ProviderRuntimeContext( + session_state=planned_state, + agent_config={"provider_tools": {"anthropic": {"temperature": 0.1}}}, + extra={"responses_extra": {"reasoning": {"effort": "low"}}}, + ) + active = ProviderRuntimeContext( + session_state=active_state, + agent_config={"provider_tools": {"anthropic": {"temperature": 0.9}}}, + extra={"responses_extra": {"reasoning": {"effort": "high"}}}, + ) + with pytest.raises(ReplayPlanError, match="model_policy_sha256"): + _run(tmp_path, sequence=("done",), provider_context=active, planned_provider_context=planned) + assert not (tmp_path / ".breadboard/replays").exists() + + +def test_redaction_covers_mapping_keys_and_rejects_collisions(tmp_path: Path) -> None: + counter = [0] + redacted = _redact({"preSECRETpost": "value"}, secrets=("SECRET",), workspace=tmp_path, counter=counter) + assert redacted == {"prepost": "value"} + assert counter == [1] + with pytest.raises(ReplayRunError, match="collide"): + _redact({"SECRET": 1, "": 2}, secrets=("SECRET",), workspace=tmp_path, counter=[0]) + + + + +def test_replay_namespace_symlink_is_rejected_without_writing_target(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + metadata = workspace / ".breadboard" + outside = tmp_path / "outside" + metadata.mkdir(parents=True) + outside.mkdir() + (metadata / "replays").symlink_to(outside, target_is_directory=True) + with pytest.raises((ReplayRunError, WorkspacePathError)): + _run(workspace, sequence=("done",)) + assert not list(outside.iterdir()) + + +def test_provider_metadata_is_not_applied_before_result_validation(tmp_path: Path) -> None: + marker = tmp_path / "provider-metadata.json" + context = types.SimpleNamespace(session_state=_MetadataSink(marker)) + _, _, result = _run(tmp_path / "workspace", sequence=("invalid_message",), provider_context=context) + assert result.execution.as_dict()["terminal_status"] == "provider_failed" + assert not marker.exists() +def test_provider_metadata_is_not_applied_when_exchange_is_cancelled(tmp_path: Path) -> None: + marker = tmp_path / "provider-metadata.json" + context = types.SimpleNamespace(session_state=_MetadataSink(marker)) + _, _, result = _run( + tmp_path / "workspace", + sequence=("done",), + provider_context=context, + cancelled=_CancelAfterProvider(), + ) + assert result.execution.as_dict()["terminal_status"] == "cancelled" + assert not marker.exists() + + +def test_provider_metadata_is_not_applied_when_exchange_times_out(tmp_path: Path) -> None: + marker = tmp_path / "provider-metadata.json" + context = types.SimpleNamespace(session_state=_MetadataSink(marker)) + _, _, result = _run( + tmp_path / "workspace", + sequence=("done",), + provider_context=context, + deadlines={"total": 10_000, "idle": 5_000, "provider_call": 999, "tool_call": 1_000}, + monotonic=_SlowMonotonic(), + ) + assert result.execution.as_dict()["terminal_status"] == "timed_out" + assert not marker.exists() + + + + +def test_provider_metadata_is_not_applied_when_later_tool_fails(tmp_path: Path) -> None: + marker = tmp_path / "provider-metadata.json" + context = types.SimpleNamespace(session_state=_MetadataSink(marker)) + _, _, result = _run(tmp_path / "workspace", sequence=("add",), provider_context=context, tool_fail=True) + assert result.execution.as_dict()["terminal_status"] == "tool_failed" + assert not marker.exists() + + +def test_provider_metadata_is_applied_after_result_validation(tmp_path: Path) -> None: + marker = tmp_path / "provider-metadata.json" + context = types.SimpleNamespace(session_state=_MetadataSink(marker)) + _, _, result = _run(tmp_path / "workspace", sequence=("done",), provider_context=context) + assert result.execution.as_dict()["terminal_status"] == "completed" + assert json.loads(marker.read_text(encoding="utf-8")) == {"api_key": "SECRET"} + + +def test_usage_accounting_is_exact_at_large_integer_and_decimal_boundaries() -> None: + accountant = replay_runner_module._UsageAccountant.from_budgets({"tokens": 9_007_199_254_740_993, "cost": 0.3}) + assert accountant.add({"total_tokens": 9_007_199_254_740_992, "cost_amount": 0.1, "cost_currency": "USD"}) is False + assert accountant.add({"total_tokens": 1, "cost_amount": 0.2, "cost_currency": "USD"}) is False + assert accountant.add({"total_tokens": 1, "cost_amount": 0, "cost_currency": "USD"}) is True + + +def test_redaction_rejects_recursive_containers(tmp_path: Path) -> None: + recursive: list[Any] = [] + recursive.append(recursive) + with pytest.raises(ReplayRunError, match="cyclic"): + _redact(recursive, secrets=(), workspace=tmp_path, counter=[0]) + + +def test_normalized_usage_preserves_large_integer_cost_exactly() -> None: + value = 9_007_199_254_740_993 + usage = normalized_provider_usage(ProviderResult([ProviderMessage("assistant", "done")], None, {"cost_amount": value}, metadata={})) + assert usage["cost_amount"] == value + assert type(usage["cost_amount"]) is int + + +def test_real_session_provider_context_excludes_runtime_only_metadata(tmp_path: Path) -> None: + from agentic_coder_prototype.state.session_state import SessionState + session_state = SessionState(str(tmp_path), "fixture") + session_state.set_provider_metadata("anthropic_rate_limits", {"tokens_remaining": 17}) + session_state.set_provider_metadata("conversation_id", "conversation-fixture") + session_state.set_provider_metadata("current_turn_index", 3) + session_state.set_provider_metadata("previous_response_id", "response-fixture") + session_state.set_provider_metadata("control_queue", object()) + context = ProviderRuntimeContext(session_state=session_state, agent_config={}, extra={}) + _, _, result = _run( + tmp_path / "workspace", + sequence=("done",), + provider_instance=_ProviderMetadataInspectingProvider(("done",)), + provider_context=context, + ) + assert result.execution.as_dict()["terminal_status"] == "completed" + + +def test_provider_context_strips_execution_capabilities(tmp_path: Path) -> None: + marker = tmp_path / "provider-capability-called" + provider = _ContextInspectingProvider(("done",)) + context = types.SimpleNamespace(session_state=None, agent_config={}, extra={"host": _ContextCapability(marker)}) + _, _, result = _run(tmp_path / "workspace", sequence=("done",), provider_instance=provider, provider_context=context) + assert result.execution.as_dict()["terminal_status"] == "completed" + assert not marker.exists() + +def test_callable_executor_identity_includes_mutable_object_state() -> None: + tool = _CallableConfiguredTool(1) + before = _runtime_type_identity(tool) + tool.value = 2 + after = _runtime_type_identity(tool) + assert before["configuration_sha256"] != after["configuration_sha256"] + + + +def test_scenario_rejects_invalid_tool_schema_before_plan_construction() -> None: + with pytest.raises(ReplayRunError, match="invalid parameters schema"): + ReplayScenario( + "fixture", + ({"role": "user", "content": "go"},), + {}, + ({"type": "function", "function": {"name": "bad", "parameters": {"type": "not-a-json-schema-type"}}},), + ) + + +@pytest.mark.parametrize("location", ("C:relative", "C:/absolute")) +def test_manifest_rejects_windows_drive_prefixed_locations(tmp_path: Path, location: str) -> None: + _, _, result = _run(tmp_path, sequence=("done",)) + malformed = result.manifest.as_dict() + malformed["entries"][0]["location_kind"] = "workspace_relative_path" + malformed["entries"][0]["location"] = location + with pytest.raises(ReplayExecutionError, match="workspace_relative_path"): + ReplayArtifactManifest.from_dict(malformed) + + +@pytest.mark.parametrize("currency", ("KSD", "ＵSD", "İSD")) +def test_usage_rejects_unicode_uppercase_currency(currency: str) -> None: + result = ProviderResult([ProviderMessage("assistant", "done")], None, {"cost_amount": 1, "cost_currency": currency}, metadata={}) + with pytest.raises(ProviderRuntimeError, match="three-letter uppercase"): + normalized_provider_usage(result) + + +def test_usage_rejects_unrepresentable_integer_cost() -> None: + result = ProviderResult([ProviderMessage("assistant", "done")], None, {"cost_amount": 10 ** 10_000, "cost_currency": "USD"}, metadata={}) + with pytest.raises(ProviderRuntimeError, match="representable"): + normalized_provider_usage(result) + + +def test_stored_usage_rejects_unrepresentable_integer_cost() -> None: + with pytest.raises(ReplayExecutionError, match="representable"): + _validate_usage({"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cost_amount": 10 ** 10_000, "cost_currency": "USD"}) diff --git a/tests/product/integrations/test_catalog.py b/tests/product/integrations/test_catalog.py index 5ef6cdd6..46678bb7 100644 --- a/tests/product/integrations/test_catalog.py +++ b/tests/product/integrations/test_catalog.py @@ -4,7 +4,10 @@ import json import pytest from jsonschema import Draft202012Validator +from agentic_coder_prototype.provider.routing import ProviderDescriptor from breadboard.product.integrations import CaptureIntegrationAdapter, IncompatibleAdapterError, IntegrationCatalog, IntegrationDescriptor, IntegrationError, ProbeReport, ProjectDeclarationError, internal_capture_adapters, load_capture_entry_points, resolve_local_capture_declaration +from breadboard.product.integrations.provider import ProviderRuntimeAdapter +from breadboard.product.integrations.host import SandboxHostAdapter def test_internal_and_external_adapters(monkeypatch: pytest.MonkeyPatch) -> None: @@ -91,3 +94,60 @@ def test_records_match_public_schemas_and_traversal_is_rejected(tmp_path: Path) assert list(Draft202012Validator(schema).iter_errors(record)) == [] declaration = {"adapter_id": "local", "source_path": "../adapter.py", "source_sha256": "sha256:" + "0" * 64, "grants": ["capture"]} with pytest.raises(ProjectDeclarationError): resolve_local_capture_declaration(declaration, project_root=str(tmp_path), adapter=internal_capture_adapters()[0]) + + +def test_provider_replay_client_identity_binds_creation_options() -> None: + descriptor = ProviderDescriptor("fixture", "fixture_runtime", "chat", True, False, False, False, "openai", None, "FIXTURE_KEY", {}) + + class Runtime: + def create_client(self, api_key: str, *, base_url: str | None = None, default_headers: dict[str, str] | None = None): + return SimpleNamespace(api_key=api_key, base_url=base_url, timeout=10, max_retries=2) + def invoke(self, **kwargs): + raise AssertionError("not used") + + adapter = ProviderRuntimeAdapter(Runtime(), descriptor) + first = adapter.create_client("secret", base_url="https://fixture.invalid", default_headers={"X-Tenant": "one"}) + second = adapter.create_client("secret", base_url="https://fixture.invalid", default_headers={"X-Tenant": "two"}) + first_identity = adapter.replay_client_identity(first, {"FIXTURE_KEY": "secret"}) + second_identity = adapter.replay_client_identity(second, {"FIXTURE_KEY": "secret"}) + assert first_identity["verified"] is True + assert first_identity["creation_options"] != second_identity["creation_options"] + assert adapter.replay_client_identity(SimpleNamespace(api_key="secret"), {"FIXTURE_KEY": "secret"})["verified"] is False + assert "secret" not in json.dumps(first_identity) + assert "one" not in json.dumps(first_identity) + + +def test_provider_replay_client_identity_supports_mapping_clients() -> None: + descriptor = ProviderDescriptor("fixture", "fixture_runtime", "chat", True, False, False, False, "openai", None, "FIXTURE_KEY", {}) + + class Runtime: + def create_client(self, api_key: str, *, base_url: str | None = None, default_headers: dict[str, str] | None = None): + return {"api_key": api_key, "base_url": base_url} + def invoke(self, **kwargs): + raise AssertionError("not used") + + adapter = ProviderRuntimeAdapter(Runtime(), descriptor) + client = adapter.create_client("secret", base_url="https://fixture.invalid") + assert adapter.replay_client_identity(client, {"FIXTURE_KEY": "secret"})["verified"] is True + client["api_key"] = "changed" + assert adapter.replay_client_identity(client, {"FIXTURE_KEY": "secret"})["verified"] is False + client["api_key"] = "secret" + client["base_url"] = "https://changed.invalid" + assert adapter.replay_client_identity(client, {"FIXTURE_KEY": "secret"})["verified"] is False + + +def test_host_adapter_rejects_missing_replay_containment_port() -> None: + sandbox = SimpleNamespace(get_workspace=lambda: "sandbox:fixture", execute=lambda command, **kwargs: None) + with pytest.raises(TypeError, match="replay_process_containment"): + SandboxHostAdapter("fixture", sandbox) + + +def test_host_probe_fails_closed_on_invalid_replay_containment_attestation() -> None: + sandbox = SimpleNamespace( + get_workspace=lambda: "sandbox:fixture", + execute=lambda command, **kwargs: None, + replay_process_containment=lambda: {"detached_descendants": "uncontained"}, + ) + report = SandboxHostAdapter("fixture", sandbox).probe() + assert report.status == "unavailable" + assert report.error == "RuntimeError" diff --git a/tests/providers/test_provider_message_contracts.py b/tests/providers/test_provider_message_contracts.py index 5cae97ce..04484bee 100644 --- a/tests/providers/test_provider_message_contracts.py +++ b/tests/providers/test_provider_message_contracts.py @@ -130,7 +130,7 @@ def __init__(self, **kwargs): assert call.arguments.startswith("{") and call.arguments.endswith("}") -def test_openai_chat_runtime_converts_null_content_to_empty_string() -> None: +def test_openai_chat_runtime_preserves_tool_transcript_and_normalizes_content() -> None: descriptor, _model = provider_router.get_runtime_descriptor("openai/gpt-4o-mini") runtime = provider_registry.create_runtime(descriptor) converted = runtime._convert_messages_to_chat( @@ -145,12 +145,19 @@ def test_openai_chat_runtime_converts_null_content_to_empty_string() -> None: "function": {"name": "tool_a", "arguments": "{}"}, } ], - } + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "{\"ok\": true}", + }, ] ) assert isinstance(converted, list) and converted assert converted[0]["role"] == "assistant" assert converted[0]["content"] == "" + assert converted[0]["tool_calls"][0]["id"] == "call_1" + assert converted[1] == {"role": "tool", "content": "{\"ok\": true}", "tool_call_id": "call_1"} def test_responses_runtime_produces_string_content(monkeypatch): diff --git a/tests/providers/test_provider_runtime_anthropic.py b/tests/providers/test_provider_runtime_anthropic.py index 3f4e7c7c..68a45d62 100644 --- a/tests/providers/test_provider_runtime_anthropic.py +++ b/tests/providers/test_provider_runtime_anthropic.py @@ -22,10 +22,10 @@ def set_provider_metadata(self, key: str, value): self._metadata[key] = value -def _anthropic_tool_schema(): +def _anthropic_tool_schema(name="fetch_data"): return [ { - "name": "fetch_data", + "name": name, "description": "Fetch data", "input_schema": { "type": "object", @@ -35,6 +35,23 @@ def _anthropic_tool_schema(): } ] +def _openai_tool_schema(name="fetch_data"): + return [ + { + "type": "function", + "function": { + "name": name, + "description": "Fetch data", + "parameters": { + "type": "object", + "properties": {}, + "required": [], + }, + }, + } + ] + + def test_anthropic_runtime_stream_success(monkeypatch): descriptor, model = provider_router.get_runtime_descriptor("anthropic/claude-3-opus") @@ -46,7 +63,7 @@ def test_anthropic_runtime_stream_success(monkeypatch): { "type": "tool_use", "id": "call-1", - "name": "fetch_data", + "name": "host_execute", "input": {"foo": "bar"}, }, {"type": "thinking", "text": "analysis"}, @@ -76,6 +93,8 @@ def get_final_usage(self): class FakeMessages: def stream(self, **kwargs): assert kwargs["model"] == model + assert kwargs["tools"] == _anthropic_tool_schema("host_execute") + assert kwargs["tool_choice"] == {"type": "tool", "name": "host_execute"} return FakeStream() def create(self, **kwargs): @@ -93,7 +112,7 @@ def __init__(self, **kwargs): client = runtime.create_client("fake-key") context = ProviderRuntimeContext( session_state=_DummySessionState(), - agent_config={}, + agent_config={"provider_tools": {"anthropic": {"tool_choice": {"type": "tool", "name": "host.execute"}}}}, stream=True, ) @@ -101,18 +120,58 @@ def __init__(self, **kwargs): client=client, model=model, messages=[{"role": "user", "content": "Hi"}], - tools=_anthropic_tool_schema(), + tools=_openai_tool_schema("host.execute"), stream=True, context=context, ) assert result.messages[0].content == "Hello" - assert result.messages[0].tool_calls[0].name == "fetch_data" + assert result.messages[0].tool_calls[0].name == "host.execute" assert result.reasoning_summaries == ["analysis"] assert result.usage == {"input_tokens": 12, "output_tokens": 34} assert result.metadata["usage"]["output_tokens"] == 34 +def test_anthropic_runtime_reuses_host_tool_alias_in_transcript() -> None: + descriptor, _ = provider_router.get_runtime_descriptor("anthropic/claude-3-opus") + runtime = provider_registry.create_runtime(descriptor) + _, converted = runtime._convert_messages( + [{"role": "assistant", "content": "", "tool_calls": [{"id": "call-1", "function": {"name": "host.execute", "arguments": "{}"}}]}], + {"host.execute": "host_execute"}, + ) + assert converted[0]["content"][0]["name"] == "host_execute" + + +def test_anthropic_host_tool_alias_avoids_declared_name_collisions() -> None: + descriptor, _ = provider_router.get_runtime_descriptor("anthropic/claude-3-opus") + runtime = provider_registry.create_runtime(descriptor) + context = ProviderRuntimeContext(session_state=_DummySessionState(), agent_config={}, stream=False) + _, first_aliases = runtime._filter_anthropic_tools( + [*_openai_tool_schema("host.execute"), *_openai_tool_schema("host_execute")], + context, + ) + generated = first_aliases["host.execute"] + filtered, aliases = runtime._filter_anthropic_tools( + [*_openai_tool_schema("host.execute"), *_openai_tool_schema("host_execute"), *_openai_tool_schema(generated)], + context, + ) + names = [tool["name"] for tool in filtered] + assert len(names) == len(set(names)) + assert aliases["host.execute"] not in {"host_execute", generated} + +def test_anthropic_filter_drops_malformed_function_tool_without_crashing() -> None: + descriptor, _ = provider_router.get_runtime_descriptor("anthropic/claude-3-opus") + runtime = provider_registry.create_runtime(descriptor) + context = ProviderRuntimeContext(session_state=_DummySessionState(), agent_config={}, stream=False) + filtered, aliases = runtime._filter_anthropic_tools( + [{"type": "function", "function": "not-a-mapping"}], + context, + ) + assert filtered is None + assert aliases == {} + + + def test_anthropic_runtime_stream_error(monkeypatch): descriptor, model = provider_router.get_runtime_descriptor("anthropic/claude-3-opus") runtime = provider_registry.create_runtime(descriptor)