|
| 1 | +"""Pure D3 evidence helpers, including the shared exact-bundle validator.""" |
| 2 | +from __future__ import annotations |
| 3 | + |
| 4 | +import hashlib |
| 5 | +import json |
| 6 | +import os |
| 7 | +import re |
| 8 | +import stat |
| 9 | +from collections.abc import Iterable, Mapping |
| 10 | +from dataclasses import dataclass |
| 11 | +from pathlib import Path |
| 12 | + |
| 13 | +FIXED_FILES = ("manifest.json", "report.html", "report.json") |
| 14 | +EVIDENCE_VERSION = "qar.d3.build_evidence.v4" |
| 15 | +DIST_RE = re.compile(r"^([A-Za-z0-9_.-]+)==([^\s=]+)$") |
| 16 | +DEPENDENCY_INVENTORY = [ |
| 17 | + ".github/workflows/qar_d3_daily_preview_artifact.yml", "pyproject.toml", "uv.lock", |
| 18 | + "scripts/d3_build_daily_preview.py", "scripts/d3_verify_daily_preview.py", "scripts/d3_evidence.py", |
| 19 | + "src/quant_advisor_research/advisory_report.py", "src/quant_advisor_research/artifact_integrity.py", |
| 20 | + "src/quant_advisor_research/artifacts.py", "src/quant_advisor_research/contracts.py", |
| 21 | + "src/quant_advisor_research/csv_utils.py", "src/quant_advisor_research/period_contract.py", |
| 22 | + "src/quant_advisor_research/preview_bundle.py", "src/quant_advisor_research/preview_workspace.py", |
| 23 | + "src/quant_advisor_research/time_contract.py", "tests/test_d3_exact_bundle.py", |
| 24 | + "examples/political_events.example.csv", "examples/political_watchlist.example.csv", |
| 25 | +] |
| 26 | + |
| 27 | + |
| 28 | +class EvidenceContractError(ValueError): |
| 29 | + def __init__(self, code: str) -> None: |
| 30 | + self.code = code |
| 31 | + super().__init__(code) |
| 32 | + |
| 33 | + |
| 34 | +@dataclass(frozen=True, slots=True) |
| 35 | +class BundleMemberSnapshot: |
| 36 | + name: str |
| 37 | + content: bytes |
| 38 | + sha256: str |
| 39 | + |
| 40 | + |
| 41 | +@dataclass(frozen=True, slots=True) |
| 42 | +class BundleSnapshot: |
| 43 | + members: tuple[BundleMemberSnapshot, ...] |
| 44 | + |
| 45 | + def member(self, name: str) -> BundleMemberSnapshot: |
| 46 | + for item in self.members: |
| 47 | + if item.name == name: |
| 48 | + return item |
| 49 | + raise EvidenceContractError("bundle_member_set_invalid") |
| 50 | + |
| 51 | + |
| 52 | +def _directory_flags() -> int: |
| 53 | + required = ("O_DIRECTORY", "O_CLOEXEC", "O_NOFOLLOW") |
| 54 | + if any(not hasattr(os, name) for name in required): |
| 55 | + raise EvidenceContractError("filesystem_unsupported") |
| 56 | + return os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC | os.O_NOFOLLOW |
| 57 | + |
| 58 | + |
| 59 | +def _member_flags() -> int: |
| 60 | + if any(not hasattr(os, name) for name in ("O_CLOEXEC", "O_NOFOLLOW")): |
| 61 | + raise EvidenceContractError("filesystem_unsupported") |
| 62 | + return os.O_RDONLY | os.O_NONBLOCK | os.O_CLOEXEC | os.O_NOFOLLOW |
| 63 | + |
| 64 | + |
| 65 | +def validate_exact_bundle(workspace: str | Path) -> BundleSnapshot: |
| 66 | + """Return an immutable FD-bound snapshot after validating all members.""" |
| 67 | + root_fd = -1 |
| 68 | + try: |
| 69 | + root_fd = os.open(workspace, _directory_flags()) |
| 70 | + root_info = os.fstat(root_fd) |
| 71 | + if not stat.S_ISDIR(root_info.st_mode) or stat.S_ISLNK(root_info.st_mode): |
| 72 | + raise EvidenceContractError("bundle_directory_invalid") |
| 73 | + names = os.listdir(root_fd) |
| 74 | + if set(names) != set(FIXED_FILES) or len(names) != len(FIXED_FILES): |
| 75 | + raise EvidenceContractError("bundle_member_set_invalid") |
| 76 | + result: list[BundleMemberSnapshot] = [] |
| 77 | + for name in FIXED_FILES: |
| 78 | + fd = -1 |
| 79 | + try: |
| 80 | + fd = os.open(name, _member_flags(), dir_fd=root_fd) |
| 81 | + info = os.fstat(fd) |
| 82 | + if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: |
| 83 | + raise EvidenceContractError("bundle_member_invalid") |
| 84 | + chunks: list[bytes] = [] |
| 85 | + while chunk := os.read(fd, 1024 * 1024): |
| 86 | + chunks.append(chunk) |
| 87 | + content = b"".join(chunks) |
| 88 | + result.append(BundleMemberSnapshot(name, content, hashlib.sha256(content).hexdigest())) |
| 89 | + finally: |
| 90 | + if fd >= 0: |
| 91 | + os.close(fd) |
| 92 | + if set(os.listdir(root_fd)) != set(FIXED_FILES): |
| 93 | + raise EvidenceContractError("bundle_member_set_invalid") |
| 94 | + return BundleSnapshot(tuple(result)) |
| 95 | + except EvidenceContractError: |
| 96 | + raise |
| 97 | + except (OSError, TypeError, ValueError): |
| 98 | + raise EvidenceContractError("bundle_member_invalid") from None |
| 99 | + finally: |
| 100 | + if root_fd >= 0: |
| 101 | + os.close(root_fd) |
| 102 | + |
| 103 | + |
| 104 | +def validate_preview_snapshot(snapshot: BundleSnapshot) -> tuple[Mapping[str, object], Mapping[str, object]]: |
| 105 | + """Readback validation using only the immutable member bytes.""" |
| 106 | + try: |
| 107 | + from quant_advisor_research.preview_bundle import _canonical_json, _manifest, _render_html, _validated_source |
| 108 | + report_bytes = snapshot.member("report.json").content |
| 109 | + html_bytes = snapshot.member("report.html").content |
| 110 | + manifest_bytes = snapshot.member("manifest.json").content |
| 111 | + report = json.loads(report_bytes.decode("utf-8")) |
| 112 | + if not isinstance(report, Mapping): |
| 113 | + raise EvidenceContractError("readback_invalid") |
| 114 | + validated = _validated_source(report) |
| 115 | + if _canonical_json(validated) != report_bytes: |
| 116 | + raise EvidenceContractError("report_bytes_noncanonical") |
| 117 | + manifest = json.loads(manifest_bytes.decode("utf-8")) |
| 118 | + if not isinstance(manifest, Mapping) or manifest != _manifest(validated, report_bytes, html_bytes): |
| 119 | + raise EvidenceContractError("manifest_mismatch") |
| 120 | + if html_bytes != _render_html(validated, report_bytes) or html_bytes.count(b'href="report.json"') != 1 or html_bytes.count(b'href="manifest.json"') != 1: |
| 121 | + raise EvidenceContractError("html_links_invalid") |
| 122 | + return validated, manifest |
| 123 | + except EvidenceContractError: |
| 124 | + raise |
| 125 | + except (UnicodeError, json.JSONDecodeError, TypeError, ValueError, OverflowError, RecursionError): |
| 126 | + raise EvidenceContractError("readback_invalid") from None |
| 127 | + |
| 128 | + |
| 129 | +def canonical_distribution_snapshot(values: Iterable[str]) -> tuple[list[str], str]: |
| 130 | + try: |
| 131 | + raw = list(values) |
| 132 | + normalized: dict[str, str] = {} |
| 133 | + for value in raw: |
| 134 | + if type(value) is not str or not (match := DIST_RE.fullmatch(value)): |
| 135 | + raise EvidenceContractError("distribution_snapshot_invalid") |
| 136 | + name = re.sub(r"[-_.]+", "-", match.group(1)).lower() |
| 137 | + item = f"{name}=={match.group(2)}" |
| 138 | + if name in normalized and normalized[name] != item: |
| 139 | + raise EvidenceContractError("distribution_snapshot_conflict") |
| 140 | + normalized[name] = item |
| 141 | + snapshot = sorted(normalized.values()) |
| 142 | + digest = hashlib.sha256("\n".join(snapshot).encode()).hexdigest() |
| 143 | + return snapshot, digest |
| 144 | + except EvidenceContractError: |
| 145 | + raise |
| 146 | + except (TypeError, ValueError, UnicodeError): |
| 147 | + raise EvidenceContractError("distribution_snapshot_invalid") from None |
| 148 | + |
| 149 | + |
| 150 | +def locked_environment_evidence(*, lock_sha256: str, uv_version: str, python_version: str, distributions: Iterable[str]) -> dict[str, object]: |
| 151 | + if type(lock_sha256) is not str or not re.fullmatch(r"[0-9a-f]{64}", lock_sha256): |
| 152 | + raise EvidenceContractError("lock_digest_invalid") |
| 153 | + if type(uv_version) is not str or not uv_version.startswith("uv "): |
| 154 | + raise EvidenceContractError("uv_version_invalid") |
| 155 | + if type(python_version) is not str or not re.fullmatch(r"3\.11(?:\.\d+)?", python_version): |
| 156 | + raise EvidenceContractError("python_version_invalid") |
| 157 | + snapshot, digest = canonical_distribution_snapshot(distributions) |
| 158 | + return {"lock_sha256": lock_sha256, "uv_version": uv_version, "python_version": python_version, "installed_distributions": snapshot, "installed_distributions_sha256": digest} |
| 159 | + |
| 160 | + |
| 161 | +def repository_file_hashes(repo_root: str | Path, paths: Iterable[str]) -> dict[str, str]: |
| 162 | + root_fd = -1 |
| 163 | + opened: list[int] = [] |
| 164 | + try: |
| 165 | + root_fd = os.open(repo_root, _directory_flags()) |
| 166 | + if not stat.S_ISDIR(os.fstat(root_fd).st_mode): |
| 167 | + raise EvidenceContractError("dependency_root_invalid") |
| 168 | + result: dict[str, str] = {} |
| 169 | + for raw in paths: |
| 170 | + relative = Path(raw) |
| 171 | + if type(raw) is not str or not raw or relative.is_absolute() or any(part in ("", ".", "..") for part in relative.parts) or raw in result: |
| 172 | + raise EvidenceContractError("dependency_path_invalid") |
| 173 | + parent_fd = root_fd |
| 174 | + traversed: list[int] = [] |
| 175 | + try: |
| 176 | + for part in relative.parts[:-1]: |
| 177 | + next_fd = os.open(part, _directory_flags(), dir_fd=parent_fd) |
| 178 | + traversed.append(next_fd) |
| 179 | + parent_fd = next_fd |
| 180 | + file_fd = os.open(relative.parts[-1], _member_flags(), dir_fd=parent_fd) |
| 181 | + opened.append(file_fd) |
| 182 | + info = os.fstat(file_fd) |
| 183 | + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: |
| 184 | + raise EvidenceContractError("dependency_file_invalid") |
| 185 | + digest = hashlib.sha256() |
| 186 | + while chunk := os.read(file_fd, 1024 * 1024): |
| 187 | + digest.update(chunk) |
| 188 | + result[raw] = digest.hexdigest() |
| 189 | + finally: |
| 190 | + for fd in reversed(traversed): |
| 191 | + os.close(fd) |
| 192 | + return dict(sorted(result.items())) |
| 193 | + except EvidenceContractError: |
| 194 | + raise |
| 195 | + except (OSError, TypeError, ValueError, UnicodeError): |
| 196 | + raise EvidenceContractError("dependency_file_invalid") from None |
| 197 | + finally: |
| 198 | + for fd in opened: |
| 199 | + os.close(fd) |
| 200 | + if root_fd >= 0: |
| 201 | + os.close(root_fd) |
| 202 | + |
| 203 | + |
| 204 | +def require_exact_locked_environment(value: object, expected: Mapping[str, object]) -> None: |
| 205 | + if not isinstance(value, Mapping) or dict(value) != dict(expected): |
| 206 | + raise EvidenceContractError("locked_environment_mismatch") |
0 commit comments