R1.0.1 contains no Myth package. Generic Myth v0.2.0 and ' + 'Full-Canon Myth v0.3.5 are separate optional default-off ' + 'nonauthorizing companions.
' + 'release' + 'capa' + 'direct' + ) + checks = VERIFIER.public_site_semantic_checks(requirement, {"/": source}) + self.assertFalse(checks["no_direct_github_release_link"]) + def test_generic_exact_identity_mutation_fails_closed(self) -> None: manifest = json.loads(VERIFIER.PUBLICATION_MANIFEST.read_text(encoding="utf-8")) generic = next( diff --git a/tools/verify_generic_myth_v0_2_0.py b/tools/verify_generic_myth_v0_2_0.py new file mode 100644 index 0000000..a676603 --- /dev/null +++ b/tools/verify_generic_myth_v0_2_0.py @@ -0,0 +1,596 @@ +#!/usr/bin/env python3 +"""Fail-closed verifier for Generic Myth Sidecar v0.2.0. + +The outer published SHA-256 remains the release identity. This verifier adds +defence in depth: an exact path inventory, strict checksum coverage, safe ZIP +metadata, pinned immutable content, semantic control validation, and scans for +the non-public predecessor or non-public classification language. +""" +from __future__ import annotations + +import hashlib +import json +import re +import stat +import sys +import zipfile +from pathlib import Path, PurePosixPath +from typing import Any + +ROOT_NAME = ( + "Project_Shadow_Generic_Myth_Sidecar_v0.2.0_" + "OPTIONAL_PUBLIC_COMPANION_2026-08-17" +) +FIXED_TIMESTAMP = (2026, 8, 17, 0, 0, 0) +OLD_PREDECESSOR_SHA256 = ( + "4d8481ed8f6af3e04d86a5c2de7e94aad47a591e1ce9c8757bbb4b0a397b90bb" +) +OLD_PREDECESSOR_SIZE_BYTES = 29763 +APACHE_2_LICENSE_SHA256 = ( + "8c6db340475136df3c1201d458fa5755698eace76e510471ecc9d857d6083dac" +) +MAX_MEMBER_BYTES = 1_000_000 +MAX_TOTAL_BYTES = 2_000_000 + +EXPECTED_PATHS = frozenset({ + "BUILD_RECEIPT.json", + "CHANGELOG.md", + "CONTROL_BOUNDARY.md", + "LICENSE", + "LICENSE-DOCS", + "NEGATIVE_TEST_RESULTS.json", + "NOTICE", + "PACKAGE_INVENTORY.json", + "PACKAGE_MANIFEST.json", + "PREDECESSOR_CUSTODY.json", + "README_FIRST.md", + "RIGHTS_AND_PROVENANCE.md", + "RIGHTS_SCAN_RESULT.json", + "SHA256SUMS", + "STATUS_AND_NONAUTHORITY.md", + "TEST_RESULTS.json", + "myth_sidecar.py", + "tests/__init__.py", + "tests/test_myth_sidecar.py", + "tools/build_release.py", + "tools/rights_scan.py", + "tools/run_negative_verifier_tests.py", + "tools/verify_package.py", +}) + +# Filled only for immutable, non-generated members. SHA256SUMS binds every +# member; these pins additionally prevent a resealed archive from substituting +# expected-path content. The verifier cannot self-pin, so its published outer +# archive hash remains the trust anchor. +PINNED_SHA256: dict[str, str] = { + "CHANGELOG.md": "ccb0f4dfd526a3b2d1ce3cc52515b5611ba9b74573d5885e8f5db414b0a98b22", + "CONTROL_BOUNDARY.md": "86094cf86808aa3f44aee39b6b8ab8a7e6797c47d7f03f468583f5a5487f355c", + "LICENSE": "8c6db340475136df3c1201d458fa5755698eace76e510471ecc9d857d6083dac", + "LICENSE-DOCS": "305d3725873a52f0620f4fcec8b2fa87798bcff22b05fdf94ab645037c164cc5", + "NOTICE": "098f995db445de9c4f77ff7137ea6883239dc7bd1f76d4be7ee40280572934d5", + "PACKAGE_INVENTORY.json": "c52c8796e03003a06389d34bd99e8840ff7f0b7fad8ed80ee648f8f86338f861", + "PACKAGE_MANIFEST.json": "1bef9d75d5f4b3927f5b23bffd53f5aad6741c7b5dbc1ee4258b3ff95c524667", + "PREDECESSOR_CUSTODY.json": "bc9a25ef3c2ba628bbdf5cdff1c159ac0d5e224629dfcfa1c1041914a1b83c2e", + "README_FIRST.md": "442ea580d93978d0d7344e1031f52b9c6ed3a29c95fd1cc24f74cc8369059ed3", + "RIGHTS_AND_PROVENANCE.md": "e35ff2a57c9c03e81751ae3b20b335722ca17a0108cad1cb145b83149288278a", + "STATUS_AND_NONAUTHORITY.md": "ccd5e5b2e61164725da3ac9c7465ba7e558c3a4c45ac3eca8fc47fa924f62baf", + "myth_sidecar.py": "eb7e9ce43394b2b624aaf3989935153ac6d1b7a71aa10f01e2f691fe8d0f6c8f", + "tests/__init__.py": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "tests/test_myth_sidecar.py": "5cb0015d8305b312857f286eab27059922a6aa1b8f04c755056b8ef5ccb2f245", + "tools/build_release.py": "14644f6a4a65d5552aa0bc4ff20257fe0ff3e2e07c23ecc59f4913343f9cc33f", + "tools/rights_scan.py": "ee8b9675f3d20786f53baf59811cfd625baf820b51ea0cc28646b269738abe5b", + "tools/run_negative_verifier_tests.py": "319902bb3f19641f0cebe3dcbf79b70e7c5088f7385831b73a2904a0b1b56691", +} + +HISTORICAL_HASH_REFERENCE_PATHS = frozenset({ + "BUILD_RECEIPT.json", + "PACKAGE_MANIFEST.json", + "PREDECESSOR_CUSTODY.json", + "RIGHTS_AND_PROVENANCE.md", + "tools/build_release.py", + "tools/verify_package.py", +}) +CLASSIFICATION_SCAN_EXEMPT_PATHS = frozenset({ + # Verification tools contain the forbidden strings solely as negative-test + # signatures. Their bytes are inventory-bound and pinned where self- + # reference permits. + "tools/run_negative_verifier_tests.py", + "tools/verify_package.py", +}) +CLASSIFICATION_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("PRIVATE", re.compile(r"\bprivate\b", re.IGNORECASE)), + ("INTERNAL", re.compile(r"\binternal\b", re.IGNORECASE)), + ("EXCLUDED", re.compile(r"\bexcluded\b", re.IGNORECASE)), + ( + "PUBLIC_MANUFACTURE_FALSE", + re.compile( + r"(?:public[ _-]*manufactur(?:e|ing)|\"public_manufacture\")" + r"\s*(?:[:=]|is)?\s*(?:false|no|0)", + re.IGNORECASE, + ), + ), +) + +EXPECTED_NEGATIVE_CASES = ( + "checksum_duplicate_entry", + "extra_benign_resealed", + "extra_classified_resealed", + "license_marker_removed_resealed", + "malformed_controls_resealed", + "malformed_json_resealed", + "old_predecessor_bytes_resealed", + "rights_marker_removed_resealed", + "zip_duplicate_path", + "zip_symlink", + "zip_traversal", +) + +Finding = dict[str, object] + + +def sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def add(findings: list[Finding], code: str, path: str | None = None, + detail: object | None = None) -> None: + item: Finding = {"code": code} + if path is not None: + item["path"] = path + if detail is not None: + item["detail"] = detail + findings.append(item) + + +def normalize(findings: list[Finding]) -> list[Finding]: + unique = { + json.dumps(item, sort_keys=True, separators=(",", ":")): item + for item in findings + } + return [unique[key] for key in sorted(unique)] + + +def strict_json(raw: bytes, path: str, findings: list[Finding]) -> Any | None: + def pairs(items: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in items: + if key in result: + raise ValueError(f"duplicate key: {key}") + result[key] = value + return result + + def constant(value: str) -> None: + raise ValueError(f"non-finite constant: {value}") + + try: + text = raw.decode("utf-8") + return json.loads(text, object_pairs_hook=pairs, parse_constant=constant) + except (UnicodeError, ValueError, json.JSONDecodeError) as exc: + add(findings, "JSON_INVALID", path, str(exc)) + return None + + +def parse_sums(raw: bytes, findings: list[Finding]) -> dict[str, str] | None: + try: + text = raw.decode("utf-8") + except UnicodeError as exc: + add(findings, "CHECKSUM_FILE_NOT_UTF8", "SHA256SUMS", str(exc)) + return None + if not text.endswith("\n"): + add(findings, "CHECKSUM_FILE_NO_FINAL_NEWLINE", "SHA256SUMS") + output: dict[str, str] = {} + previous = "" + for number, line in enumerate(text.splitlines(), 1): + match = re.fullmatch(r"([0-9a-f]{64}) (.+)", line) + if match is None: + add(findings, "CHECKSUM_LINE_MALFORMED", "SHA256SUMS", number) + continue + digest_value, name = match.groups() + path = PurePosixPath(name) + if ( + path.is_absolute() + or ".." in path.parts + or "." in path.parts + or "\\" in name + or "\x00" in name + or name.startswith("/") + or "//" in name + ): + add(findings, "CHECKSUM_PATH_UNSAFE", name, number) + continue + if name in output: + add(findings, "CHECKSUM_PATH_DUPLICATE", name, number) + continue + if previous and name <= previous: + add(findings, "CHECKSUM_PATH_NOT_STRICTLY_SORTED", name, number) + previous = name + output[name] = digest_value + return output + + +def expect_equal(value: Any, expected: Any, path: str, + findings: list[Finding]) -> None: + if type(value) is not type(expected) or value != expected: + add(findings, "SEMANTIC_MISMATCH", path, {"expected": expected, "actual": value}) + + +def require_markers(files: dict[str, bytes], path: str, markers: tuple[str, ...], + findings: list[Finding]) -> None: + raw = files.get(path) + if raw is None: + return + try: + text = raw.decode("utf-8") + except UnicodeError as exc: + add(findings, "TEXT_NOT_UTF8", path, str(exc)) + return + normalized = " ".join(text.split()) + for marker in markers: + if " ".join(marker.split()) not in normalized: + add(findings, "REQUIRED_MARKER_MISSING", path, marker) + + +def validate_inventory(files: dict[str, bytes], findings: list[Finding]) -> None: + actual = set(files) + for path in sorted(EXPECTED_PATHS - actual): + add(findings, "INVENTORY_MISSING", path) + for path in sorted(actual - EXPECTED_PATHS): + add(findings, "INVENTORY_EXTRA", path) + raw = files.get("PACKAGE_INVENTORY.json") + if raw is None: + return + value = strict_json(raw, "PACKAGE_INVENTORY.json", findings) + expected = { + "authorizes": False, + "path_count": len(EXPECTED_PATHS), + "paths": sorted(EXPECTED_PATHS), + "root_name": ROOT_NAME, + "schema": "project-shadow.generic-myth-sidecar-path-inventory.v1", + } + if value is not None: + expect_equal(value, expected, "PACKAGE_INVENTORY.json", findings) + + +def validate_checksums(files: dict[str, bytes], findings: list[Finding]) -> None: + raw = files.get("SHA256SUMS") + if raw is None: + return + listed = parse_sums(raw, findings) + if listed is None: + return + actual_paths = set(files) - {"SHA256SUMS"} + for path in sorted(actual_paths - set(listed)): + add(findings, "CHECKSUM_ENTRY_MISSING", path) + for path in sorted(set(listed) - actual_paths): + add(findings, "CHECKSUM_ENTRY_EXTRA", path) + for path in sorted(actual_paths & set(listed)): + actual = sha(files[path]) + if listed[path] != actual: + add(findings, "CHECKSUM_MISMATCH", path, { + "expected": listed[path], "actual": actual, + }) + + +def validate_pins(files: dict[str, bytes], findings: list[Finding]) -> None: + for path, expected in sorted(PINNED_SHA256.items()): + raw = files.get(path) + if raw is not None and sha(raw) != expected: + add(findings, "PINNED_CONTENT_MISMATCH", path, { + "expected": expected, "actual": sha(raw), + }) + + +def validate_semantics(files: dict[str, bytes], findings: list[Finding]) -> None: + manifest = strict_json(files["PACKAGE_MANIFEST.json"], "PACKAGE_MANIFEST.json", findings) \ + if "PACKAGE_MANIFEST.json" in files else None + expected_manifest = { + "authority": {"action_authority": "NONE", "authorizes": False, + "exact_hash_admitted_into_r1": False, "production_authorized": False}, + "boundaries": { + "changes_operational_result": False, "default_off": True, + "feedback_allowed": False, "gate_input_eligible": False, + "model_context_eligible": False, "operational_receipt_embedded": False, + "production_deployment_eligible": False, "routing_input_eligible": False, + "score_input_eligible": False, "terminal_only": True, + "terminal_processing_states": ["NONACTION_COMPLETE", "SIMULATION_READY"], + "tool_argument_eligible": False, + }, + "component_id": "PROJECT_SHADOW_GENERIC_MYTH_SIDECAR", + "distribution": {"canonical_r1_component": False, + "public_distribution_permitted_for_eligible_original_material": True, + "release_profile": "OPTIONAL_PUBLIC_COMPANION", "required_for_r1": False}, + "generic_registers": ["Caregiver", "Clockmaker", "Poet"], + "predecessor": { + "custody_identifier": "PROJECT_SHADOW_GENERIC_MYTH_SIDECAR_V0.1.1_HISTORICAL", + "sha256": OLD_PREDECESSOR_SHA256, + "status_not_inherited": "EARLIER_NONPUBLIC_STATUS_BOUNDARY", "version": "0.1.1", + }, + "rights": {"code": "Apache-2.0", "documentation_and_data": "CC-BY-4.0", + "named_third_party_expressive_material_included": False}, + "schema": "project-shadow.generic-myth-sidecar-package-manifest.v2", + "status": "OPTIONAL_PUBLIC_COMPANION_DEFAULT_OFF_TERMINAL_ONLY_NONAUTHORIZING", + "tests": {"expected_unit_tests": 32, "status": "GENERATED_BY_DETERMINISTIC_BUILD"}, + "version": "0.2.0", + } + if manifest is not None: + expect_equal(manifest, expected_manifest, "PACKAGE_MANIFEST.json", findings) + custody = strict_json(files["PREDECESSOR_CUSTODY.json"], "PREDECESSOR_CUSTODY.json", findings) \ + if "PREDECESSOR_CUSTODY.json" in files else None + expected_custody = { + "changes_operational_authority": False, + "predecessor_custody_identifier": "PROJECT_SHADOW_GENERIC_MYTH_SIDECAR_V0.1.1_HISTORICAL", + "predecessor_sha256": OLD_PREDECESSOR_SHA256, + "predecessor_size_bytes": OLD_PREDECESSOR_SIZE_BYTES, + "relationship": "LICENSED_SUCCESSOR_WITH_CORRECTED_PUBLIC_DISTRIBUTION_BOUNDARY", + "schema": "project-shadow.generic-myth-sidecar-predecessor-custody.v1", + "successor_version": "0.2.0", + } + if custody is not None: + expect_equal(custody, expected_custody, "PREDECESSOR_CUSTODY.json", findings) + test_result = strict_json(files["TEST_RESULTS.json"], "TEST_RESULTS.json", findings) \ + if "TEST_RESULTS.json" in files else None + expected_test_result = { + "authorizes": False, "default_off": True, "feedback_allowed": False, + "production_deployment_eligible": False, + "schema": "project-shadow.generic-myth-sidecar-test-result.v2", + "status": "PASS", "terminal_only": True, "unit_tests": 32, + } + if test_result is not None: + expect_equal(test_result, expected_test_result, "TEST_RESULTS.json", findings) + build = strict_json(files["BUILD_RECEIPT.json"], "BUILD_RECEIPT.json", findings) \ + if "BUILD_RECEIPT.json" in files else None + expected_build = { + "archive_compression": "ZIP_STORED", + "archive_timestamp": "2026-08-17T00:00:00Z_DECLARED_FIXED_FOR_REPRODUCIBILITY", + "authorizes": False, "deterministic_sorted_paths": True, + "expected_unit_tests": 32, "negative_verifier_tests": len(EXPECTED_NEGATIVE_CASES), + "negative_verifier_tests_status": "PASS", "predecessor_sha256": OLD_PREDECESSOR_SHA256, + "rights_scan_status": "PASS", + "schema": "project-shadow.generic-myth-sidecar-build-receipt.v2", + "status": "PASS", "unit_test_status": "PASS", "version": "0.2.0", + } + if build is not None: + expect_equal(build, expected_build, "BUILD_RECEIPT.json", findings) + rights = strict_json(files["RIGHTS_SCAN_RESULT.json"], "RIGHTS_SCAN_RESULT.json", findings) \ + if "RIGHTS_SCAN_RESULT.json" in files else None + if rights is not None: + expected_rights = { + "authorizes": False, "binary_expressive_assets_found": 0, "files_scanned": 19, + "findings": [], "named_third_party_material_claim": "NONE_INCLUDED_BY_PACKAGE_DESIGN", + "schema": "project-shadow.generic-myth-rights-scan.v1", "status": "PASS", + } + expect_equal(rights, expected_rights, "RIGHTS_SCAN_RESULT.json", findings) + negative = strict_json(files["NEGATIVE_TEST_RESULTS.json"], "NEGATIVE_TEST_RESULTS.json", findings) \ + if "NEGATIVE_TEST_RESULTS.json" in files else None + if negative is not None: + expected_negative = { + "all_mutations_rejected": True, "authorizes": False, "baseline_status": "PASS", + "cases": [{"case": name, "status": "REJECTED_AS_EXPECTED"} + for name in EXPECTED_NEGATIVE_CASES], + "negative_cases": len(EXPECTED_NEGATIVE_CASES), + "schema": "project-shadow.generic-myth-sidecar-negative-verifier-tests.v1", + "status": "PASS", + } + expect_equal(negative, expected_negative, "NEGATIVE_TEST_RESULTS.json", findings) + if "LICENSE" in files and sha(files["LICENSE"]) != APACHE_2_LICENSE_SHA256: + add(findings, "LICENSE_HASH_MISMATCH", "LICENSE", { + "expected": APACHE_2_LICENSE_SHA256, "actual": sha(files["LICENSE"]), + }) + require_markers(files, "LICENSE-DOCS", ( + "Creative Commons Attribution 4.0 International (CC BY 4.0)", + "applies only to eligible original material", + "implies no endorsement, certification, production authorization, efficacy, or safety", + ), findings) + require_markers(files, "NOTICE", ( + "optional public companion", + "default-off, terminal-only, nonauthorizing, no-feedback, outside canonical R1", + "ineligible for production deployment", "supplies no certification", + ), findings) + require_markers(files, "RIGHTS_AND_PROVENANCE.md", ( + "Apache License 2.0", "Creative Commons Attribution 4.0 International", + "generic original registers only", "retained as historical custody evidence", + "earlier non-public status metadata is not carried forward", + ), findings) + require_markers(files, "CONTROL_BOUNDARY.md", ( + "Default off; explicit opt-in is required.", + "Terminal only: `SIMULATION_READY` or `NONACTION_COMPLETE`.", + "Nonauthorizing: `authorizes=false`, `action_authority=NONE`.", + "No feedback to the operational result or Project Shadow runtime.", + "No production deployment eligibility.", + ), findings) + require_markers(files, "STATUS_AND_NONAUTHORITY.md", ( + "optional\nstandalone Project Shadow companion", + "does not make it part\nof canonical R1", + "supplies no\nauthority, evidence, approval, instruction, capability, or permission to act", + ), findings) + + +def validate_content_scans(files: dict[str, bytes], findings: list[Finding]) -> None: + for path, raw in sorted(files.items()): + if path == "SHA256SUMS": + continue + digest = sha(raw) + if digest == OLD_PREDECESSOR_SHA256: + add(findings, "OLD_PREDECESSOR_ARCHIVE_PRESENT", path, { + "sha256": digest, "size_bytes": len(raw), + }) + if (OLD_PREDECESSOR_SHA256.encode("ascii") in raw + and path not in HISTORICAL_HASH_REFERENCE_PATHS): + add(findings, "OLD_HASH_REFERENCE_OUTSIDE_EXEMPTION", path) + if path in CLASSIFICATION_SCAN_EXEMPT_PATHS: + continue + try: + text = raw.decode("utf-8") + except UnicodeError as exc: + add(findings, "CONTENT_NOT_UTF8", path, str(exc)) + continue + for label, pattern in CLASSIFICATION_PATTERNS: + if pattern.search(text): + add(findings, "NONPUBLIC_CLASSIFICATION_TOKEN", path, label) + + +def validate_payload(files: dict[str, bytes], findings: list[Finding]) -> list[Finding]: + validate_inventory(files, findings) + validate_checksums(files, findings) + validate_pins(files, findings) + validate_semantics(files, findings) + validate_content_scans(files, findings) + return normalize(findings) + + +def read_tree(root: Path, findings: list[Finding]) -> dict[str, bytes]: + files: dict[str, bytes] = {} + if root.name != ROOT_NAME: + add(findings, "ROOT_NAME_MISMATCH", root.name, ROOT_NAME) + try: + entries = sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()) + except OSError as exc: + add(findings, "TREE_ENUMERATION_ERROR", str(root), str(exc)) + return files + for entry in entries: + relative = entry.relative_to(root).as_posix() + try: + mode = entry.lstat().st_mode + except OSError as exc: + add(findings, "TREE_LSTAT_ERROR", relative, str(exc)) + continue + if stat.S_ISLNK(mode): + add(findings, "TREE_SYMLINK", relative) + continue + if stat.S_ISDIR(mode): + continue + if not stat.S_ISREG(mode): + add(findings, "TREE_NONREGULAR_MEMBER", relative, oct(mode)) + continue + size = entry.stat().st_size + if size > MAX_MEMBER_BYTES: + add(findings, "TREE_MEMBER_TOO_LARGE", relative, size) + continue + try: + files[relative] = entry.read_bytes() + except OSError as exc: + add(findings, "TREE_READ_ERROR", relative, str(exc)) + total = sum(map(len, files.values())) + if total > MAX_TOTAL_BYTES: + add(findings, "TREE_TOTAL_TOO_LARGE", detail=total) + return files + + +def verify_tree(root: Path) -> list[Finding]: + findings: list[Finding] = [] + files = read_tree(root, findings) + return validate_payload(files, findings) + + +def safe_zip_relative(name: str, findings: list[Finding]) -> str | None: + if "\x00" in name: + add(findings, "ZIP_PATH_NUL", name) + return None + if "\\" in name: + add(findings, "ZIP_PATH_BACKSLASH", name) + return None + if name.startswith("/") or name.endswith("/") or "//" in name: + add(findings, "ZIP_PATH_NONCANONICAL", name) + return None + parts = name.split("/") + if any(part in {"", ".", ".."} for part in parts): + add(findings, "ZIP_PATH_TRAVERSAL", name) + return None + if not parts or parts[0] != ROOT_NAME or len(parts) < 2: + add(findings, "ZIP_ROOT_MISMATCH", name, ROOT_NAME) + return None + relative = "/".join(parts[1:]) + path = PurePosixPath(relative) + if path.is_absolute() or path.as_posix() != relative: + add(findings, "ZIP_PATH_NONCANONICAL", name) + return None + return relative + + +def verify_zip(path: Path) -> list[Finding]: + findings: list[Finding] = [] + files: dict[str, bytes] = {} + seen_raw: set[str] = set() + seen_relative: set[str] = set() + total = 0 + try: + with zipfile.ZipFile(path) as archive: + if archive.comment: + add(findings, "ZIP_ARCHIVE_COMMENT_PRESENT") + infos = archive.infolist() + if not infos: + add(findings, "ZIP_EMPTY") + for info in infos: + name = info.filename + if name in seen_raw: + add(findings, "ZIP_DUPLICATE_RAW_PATH", name) + seen_raw.add(name) + relative = safe_zip_relative(name, findings) + if relative is None: + continue + if relative in seen_relative: + add(findings, "ZIP_DUPLICATE_NORMALIZED_PATH", relative) + continue + seen_relative.add(relative) + mode = info.external_attr >> 16 + kind = stat.S_IFMT(mode) + if kind == stat.S_IFLNK: + add(findings, "ZIP_SYMLINK", relative) + continue + if kind != stat.S_IFREG: + add(findings, "ZIP_NONREGULAR_MEMBER", relative, oct(mode)) + continue + expected_permissions = 0o755 if relative.endswith(".py") else 0o644 + if stat.S_IMODE(mode) != expected_permissions: + add(findings, "ZIP_MODE_MISMATCH", relative, { + "expected": oct(expected_permissions), + "actual": oct(stat.S_IMODE(mode)), + }) + if info.create_system != 3: + add(findings, "ZIP_CREATE_SYSTEM_MISMATCH", relative, info.create_system) + if info.date_time != FIXED_TIMESTAMP: + add(findings, "ZIP_TIMESTAMP_MISMATCH", relative, list(info.date_time)) + if info.compress_type != zipfile.ZIP_STORED: + add(findings, "ZIP_COMPRESSION_MISMATCH", relative, info.compress_type) + if info.flag_bits & 0x1: + add(findings, "ZIP_ENCRYPTED_MEMBER", relative) + if info.extra: + add(findings, "ZIP_MEMBER_EXTRA_FIELD", relative) + if info.comment: + add(findings, "ZIP_MEMBER_COMMENT_PRESENT", relative) + if info.file_size > MAX_MEMBER_BYTES: + add(findings, "ZIP_MEMBER_TOO_LARGE", relative, info.file_size) + continue + total += info.file_size + if total > MAX_TOTAL_BYTES: + add(findings, "ZIP_TOTAL_TOO_LARGE", detail=total) + continue + try: + files[relative] = archive.read(info) + except (OSError, RuntimeError, zipfile.BadZipFile) as exc: + add(findings, "ZIP_MEMBER_READ_ERROR", relative, str(exc)) + except (OSError, zipfile.BadZipFile) as exc: + add(findings, "ZIP_OPEN_ERROR", str(path), f"{type(exc).__name__}:{exc}") + return normalize(findings) + return validate_payload(files, findings) + + +def main() -> int: + path = Path(sys.argv[1] if len(sys.argv) > 1 else ".").resolve() + findings = verify_tree(path) if path.is_dir() else verify_zip(path) + result = { + "schema": "project-shadow.package-verification.v3", + "status": "PASS" if not findings else "FAIL", + "findings": findings, + "inventory_path_count": len(EXPECTED_PATHS), + "old_predecessor_embedded": any( + item.get("code") == "OLD_PREDECESSOR_ARCHIVE_PRESENT" for item in findings + ), + "authorizes": False, + } + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return 0 if not findings else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verify_outer_release.py b/tools/verify_outer_release.py new file mode 100644 index 0000000..b89b036 --- /dev/null +++ b/tools/verify_outer_release.py @@ -0,0 +1,976 @@ +#!/usr/bin/env python3 +"""Read-only verifier for Project Shadow R1.0.1 outer directories or ZIPs.""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import re +import stat +import sys +import zipfile +from datetime import datetime +from pathlib import Path, PurePosixPath +from typing import Any + + +TITLE = "Project Shadow 1.0.1 — R1 Reference Packaging Correction" +ROOT = "Project_Shadow_R1.0.1_Public_Reference_2026-08-17" +ARCHIVE = f"{ROOT}.zip" +TAG = "r1.0.1-2026-08-17" +FIXED_ZIP_TIME = (2026, 8, 17, 0, 0, 0) + +INNER_ROOT = "Project_Shadow_R1.0.1_Runtime_Family_Myth_Decoupled_2026-08-17" +INNER_FILENAME = f"{INNER_ROOT}.zip" +INNER_EXPECTED_INVENTORY_PATH = "08_VERIFICATION/EXPECTED_FILE_INVENTORY.json" +INNER_EXPECTED_INVENTORY_SHA256 = "4cdc1611bf11d56694b4868eebbc705e339a0b6a76455d4cd75374fb0b7b0c22" +INNER_EXPECTED_FILE_COUNT = 85 +ADMITTED_INNER_BYTES = 5_463_189 +ADMITTED_INNER_SHA256 = "c8c32b12432c954b1a6f852c0c9f81bbbd40167e936be057d4c3de1a0aa3a623" +OLD_INNER_FILENAME = "Project_Shadow_R1_BETA2_Runtime_Successor_Candidate_2026-08-10.zip" +OLD_INNER_SHA256 = "075b41ea4186b2d2edb0ed246ab7662cf8bbdf3160294e3eca176b9d0857b108" +OLD_INNER_BYTES = 5_442_808 + +PC_FILENAME = "Project_Shadow_PS-R1-PC-S1_v0.1.0-beta.5_POST_REAUDIT_CORRECTION_CANDIDATE_2026-08-10.zip" +PC_SHA256 = "1ffdba41025c0b81da92d0bbb22d0eaa69488cffbc80936365034669110448d7" +PC_BYTES = 1_935_482 + +OLD_OUTER_FILENAME = "Project_Shadow_R1_Public_Release_Candidate_2026-08-14.zip" +OLD_OUTER_SHA256 = "2f8fe1530b6a83294d15011df95853aaecf08fa4dba756f0c2e91dd089e1b1ec" +OLD_OUTER_BYTES = 7_679_812 +OLD_OUTER_TAG = "r1-2026-08-14" + +REMOVED_MYTH_FILENAME = "Project_Shadow_R1_Myth_Sidecar_Generic_v0.1.1_PRIVATE_2026-08-07.zip" +REMOVED_MYTH_SHA256 = "4d8481ed8f6af3e04d86a5c2de7e94aad47a591e1ce9c8757bbb4b0a397b90bb" +REMOVED_MYTH_BYTES = 29_763 + +GENERIC = { + "filename": "Project_Shadow_Generic_Myth_Sidecar_v0.2.0_OPTIONAL_PUBLIC_COMPANION_2026-08-17.zip", + "bytes": 93_676, + "sha256": "6e7a362d4135f9d626dcfef463bfb1f7166226b3cf8a4c02a953ab39af1538bf", + "tag": "generic-myth-v0.2.0", + "version": "0.2.0", + "mixed_rights": False, +} +FULL_CANON = { + "filename": "Project_Shadow_Full_Canon_Myth_Sidecar_v0.3.5_OPTIONAL_PUBLIC_COMPANION_2026-08-17.zip", + "bytes": 1_428_812, + "sha256": "2b55867fe7c502a0defd8d6f2e9b53fbd1caaf1b0f225a438bd45b04a3e7bae2", + "tag": "myth-v0.3.5", + "version": "0.3.5", + "mixed_rights": True, +} + +ACTIVE_PREFIXES = ( + "02_EDITIONS/", + "03_PS_LANGUAGE/", + "04_PYTHON_REFERENCES/", + "05_KERNELS/", + "06_PROMPTS/", +) +MAX_MEMBERS = 50_000 +MAX_UNCOMPRESSED = 1_500_000_000 +MAX_RECURSION = 8 +HEX64 = re.compile(r"^[0-9a-f]{64}$") + +OUTER_PENDING_INVENTORY = frozenset( + { + "00_START_HERE.md", + "01_AUTHORITY/INNER_ADMISSION_SLOT.json", + "02_STATUS/PUBLIC_REFERENCE_STATUS.json", + "03_AUDIT/R1_1.0.1_DESCENDANT_BYTE_IDENTITY_MAP.json", + "03_AUDIT/OUTER_BUILD_RECEIPT.json", + "04_CORRECTION/CAPA_PS-R1-PRIVATE-MYTH-PUBLIC-BOUNDARY-001.json", + "04_CORRECTION/CORRECTION_NOTICE.md", + f"05_COMPONENTS/R1/{INNER_FILENAME}", + f"05_COMPONENTS/PRIMITIVE_COMMONS/{PC_FILENAME}", + "06_EXTERNAL_SIDECARS/GENERIC_MYTH_V0.2.0_REFERENCE.json", + "06_EXTERNAL_SIDECARS/FULL_CANON_MYTH_V0.3.5_REFERENCE.json", + "07_HISTORICAL/PRESERVED_AUGUST_14_R1_REFERENCE.json", + "LICENSE", + "LICENSE-DOCS", + "NOTICE", + "RIGHTS_MANIFEST.json", + "SAFETY_AND_SECRET_SCAN_REPORT.json", + "tools/verify_outer_release.py", + "PACKAGE_MANIFEST.json", + "SHA256SUMS", + } +) +OUTER_BOUND_INVENTORY = OUTER_PENDING_INVENTORY | { + "01_AUTHORITY/INNER_EXACT_HASH_MAINTAINER_ADMISSION.json" +} +FORBIDDEN_CURRENT_CLASSIFICATION_TOKENS = ( + b"PRIVATE_" + b"INTERNAL", + b"EXCLUDED_" + b"FROM_PUBLIC", + b'"public_manufacture_' + b'eligible":false', + b'"public_manufacture_' + b'eligible": false', +) +INNER_SEMANTIC_EXCLUDED_PREFIXES = ("07_EXTERNAL_SIDECARS/", "11_HISTORICAL/") + + +class VerificationError(RuntimeError): + """A verification control failed.""" + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def validate_member_name(name: str) -> None: + if not name or "\\" in name or name.startswith("/"): + raise VerificationError(f"unsafe ZIP member name: {name!r}") + path = PurePosixPath(name.rstrip("/")) + if not path.parts or any(part in {"", ".", ".."} for part in path.parts): + raise VerificationError(f"unsafe ZIP member name: {name!r}") + + +def safe_infos(archive: zipfile.ZipFile) -> list[zipfile.ZipInfo]: + infos = archive.infolist() + if len(infos) > MAX_MEMBERS: + raise VerificationError("ZIP member count exceeds safety limit") + total = 0 + seen: set[str] = set() + for info in infos: + validate_member_name(info.filename) + if info.filename in seen: + raise VerificationError(f"duplicate ZIP member: {info.filename}") + seen.add(info.filename) + mode = (info.external_attr >> 16) & 0xFFFF + if stat.S_ISLNK(mode): + raise VerificationError(f"symlink ZIP member rejected: {info.filename}") + total += info.file_size + if total > MAX_UNCOMPRESSED: + raise VerificationError("ZIP uncompressed size exceeds safety limit") + return infos + + +def one_zip_root(archive: zipfile.ZipFile) -> str: + roots = { + PurePosixPath(info.filename.rstrip("/")).parts[0] + for info in safe_infos(archive) + } + if len(roots) != 1: + raise VerificationError(f"ZIP must contain exactly one root: {sorted(roots)}") + return next(iter(roots)) + + +class PackageReader: + """Read-only abstraction over a package directory or outer ZIP.""" + + def __init__(self, source: Path): + self.source = source + self.archive: zipfile.ZipFile | None = None + self.archive_bytes: bytes | None = None + self.root = ROOT + self.deterministic_metadata_verified = False + if source.is_dir(): + if source.name != ROOT: + raise VerificationError(f"directory root must be named {ROOT}") + self.kind = "directory" + self._files = { + path.relative_to(source).as_posix() + for path in source.rglob("*") + if path.is_file() + } + for path in source.rglob("*"): + if path.is_symlink(): + raise VerificationError(f"symlink rejected in directory package: {path}") + elif source.is_file(): + if source.name != ARCHIVE: + raise VerificationError(f"outer ZIP must be named {ARCHIVE}") + self.kind = "zip" + self.archive_bytes = source.read_bytes() + self.archive = zipfile.ZipFile(io.BytesIO(self.archive_bytes)) + root = one_zip_root(self.archive) + if root != ROOT: + raise VerificationError(f"outer ZIP root must be {ROOT}, found {root}") + infos = safe_infos(self.archive) + self._files = { + PurePosixPath(info.filename).relative_to(root).as_posix() + for info in infos + if not info.is_dir() + } + for info in infos: + if info.is_dir(): + continue + mode = (info.external_attr >> 16) & 0xFFFF + if info.date_time != FIXED_ZIP_TIME: + raise VerificationError(f"non-deterministic timestamp on {info.filename}") + if stat.S_IMODE(mode) != 0o644 or not stat.S_ISREG(mode): + raise VerificationError(f"non-deterministic mode on {info.filename}: {oct(mode)}") + if info.compress_type != zipfile.ZIP_DEFLATED: + raise VerificationError(f"non-deterministic compression on {info.filename}") + self.deterministic_metadata_verified = True + else: + raise VerificationError(f"package path not found: {source}") + + def files(self) -> set[str]: + return set(self._files) + + def read(self, relative: str) -> bytes: + if relative not in self._files: + raise VerificationError(f"missing package file: {relative}") + if self.kind == "directory": + return (self.source / PurePosixPath(relative)).read_bytes() + assert self.archive is not None + return self.archive.read(f"{ROOT}/{relative}") + + def close(self) -> None: + if self.archive is not None: + self.archive.close() + + +def read_json(reader: PackageReader, path: str) -> dict[str, Any]: + try: + value = json.loads(reader.read(path)) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise VerificationError(f"invalid UTF-8 JSON: {path}") from exc + if not isinstance(value, dict): + raise VerificationError(f"expected JSON object: {path}") + return value + + +def parse_sums(data: bytes) -> dict[str, str]: + try: + text = data.decode("utf-8") + except UnicodeDecodeError as exc: + raise VerificationError("SHA256SUMS is not UTF-8") from exc + result: dict[str, str] = {} + for number, line in enumerate(text.splitlines(), start=1): + match = re.fullmatch(r"([0-9a-f]{64}) ([^\r\n]+)", line) + if not match: + raise VerificationError(f"invalid SHA256SUMS line {number}") + digest, path = match.groups() + validate_member_name(path) + if path in result: + raise VerificationError(f"duplicate SHA256SUMS path: {path}") + result[path] = digest + return result + + +def verify_sums(reader: PackageReader) -> None: + sums = parse_sums(reader.read("SHA256SUMS")) + expected = reader.files() - {"SHA256SUMS"} + if set(sums) != expected: + missing = sorted(expected - set(sums)) + extra = sorted(set(sums) - expected) + raise VerificationError(f"SHA256SUMS coverage mismatch; missing={missing}, extra={extra}") + for path, expected_hash in sums.items(): + if sha256_bytes(reader.read(path)) != expected_hash: + raise VerificationError(f"SHA256SUMS mismatch: {path}") + + +def exact_outer_inventory_mode(paths: set[str]) -> str: + """Match the actual package against a code-owned, authority-specific set.""" + if paths == OUTER_PENDING_INVENTORY: + return "PENDING" + if paths == OUTER_BOUND_INVENTORY: + return "BOUND" + allowed_union = OUTER_BOUND_INVENTORY + missing = sorted(OUTER_PENDING_INVENTORY - paths) + extra = sorted(paths - allowed_union) + if not extra and paths - OUTER_PENDING_INVENTORY: + extra = sorted(paths - OUTER_PENDING_INVENTORY) + raise VerificationError( + f"code-owned exact outer file inventory mismatch; missing={missing}, extra={extra}" + ) + + +def verify_inner_exact_inventory(inner_data: bytes) -> set[str]: + try: + archive = zipfile.ZipFile(io.BytesIO(inner_data)) + except zipfile.BadZipFile as exc: + raise VerificationError("corrected inner component is not a ZIP") from exc + with archive: + root = one_zip_root(archive) + if root != INNER_ROOT: + raise VerificationError(f"corrected inner root mismatch: {root}") + files = { + PurePosixPath(info.filename).relative_to(root).as_posix(): archive.read(info) + for info in safe_infos(archive) + if not info.is_dir() + } + inventory_raw = files.get(INNER_EXPECTED_INVENTORY_PATH) + if inventory_raw is None or sha256_bytes(inventory_raw) != INNER_EXPECTED_INVENTORY_SHA256: + raise VerificationError("inner code-owned exact inventory hash mismatch") + try: + inventory = json.loads(inventory_raw) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise VerificationError("invalid inner exact inventory JSON") from exc + paths = inventory.get("paths") + if ( + inventory.get("schema") != "project-shadow.exact-file-inventory.v1" + or inventory.get("status") != "EXACT_PATH_SET_REQUIRED_NO_UNINDEXED_FILES" + or inventory.get("expected_file_count") != INNER_EXPECTED_FILE_COUNT + or not isinstance(paths, list) + or len(paths) != INNER_EXPECTED_FILE_COUNT + or len(set(paths)) != INNER_EXPECTED_FILE_COUNT + or set(paths) != set(files) + ): + raise VerificationError("inner exact file inventory does not match the code-frozen 85-path package") + if "SHA256SUMS" not in files: + raise VerificationError("inner SHA256SUMS is missing") + sums = parse_sums(files["SHA256SUMS"]) + expected_sum_paths = set(files) - {"SHA256SUMS"} + if set(sums) != expected_sum_paths: + raise VerificationError("inner SHA256SUMS does not have exact full-file coverage") + for path, expected_hash in sums.items(): + if sha256_bytes(files[path]) != expected_hash: + raise VerificationError(f"inner SHA256SUMS mismatch: {path}") + return set(files) + + +def scan_classification_payload( + data: bytes, + label: str, + *, + depth: int = 0, + budget: list[int] | None = None, +) -> tuple[int, int]: + if budget is None: + budget = [0, 0] + budget[0] += 1 + budget[1] += len(data) + if budget[0] > MAX_MEMBERS or budget[1] > MAX_UNCOMPRESSED: + raise VerificationError("current-payload semantic scan safety budget exceeded") + for token in FORBIDDEN_CURRENT_CLASSIFICATION_TOKENS: + if token in data: + raise VerificationError( + f"nonpublic classification token at {label}: {token.decode('ascii')}" + ) + if not data.startswith(b"PK\x03\x04"): + return 1, len(data) + if depth >= MAX_RECURSION: + raise VerificationError(f"current-payload semantic recursion limit exceeded at {label}") + try: + archive = zipfile.ZipFile(io.BytesIO(data)) + except zipfile.BadZipFile as exc: + raise VerificationError(f"ZIP-signature payload is malformed at {label}") from exc + members = 1 + unpacked = len(data) + with archive: + for info in safe_infos(archive): + if info.is_dir(): + continue + child_members, child_bytes = scan_classification_payload( + archive.read(info), + f"{label}!/{info.filename}", + depth=depth + 1, + budget=budget, + ) + members += child_members + unpacked += child_bytes + return members, unpacked + + +def scan_all_current_inner_payloads(inner_data: bytes) -> tuple[int, int]: + members = 0 + unpacked = 0 + budget = [0, 0] + with zipfile.ZipFile(io.BytesIO(inner_data)) as archive: + root = one_zip_root(archive) + for info in safe_infos(archive): + if info.is_dir(): + continue + relative = PurePosixPath(info.filename).relative_to(root).as_posix() + if relative.startswith(INNER_SEMANTIC_EXCLUDED_PREFIXES): + continue + child_members, child_bytes = scan_classification_payload( + archive.read(info), + relative, + budget=budget, + ) + members += child_members + unpacked += child_bytes + return members, unpacked + + +def scan_outer_current_controls(reader: PackageReader) -> tuple[int, int]: + members = 0 + unpacked = 0 + budget = [0, 0] + excluded_prefixes = ( + "05_COMPONENTS/R1/", + "05_COMPONENTS/PRIMITIVE_COMMONS/", + "06_EXTERNAL_SIDECARS/", + "07_HISTORICAL/", + ) + for path in sorted(reader.files()): + if path.startswith(excluded_prefixes): + continue + child_members, child_bytes = scan_classification_payload( + reader.read(path), + path, + budget=budget, + ) + members += child_members + unpacked += child_bytes + return members, unpacked + + +def active_rows(index: dict[str, Any]) -> dict[str, dict[str, Any]]: + rows: dict[str, dict[str, Any]] = {} + for row in index.get("artifacts", []): + path = row.get("path") + if isinstance(path, str) and path.startswith(ACTIVE_PREFIXES): + if path in rows: + raise VerificationError(f"duplicate active artifact path: {path}") + rows[path] = row + if index.get("active_descendant_count") != 27 or len(rows) != 27: + raise VerificationError(f"corrected inner does not expose exactly 27 active descendants: {len(rows)}") + return rows + + +def nested_inner(inner_data: bytes) -> tuple[dict[str, Any], dict[str, dict[str, Any]], str]: + try: + archive = zipfile.ZipFile(io.BytesIO(inner_data)) + except zipfile.BadZipFile as exc: + raise VerificationError("inner R1 component is not a ZIP") from exc + with archive: + root = one_zip_root(archive) + if root != INNER_ROOT: + raise VerificationError(f"corrected inner root mismatch: {root}") + try: + index = json.loads(archive.read(f"{root}/08_VERIFICATION/ARTIFACT_INDEX.json")) + package_manifest = json.loads(archive.read(f"{root}/PACKAGE_MANIFEST.json")) + except (KeyError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise VerificationError("corrected inner controls are missing or invalid") from exc + rows = active_rows(index) + for path, row in rows.items(): + try: + payload = archive.read(f"{root}/{path}") + except KeyError as exc: + raise VerificationError(f"corrected inner indexed payload is missing: {path}") from exc + if len(payload) != row.get("bytes") or sha256_bytes(payload) != row.get("sha256"): + raise VerificationError(f"corrected inner index identity mismatch: {path}") + required = { + "active_descendant_count": 27, + "private_myth_member_removed": True, + "operational_artifact_bytes_changed": 0, + } + for key, expected in required.items(): + if package_manifest.get(key) != expected: + raise VerificationError(f"corrected inner PACKAGE_MANIFEST requires {key}={expected!r}") + if package_manifest.get("public_manufacture_eligible") is False: + raise VerificationError("corrected inner PACKAGE_MANIFEST forbids public manufacture") + return index, rows, root + + +def semantic_walk(value: Any, location: str) -> None: + if isinstance(value, dict): + for key, child in value.items(): + key_lower = str(key).lower() + if key_lower == "public_manufacture_eligible" and child is False: + raise VerificationError(f"active manifest has public_manufacture_eligible=false at {location}.{key}") + semantic_walk(child, f"{location}.{key}") + elif isinstance(value, list): + for index, child in enumerate(value): + semantic_walk(child, f"{location}[{index}]") + elif isinstance(value, str) and value.upper() in { + "PRIVATE_" + "INTERNAL", + "EXCLUDED_" + "FROM_PUBLIC", + }: + raise VerificationError(f"active manifest has forbidden semantic value {value!r} at {location}") + + +def is_active_manifest_path(name: str) -> bool: + parts = [part.upper() for part in PurePosixPath(name).parts] + excluded = {"HISTORICAL", "HISTORY", "EVIDENCE", "AUDIT", "RECEIPTS", "ADMISSION"} + if any(part in excluded for part in parts[:-1]): + return False + basename = parts[-1] + return basename in {"PACKAGE_MANIFEST.JSON", "MANIFEST.JSON"} + + +def verify_active_manifest_semantics(inner_data: bytes, rows: dict[str, dict[str, Any]], root: str) -> int: + checked = 0 + with zipfile.ZipFile(io.BytesIO(inner_data)) as inner: + for path in sorted(rows): + if not path.lower().endswith(".zip"): + continue + artifact = inner.read(f"{root}/{path}") + try: + child = zipfile.ZipFile(io.BytesIO(artifact)) + except zipfile.BadZipFile as exc: + raise VerificationError(f"active indexed ZIP is invalid: {path}") from exc + with child: + for info in safe_infos(child): + if info.is_dir() or not is_active_manifest_path(info.filename): + continue + try: + manifest = json.loads(child.read(info)) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise VerificationError(f"invalid active manifest JSON: {path}!/{info.filename}") from exc + semantic_walk(manifest, f"{path}!/{info.filename}") + checked += 1 + return checked + + +def recursive_forbidden_scan( + data: bytes, + forbidden: set[str], + label: str, + *, + depth: int = 0, + budget: list[int] | None = None, +) -> tuple[int, int]: + if budget is None: + budget = [0, 0] + if depth > MAX_RECURSION: + raise VerificationError(f"nested ZIP recursion limit exceeded at {label}") + try: + archive = zipfile.ZipFile(io.BytesIO(data)) + except zipfile.BadZipFile: + return 0, 0 + members = 0 + unpacked = 0 + with archive: + for info in safe_infos(archive): + if info.is_dir(): + continue + payload = archive.read(info) + budget[0] += 1 + budget[1] += len(payload) + if budget[0] > MAX_MEMBERS or budget[1] > MAX_UNCOMPRESSED: + raise VerificationError("recursive scan safety budget exceeded") + digest = sha256_bytes(payload) + if digest in forbidden: + raise VerificationError(f"forbidden embedded payload {digest} at {label}!/{info.filename}") + members += 1 + unpacked += len(payload) + if info.filename.lower().endswith(".zip"): + child_count, child_bytes = recursive_forbidden_scan( + payload, + forbidden, + f"{label}!/{info.filename}", + depth=depth + 1, + budget=budget, + ) + members += child_count + unpacked += child_bytes + return members, unpacked + + +def validate_external_reference(value: dict[str, Any], expected: dict[str, Any]) -> None: + for key in ("filename", "bytes", "sha256", "tag", "version", "mixed_rights"): + if value.get(key) != expected[key]: + raise VerificationError(f"external sidecar reference mismatch: {key}") + requirements = { + "canonical_r1": False, + "default_enabled": False, + "embedded": False, + "feedback_into_r1": False, + "operational_authority": False, + "optional": True, + "production_deployment": "HARD_OFF", + "relationship_to_r1": "SEPARATE_OPTIONAL_PUBLIC_COMPANION", + "required_for_r1": False, + "terminal_only": True, + } + for key, expected_value in requirements.items(): + if value.get(key) != expected_value: + raise VerificationError(f"external sidecar boundary mismatch: {key}") + + +def validate_inner_sidecar_references(inner_data: bytes) -> None: + references = ( + ( + "07_EXTERNAL_SIDECARS/PROJECT_SHADOW_GENERIC_MYTH_REFERENCE.json", + GENERIC, + ), + ( + "07_EXTERNAL_SIDECARS/PROJECT_SHADOW_FULL_CANON_MYTH_V0.3.5_REFERENCE.json", + FULL_CANON, + ), + ) + with zipfile.ZipFile(io.BytesIO(inner_data)) as archive: + root = one_zip_root(archive) + for relative, expected in references: + try: + value = json.loads(archive.read(f"{root}/{relative}")) + except (KeyError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise VerificationError(f"invalid corrected inner sidecar reference: {relative}") from exc + for key in ("filename", "bytes", "sha256", "version", "mixed_rights"): + if value.get(key) != expected[key]: + raise VerificationError( + f"corrected inner sidecar reference mismatch: {relative}:{key}" + ) + boundary = { + "authority_effect_on_r1": "NONE", + "authorizes": False, + "default_enabled": False, + "included_in_r1": False, + "operational_authority": False, + "publication_status": "EXTERNAL_TO_THIS_FAMILY", + "relationship_to_r1": "SEPARATE_OPTIONAL_COMPANION", + "required_for_r1": False, + "terminal_only": True, + } + for key, expected_value in boundary.items(): + if value.get(key) != expected_value: + raise VerificationError( + f"corrected inner sidecar boundary mismatch: {relative}:{key}" + ) + + +def validate_admission_record(data: bytes, inner_hash: str, inner_size: int) -> None: + try: + record = json.loads(data) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise VerificationError("invalid embedded inner admission record") from exc + if not isinstance(record, dict): + raise VerificationError("embedded inner admission record must be a JSON object") + expected_top_keys = { + "schema", + "decision", + "inner", + "maintainer_confirmation", + "non_authorizations", + } + if set(record) != expected_top_keys: + raise VerificationError( + "embedded inner admission record has unexpected or missing top-level keys" + ) + if inner_hash != ADMITTED_INNER_SHA256 or inner_size != ADMITTED_INNER_BYTES: + raise VerificationError( + "inner admission applies only to the exact maintainer-admitted artifact" + ) + expected_statement = ( + "I admit the exact Myth-free R1.0.1 inner family—5,463,189 bytes, SHA-256 " + "c8c32b12432c954b1a6f852c0c9f81bbbd40167e936be057d4c3de1a0aa3a623—for R1.0.1 " + "reference packaging. Its 27 active descendants are byte-identical to the August 14 " + "predecessor, no Myth payload is embedded, and this admission does not authorize production, " + "deployment, or publication." + ) + if record.get("schema") != "project-shadow.r1.0.1-inner-exact-hash-maintainer-admission.v1": + raise VerificationError("unsupported inner admission record schema") + if record.get("decision") != "ADMIT_EXACT_HASH_FOR_R1.0.1_REFERENCE_PACKAGING": + raise VerificationError("inner admission record has wrong decision") + inner = record.get("inner") + expected_inner_keys = { + "filename", + "bytes", + "sha256", + "active_descendant_count", + "operational_descendant_bytes_changed", + "myth_payload_embedded", + } + if not isinstance(inner, dict) or set(inner) != expected_inner_keys: + raise VerificationError("inner admission record has unexpected or missing inner keys") + expected_inner = { + "filename": INNER_FILENAME, + "bytes": ADMITTED_INNER_BYTES, + "sha256": ADMITTED_INNER_SHA256, + "active_descendant_count": 27, + "operational_descendant_bytes_changed": 0, + "myth_payload_embedded": False, + } + for key, expected_value in expected_inner.items(): + if inner.get(key) != expected_value: + raise VerificationError(f"inner admission record mismatch: inner.{key}") + confirmation = record.get("maintainer_confirmation") + if not isinstance(confirmation, dict) or set(confirmation) != { + "confirmed_by", + "confirmed_at", + "statement", + }: + raise VerificationError( + "inner admission record has unexpected or missing maintainer_confirmation keys" + ) + if not isinstance(confirmation.get("confirmed_by"), str) or not confirmation["confirmed_by"].strip(): + raise VerificationError("inner admission record has no confirmed_by") + confirmed_at = confirmation.get("confirmed_at") + if not isinstance(confirmed_at, str): + raise VerificationError("inner admission record has no UTC confirmed_at") + try: + datetime.strptime(confirmed_at, "%Y-%m-%dT%H:%M:%SZ") + except ValueError as exc: + raise VerificationError( + "inner admission confirmed_at must be strict UTC RFC3339 seconds" + ) from exc + if confirmation.get("statement") != expected_statement: + raise VerificationError("inner admission record statement mismatch") + if record.get("non_authorizations") != { + "production_authorized": False, + "deployment_authorized": False, + "publication_authorized": False, + }: + raise VerificationError("inner admission record changed a non-authorization") + + +def verify_package(source: Path) -> dict[str, Any]: + checks: list[str] = [] + reader = PackageReader(source) + try: + inventory_mode = exact_outer_inventory_mode(reader.files()) + checks.append(f"CODE_OWNED_EXACT_OUTER_INVENTORY_PASS;MODE={inventory_mode}") + verify_sums(reader) + checks.append("SHA256SUMS_FULL_COVERAGE_PASS") + + manifest = read_json(reader, "PACKAGE_MANIFEST.json") + fixed = { + "schema": "project-shadow.r1.0.1-public-reference-package-manifest.v1", + "release_identity": TITLE, + "root": ROOT, + "tag": TAG, + "active_descendant_count": 27, + "operational_descendant_bytes_changed": 0, + "myth_payloads_embedded": 0, + "publication_authorized": False, + } + for key, expected in fixed.items(): + if manifest.get(key) != expected: + raise VerificationError(f"PACKAGE_MANIFEST mismatch: {key}") + if manifest.get("capa") != { + "id": "PS-R1-PRIVATE-MYTH-PUBLIC-BOUNDARY-001", + "status": "IMPLEMENTED_PENDING_EFFECTIVENESS", + }: + raise VerificationError("PACKAGE_MANIFEST CAPA status mismatch") + expected_inventory_count = ( + len(OUTER_BOUND_INVENTORY) if inventory_mode == "BOUND" else len(OUTER_PENDING_INVENTORY) + ) + if manifest.get("exact_file_inventory_count") != expected_inventory_count: + raise VerificationError("PACKAGE_MANIFEST exact file inventory count mismatch") + listed = { + row.get("path"): (row.get("bytes"), row.get("sha256")) + for row in manifest.get("files", []) + } + expected_listed = reader.files() - {"PACKAGE_MANIFEST.json", "SHA256SUMS"} + if set(listed) != expected_listed: + raise VerificationError("PACKAGE_MANIFEST file coverage mismatch") + for path, (size, digest) in listed.items(): + data = reader.read(path) + if len(data) != size or sha256_bytes(data) != digest: + raise VerificationError(f"PACKAGE_MANIFEST file identity mismatch: {path}") + checks.append("PACKAGE_MANIFEST_PASS") + + inner_path = f"05_COMPONENTS/R1/{INNER_FILENAME}" + inner_data = reader.read(inner_path) + inner_expected = manifest.get("components", {}).get("inner_r1", {}) + inner_hash = sha256_bytes(inner_data) + if ( + inner_expected.get("filename") != INNER_FILENAME + or inner_expected.get("bytes") != len(inner_data) + or inner_expected.get("sha256") != inner_hash + or not HEX64.fullmatch(inner_hash) + ): + raise VerificationError("corrected inner component exact identity mismatch") + index, rows, inner_root = nested_inner(inner_data) + inner_inventory = verify_inner_exact_inventory(inner_data) + validate_inner_sidecar_references(inner_data) + if len(inner_inventory) != INNER_EXPECTED_FILE_COUNT: + raise VerificationError("corrected inner exact inventory count mismatch") + semantic_members, semantic_bytes = scan_all_current_inner_payloads(inner_data) + semantic_count = verify_active_manifest_semantics(inner_data, rows, inner_root) + checks.append( + "INNER_EXACT_INVENTORY_85_AND_CURRENT_SEMANTICS_PASS;" + f"CURRENT_MEMBERS={semantic_members};CURRENT_BYTES={semantic_bytes};" + f"ACTIVE_MANIFESTS_CHECKED={semantic_count}" + ) + checks.append("INNER_ACTIVE_DESCENDANTS_27_PASS") + + pc_path = f"05_COMPONENTS/PRIMITIVE_COMMONS/{PC_FILENAME}" + pc_data = reader.read(pc_path) + if len(pc_data) != PC_BYTES or sha256_bytes(pc_data) != PC_SHA256: + raise VerificationError("Primitive Commons beta.5 is not frozen byte-exact") + if manifest.get("components", {}).get("primitive_commons_beta5") != { + "bytes": PC_BYTES, + "filename": PC_FILENAME, + "sha256": PC_SHA256, + }: + raise VerificationError("Primitive Commons manifest identity mismatch") + checks.append("PRIMITIVE_COMMONS_BETA5_EXACT_PASS") + + identity = read_json(reader, "03_AUDIT/R1_1.0.1_DESCENDANT_BYTE_IDENTITY_MAP.json") + if ( + identity.get("active_descendant_count") != 27 + or identity.get("comparison_result") != "27_OF_27_BYTE_IDENTICAL" + or identity.get("operational_descendant_bytes_changed") != 0 + ): + raise VerificationError("descendant byte-identity summary mismatch") + mapped = {row.get("path"): row for row in identity.get("active_descendants", [])} + if set(mapped) != set(rows) or len(mapped) != 27: + raise VerificationError("descendant byte-identity map coverage mismatch") + for path, current in rows.items(): + row = mapped[path] + if ( + row.get("bytes") != current.get("bytes") + or row.get("sha256") != current.get("sha256") + or row.get("predecessor_bytes") != current.get("bytes") + or row.get("predecessor_sha256") != current.get("sha256") + or row.get("status") != "BYTE_IDENTICAL_TO_AUGUST_14_ACTIVE_DESCENDANT" + ): + raise VerificationError(f"descendant byte-identity map mismatch: {path}") + predecessor = identity.get("predecessor_inner", {}) + if predecessor != { + "bytes": OLD_INNER_BYTES, + "filename": OLD_INNER_FILENAME, + "sha256": OLD_INNER_SHA256, + }: + raise VerificationError("predecessor inner identity mismatch") + removed = identity.get("removed_non_active_payload", {}) + if removed != { + "bytes": REMOVED_MYTH_BYTES, + "embedded_in_successor": False, + "filename": REMOVED_MYTH_FILENAME, + "sha256": REMOVED_MYTH_SHA256, + }: + raise VerificationError("removed generic Myth identity mismatch") + checks.append("DESCENDANT_BYTE_IDENTITY_27_OF_27_PASS") + + generic_ref = read_json(reader, "06_EXTERNAL_SIDECARS/GENERIC_MYTH_V0.2.0_REFERENCE.json") + full_ref = read_json(reader, "06_EXTERNAL_SIDECARS/FULL_CANON_MYTH_V0.3.5_REFERENCE.json") + validate_external_reference(generic_ref, GENERIC) + validate_external_reference(full_ref, FULL_CANON) + checks.append("OPTIONAL_EXTERNAL_SIDECAR_REFERENCES_PASS") + + historical = read_json(reader, "07_HISTORICAL/PRESERVED_AUGUST_14_R1_REFERENCE.json") + expected_historical = { + "archive_embedded": False, + "bytes": OLD_OUTER_BYTES, + "filename": OLD_OUTER_FILENAME, + "preservation": "IMMUTABLE_HISTORICAL_RELEASE_EVIDENCE", + "sha256": OLD_OUTER_SHA256, + "superseded": True, + "supersession_scope": "PACKAGING_BOUNDARY_ONLY", + "tag": OLD_OUTER_TAG, + } + for key, expected in expected_historical.items(): + if historical.get(key) != expected: + raise VerificationError(f"historical predecessor reference mismatch: {key}") + checks.append("HISTORICAL_PREDECESSOR_REFERENCE_ONLY_PASS") + + capa = read_json(reader, "04_CORRECTION/CAPA_PS-R1-PRIVATE-MYTH-PUBLIC-BOUNDARY-001.json") + if ( + capa.get("capa_id") != "PS-R1-PRIVATE-MYTH-PUBLIC-BOUNDARY-001" + or capa.get("status") != "IMPLEMENTED_PENDING_EFFECTIVENESS" + or capa.get("closure_state") != "NOT_CLOSED" + or capa.get("historical_august_14_archive_mutated") is not False + ): + raise VerificationError("CAPA state is not implemented-pending-effectiveness/not-closed") + checks.append("CAPA_IMPLEMENTED_PENDING_EFFECTIVENESS_PASS") + + slot = read_json(reader, "01_AUTHORITY/INNER_ADMISSION_SLOT.json") + authority = slot.get("status") + admission_path = "01_AUTHORITY/INNER_EXACT_HASH_MAINTAINER_ADMISSION.json" + if slot.get("inner") != { + "active_descendant_count": 27, + "bytes": len(inner_data), + "filename": INNER_FILENAME, + "sha256": inner_hash, + }: + raise VerificationError("inner authority slot exact identity mismatch") + if authority == "PENDING_EXACT_HASH_MAINTAINER_ADMISSION": + if inventory_mode != "PENDING": + raise VerificationError("pending authority slot disagrees with code-owned outer inventory") + if slot.get("admission_record_embedded") is not False or admission_path in reader.files(): + raise VerificationError("pending inner authority slot unexpectedly embeds a record") + expected_status = "NONPUBLISHABLE_PENDING_INNER_EXACT_HASH_ADMISSION" + release_state = "PASS_NONPUBLISHABLE_PENDING_INNER_AUTHORITY" + elif authority == "BOUND_EXACT_HASH_MAINTAINER_ADMISSION": + if inventory_mode != "BOUND": + raise VerificationError("bound authority slot disagrees with code-owned outer inventory") + if slot.get("admission_record_embedded") is not True or admission_path not in reader.files(): + raise VerificationError("bound inner authority slot is missing its record") + admission = reader.read(admission_path) + if sha256_bytes(admission) != slot.get("admission_record_sha256"): + raise VerificationError("inner admission record hash mismatch") + validate_admission_record(admission, inner_hash, len(inner_data)) + expected_status = "AWAITING_EXTERNAL_EXACT_HASH_PUBLIC_RELEASE_AUTHORIZATION" + release_state = "PASS_AWAITING_EXTERNAL_OUTER_AUTHORIZATION" + else: + raise VerificationError(f"unknown inner authority status: {authority!r}") + status = read_json(reader, "02_STATUS/PUBLIC_REFERENCE_STATUS.json") + if status.get("status") != expected_status or manifest.get("status") != expected_status: + raise VerificationError("authority mode and package status disagree") + if status.get("inner_authority_gate") != authority: + raise VerificationError("status inner authority gate mismatch") + outer_gate = status.get("outer_publication_gate", {}) + if outer_gate != { + "authorization_is_external_to_archive": True, + "publication_authorized": False, + "status": "PENDING_EXACT_OUTER_ZIP_HASH_AUTHORIZATION_AFTER_BUILD", + }: + raise VerificationError("outer publication gate must remain external and pending") + if any("OUTER" in PurePosixPath(path).name.upper() and "AUTHORIZATION" in PurePosixPath(path).name.upper() for path in reader.files()): + raise VerificationError("outer exact-hash authorization must not be embedded") + checks.append(f"AUTHORITY_BOUNDARY_PASS;INNER={authority};OUTER=PENDING_EXTERNAL") + + outer_semantic_members, outer_semantic_bytes = scan_outer_current_controls(reader) + checks.append( + "OUTER_CURRENT_CONTROL_SEMANTICS_PASS;" + f"MEMBERS={outer_semantic_members};BYTES={outer_semantic_bytes}" + ) + + forbidden = { + REMOVED_MYTH_SHA256, + GENERIC["sha256"], + FULL_CANON["sha256"], + OLD_OUTER_SHA256, + } + scan_members = 0 + scan_bytes = 0 + budget = [0, 0] + for path in sorted(reader.files()): + payload = reader.read(path) + digest = sha256_bytes(payload) + if digest in forbidden: + raise VerificationError(f"forbidden exact payload embedded as {path}") + if path.lower().endswith(".zip"): + count, size = recursive_forbidden_scan(payload, forbidden, path, budget=budget) + scan_members += count + scan_bytes += size + checks.append(f"RECURSIVE_FORBIDDEN_PAYLOAD_SCAN_PASS;MEMBERS={scan_members};BYTES={scan_bytes}") + + safety = read_json(reader, "SAFETY_AND_SECRET_SCAN_REPORT.json") + if safety.get("status") != "PASS" or safety.get("findings") != []: + raise VerificationError("embedded safety report is not PASS with zero findings") + checks.append("EMBEDDED_SAFETY_REPORT_PASS") + + return { + "authority_state": authority, + "checks": checks, + "deterministic_zip_metadata_verified": reader.deterministic_metadata_verified, + "integrity": "PASS", + "outer_bytes": source.stat().st_size if source.is_file() else None, + "outer_sha256": sha256_file(source) if source.is_file() else None, + "release_state": release_state, + "source": str(source), + "status": "PASS", + "tag": TAG, + } + finally: + reader.close() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("package", help=f"{ROOT} directory or {ARCHIVE}") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + report = verify_package(Path(args.package).resolve()) + except (VerificationError, OSError, zipfile.BadZipFile, json.JSONDecodeError) as exc: + report = {"status": "FAIL", "integrity": "FAIL", "error": str(exc)} + print(json.dumps(report, indent=2, sort_keys=True)) + return 1 + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verify_repository_evidence.py b/tools/verify_repository_evidence.py index e64112f..c40ad2e 100644 --- a/tools/verify_repository_evidence.py +++ b/tools/verify_repository_evidence.py @@ -10,15 +10,19 @@ import argparse import hashlib +import html import json import os import re import subprocess import sys +import tempfile import urllib.error import urllib.parse import urllib.request +from datetime import datetime, timezone from pathlib import Path, PurePosixPath +from html.parser import HTMLParser from typing import Any, Iterable @@ -65,6 +69,11 @@ CURRENT_REDOWNLOAD = ( ROOT / "governance" / "R1_0_1_PUBLIC_REDOWNLOAD_VERIFICATION_2026-08-17.json" ) +SIX_SITE_EFFECTIVENESS = ( + ROOT + / "governance" + / "R1_0_1_SIX_PUBLIC_SITES_EFFECTIVENESS_VERIFICATION_2026-08-17.json" +) R1_PACKAGE_MANIFEST = ROOT / "governance" / "R1_PACKAGE_MANIFEST.json" R1_RELEASE_GATES = ( ROOT / "governance" / "PROJECT_SHADOW_RELEASE_GATES_2026-08-14.json" @@ -138,6 +147,12 @@ "bytes": 5_731_663, "sha256": "6f6f1e16d5e9a20e62403f14af7ce8629ce2d702528fb7f80aaf4a14deb7a1d1", } +CURRENT_OUTER_VERIFIER_SHA256 = ( + "721c384b245ca654c087d184bbfe5725d85d41536250140467d7cd913e6a1ccb" +) +GENERIC_V0_2_0_VERIFIER_SHA256 = ( + "0d84c8f90da35a16abbd410744ebd7df6f06a836e0c0b830b5213fb598087e9b" +) GATE_1_CONFIRMED_BY = "Phillip Linstrum" GATE_1_CONFIRMED_AT = "2026-08-18T00:09:39Z" GENERIC_AUTHORIZATION_STATEMENT = ( @@ -165,14 +180,129 @@ RELEASE_PLACEHOLDER_RE = re.compile( r"<(?:GENERIC|INNER|OUTER)_[A-Z0-9_]+>" ) -PUBLIC_SITE_URLS = ( - "https://projectshadow.frylock117.chatgpt.site", - "https://pausebeforeharm.frylock117.chatgpt.site", - "https://civicqa.frylock117.chatgpt.site", - "https://americanrepairmanual.frylock117.chatgpt.site", - "https://therecord.frylock117.chatgpt.site", - "https://almsivi.frylock117.chatgpt.site", +CAPA_ID = "PS-R1-PRIVATE-MYTH-PUBLIC-BOUNDARY-001" +CURRENT_REDOWNLOAD_RECORD = ( + "governance/R1_0_1_PUBLIC_REDOWNLOAD_VERIFICATION_2026-08-17.json" +) +SIX_SITE_EFFECTIVENESS_RECORD = ( + "governance/" + "R1_0_1_SIX_PUBLIC_SITES_EFFECTIVENESS_VERIFICATION_2026-08-17.json" ) +CAPA_EFFECTIVENESS_RECORDS = ( + CURRENT_REDOWNLOAD_RECORD, + SIX_SITE_EFFECTIVENESS_RECORD, +) +HF_SPACE_RESOLVE_BASE = ( + "https://huggingface.co/spaces/ProjectShadow/" + "project-shadow-r1-reference/resolve/main" +) +PUBLIC_SITE_REQUIREMENTS = ( + { + "site_id": "project-shadow", + "base_url": "https://projectshadow.frylock117.chatgpt.site", + "routes": ("/", "/release", "/status", "/capa"), + "specific_checks": ( + "outer_exact_identity", + "inner_exact_identity", + "generic_exact_identity", + "full_canon_exact_identity", + "member_set_identity", + "capa_id", + "capa_state", + ), + }, + { + "site_id": "pause-before-harm", + "base_url": "https://pausebeforeharm.frylock117.chatgpt.site", + "routes": ("/",), + "specific_checks": (), + }, + { + "site_id": "civic-qa", + "base_url": "https://civicqa.frylock117.chatgpt.site", + "routes": ("/",), + "specific_checks": (), + }, + { + "site_id": "american-repair-manual", + "base_url": "https://americanrepairmanual.frylock117.chatgpt.site", + "routes": ("/", "/manual.html"), + "specific_checks": ("archived_manual_current_boundary",), + }, + { + "site_id": "the-record", + "base_url": "https://therecord.frylock117.chatgpt.site", + "routes": ("/", "/corrections", "/national.html"), + "specific_checks": ( + "capa_state", + "national_trump_record", + ), + }, + { + "site_id": "almsivi", + "base_url": "https://almsivi.frylock117.chatgpt.site", + "routes": ("/", "/technical/project-shadow"), + "specific_checks": ( + "outer_exact_identity", + "inner_exact_identity", + "generic_exact_identity", + "full_canon_exact_identity", + "member_set_identity", + "preserved_predecessor", + ), + }, +) +PUBLIC_SITE_URLS = tuple( + str(requirement["base_url"]) for requirement in PUBLIC_SITE_REQUIREMENTS +) +COMMON_SITE_CHECKS = ( + "r1_0_1_current", + "contains_no_myth_package", + "generic_v0_2_0", + "full_canon_v0_3_5", + "sidecars_separate", + "sidecars_optional", + "sidecars_default_off", + "sidecars_nonauthorizing", + "canonical_release_link", + "capa_link", +) +OPERATIONAL_MEMBER_SET_SHA256 = ( + "7a557efad953cbafd9e3ea9eb29b2d3e3e1bc6ab99dcf6b9ae7a99c487b0754d" +) +SITE_ROUTE_FINAL_PATH_OVERRIDES = { + ("american-repair-manual", "/manual.html"): "/manual", + ("the-record", "/national.html"): "/national", +} +SITE_ROUTE_MAX_BYTES_OVERRIDES = { + ("the-record", "/national.html"): 16 * 1024 * 1024, +} +SITE_RECEIPT_METHOD = { + "anonymous_https": True, + "http_status_required": 200, + "default_max_response_bytes": 2 * 1024 * 1024, + "route_max_response_bytes": { + "the-record:/national.html": 16 * 1024 * 1024, + }, + "response_hash_algorithm": "SHA-256", + "content_type_prefix": "text/html", + "same_origin_final_url_required": True, + "semantic_content_checks": True, +} +CURRENT_MIRROR_PATHS = { + "generic-myth-v0.2.0": ( + "releases/generic-myth-v0.2.0/" + "Project_Shadow_Generic_Myth_Sidecar_v0.2.0_OPTIONAL_PUBLIC_COMPANION_2026-08-17.zip" + ), + "r1.0.1-2026-08-17": ( + "releases/r1.0.1-2026-08-17/" + "Project_Shadow_R1.0.1_Public_Reference_2026-08-17.zip" + ), +} +PUBLIC_DOWNLOAD_FINAL_HOSTS = { + "GITHUB": frozenset({"github.com", "release-assets.githubusercontent.com"}), + "HUGGING_FACE": frozenset({"huggingface.co"}), +} USER_AGENT = "Project-Shadow-read-only-evidence-verifier/1.0" MAX_LINK_RESPONSE_BYTES = 2 * 1024 * 1024 @@ -229,6 +359,7 @@ "not ", "no ", "never ", + "neither ", "without ", "does not ", "do not ", @@ -404,6 +535,21 @@ def phase_name(value: str) -> str: return normalized +def parse_rfc3339_utc(label: str, value: Any) -> datetime: + if not isinstance(value, str) or re.fullmatch( + r"20\d\d-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])T" + r"(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\dZ", + value, + ) is None: + raise EvidenceError(f"{label} is not RFC3339 UTC") + try: + return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace( + tzinfo=timezone.utc + ) + except ValueError as exc: + raise EvidenceError(f"{label} is not a real UTC timestamp") from exc + + def published_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: return [row for row in rows if row.get("publication_state") in PUBLISHED_STATES] @@ -883,8 +1029,13 @@ def validate_public_documentation( ), ) generic_markers = [ - "generic-myth-v0.2.0", - "never embedded in R1.0.1", + "Project Shadow Generic Myth Sidecar v0.2.0", + GENERIC_FINAL["filename"], + GENERIC_FINAL["sha256"], + "separate from canonical Project Shadow R1", + "default-off", + "Terminal-only", + "Nonauthorizing", "No production or operational deployment is authorized", ] r1_markers = [ @@ -944,7 +1095,7 @@ def validate_public_documentation( ROOT / "README.md", ( "POSTPUBLICATION", - "IMPLEMENTED_PENDING_EFFECTIVENESS", + "CLOSED_EFFECTIVE", "r1.0.1-2026-08-17", "generic-myth-v0.2.0", ), @@ -968,29 +1119,47 @@ def validate_current_redownload( raise EvidenceError("unsupported R1.0.1 redownload schema") if record.get("status") != "VERIFIED" or record.get("anonymous_download") is not True: raise EvidenceError("R1.0.1 redownload record is not anonymous/VERIFIED") + recorded_at = record.get("recorded_at") + parse_rfc3339_utc("R1.0.1 redownload timestamp", recorded_at) + if record.get("method") != { + "anonymous_https": True, + "exact_byte_count_and_sha256": True, + "http_status_required": 200, + }: + raise EvidenceError("R1.0.1 redownload method is incomplete") check_nonclaims("R1.0.1 redownload record", record.get("nonclaims")) observations = record.get("observations") if not isinstance(observations, list): raise EvidenceError("R1.0.1 redownload observations missing") - expected: dict[tuple[str, str], dict[str, Any]] = {} + expected: dict[tuple[str, str], tuple[dict[str, Any], str]] = {} for tag, role in ( ("generic-myth-v0.2.0", "OPTIONAL_GENERIC_COMPANION"), ("r1.0.1-2026-08-17", "R1_REFERENCE_CORRECTED"), ): asset = by_tag[tag]["asset"] - expected[(role, "GITHUB")] = asset - expected[(role, "HUGGING_FACE")] = asset + expected[(role, "GITHUB")] = (asset, asset["download_url"]) + expected[(role, "HUGGING_FACE")] = ( + asset, + f"{HF_SPACE_RESOLVE_BASE}/{CURRENT_MIRROR_PATHS[tag]}", + ) observed_keys: set[tuple[str, str]] = set() for row in observations: if not isinstance(row, dict): raise EvidenceError("invalid R1.0.1 redownload observation") key = (str(row.get("role")), str(row.get("host"))) - asset = expected.get(key) - if asset is None or key in observed_keys: + expected_row = expected.get(key) + if expected_row is None or key in observed_keys: raise EvidenceError(f"unexpected/duplicate redownload observation: {key}") + asset, expected_url = expected_row observed_keys.add(key) + final_host = row.get("final_host") + if not isinstance(final_host, str): + raise EvidenceError(f"redownload final host missing: {key}") + validate_public_download_final_host(key[1], f"https://{final_host}/") if ( row.get("filename") != asset["filename"] + or row.get("download_url") != expected_url + or row.get("http_status") != 200 or row.get("bytes_expected") != asset["bytes"] or row.get("bytes_observed") != asset["bytes"] or row.get("sha256_expected") != asset["sha256"] @@ -1000,6 +1169,190 @@ def validate_current_redownload( raise EvidenceError(f"redownload identity mismatch: {key}") if observed_keys != set(expected): raise EvidenceError("redownload record lacks GitHub/Hugging Face coverage") + generic_verification = record.get("generic_v0_2_0_bounded_verification") + if generic_verification != { + "tool": "tools/verify_generic_myth_v0_2_0.py", + "tool_sha256": GENERIC_V0_2_0_VERIFIER_SHA256, + "target": GENERIC_FINAL, + "inventory_path_count": 23, + "bounded_archive_verifier": True, + "old_predecessor_embedded": False, + "observations": [ + {"host": "GITHUB", "status": "PASS"}, + {"host": "HUGGING_FACE", "status": "PASS"}, + ], + }: + raise EvidenceError("Generic v0.2.0 public-download verification mismatch") + recursive = record.get("r1_0_1_recursive_verification") + if recursive != { + "tool": "tools/verify_outer_release.py", + "tool_sha256": CURRENT_OUTER_VERIFIER_SHA256, + "target": OUTER_FINAL, + "status": "PASS", + "recursive_forbidden_payload_scan": True, + "zero_embedded_myth_payload": True, + "observations": [ + {"host": "GITHUB", "status": "PASS"}, + {"host": "HUGGING_FACE", "status": "PASS"}, + ], + }: + raise EvidenceError("R1.0.1 recursive public-download verification mismatch") + + +def required_site_checks(requirement: dict[str, Any]) -> tuple[str, ...]: + satellite_checks = ( + ("no_direct_github_release_link",) + if requirement["site_id"] != "project-shadow" + else () + ) + return COMMON_SITE_CHECKS + satellite_checks + tuple(requirement["specific_checks"]) + + +def exact_public_site_url(base_url: str, route: str) -> str: + if not route.startswith("/") or route.startswith("//"): + raise EvidenceError(f"unsafe public-site route: {route!r}") + return base_url + route + + +def site_route_key(requirement: dict[str, Any], route: str) -> tuple[str, str]: + return (str(requirement["site_id"]), route) + + +def site_route_max_bytes(requirement: dict[str, Any], route: str) -> int: + return SITE_ROUTE_MAX_BYTES_OVERRIDES.get( + site_route_key(requirement, route), + MAX_LINK_RESPONSE_BYTES, + ) + + +def expected_site_final_url(requirement: dict[str, Any], route: str) -> str: + final_path = SITE_ROUTE_FINAL_PATH_OVERRIDES.get( + site_route_key(requirement, route), + route, + ) + return exact_public_site_url(str(requirement["base_url"]), final_path) + + +def validate_six_public_sites_effectiveness( + record: dict[str, Any], + by_tag: dict[str, dict[str, Any]], +) -> None: + """Validate retained route observations and corrected-boundary semantics.""" + if record.get("schema") != ( + "project-shadow.r1.0.1-six-public-sites-effectiveness-verification.v1" + ): + raise EvidenceError("unsupported six-public-sites effectiveness schema") + if ( + record.get("capa_id") != CAPA_ID + or record.get("status") != "VERIFIED" + or record.get("all_six_verified") is not True + ): + raise EvidenceError("six-public-sites effectiveness receipt is not VERIFIED") + verified_at = record.get("verified_at") + parse_rfc3339_utc( + "six-public-sites effectiveness timestamp", + verified_at, + ) + if record.get("method") != SITE_RECEIPT_METHOD: + raise EvidenceError("six-public-sites effectiveness method is incomplete") + check_nonclaims("six-public-sites effectiveness receipt", record.get("nonclaims")) + + bindings = record.get("artifact_bindings") + if not isinstance(bindings, dict): + raise EvidenceError("six-public-sites effectiveness receipt lacks artifact bindings") + expected_bindings = { + "r1_0_1_outer": by_tag["r1.0.1-2026-08-17"]["asset"], + "r1_0_1_inner": INNER_FINAL, + "generic_myth_v0_2_0": by_tag["generic-myth-v0.2.0"]["asset"], + "full_canon_myth_v0_3_5": by_tag["myth-v0.3.5"]["asset"], + } + if set(bindings) != set(expected_bindings): + raise EvidenceError("six-public-sites artifact-binding coverage mismatch") + for key, expected in expected_bindings.items(): + observed = bindings.get(key) + if not isinstance(observed, dict): + raise EvidenceError(f"six-public-sites artifact binding is invalid: {key}") + if set(observed) != {"filename", "bytes", "sha256"}: + raise EvidenceError(f"six-public-sites artifact binding has extra fields: {key}") + for field in ("filename", "bytes", "sha256"): + if observed.get(field) != expected[field]: + raise EvidenceError( + f"six-public-sites artifact binding mismatch: {key}.{field}" + ) + + sites = record.get("sites") + if not isinstance(sites, list) or len(sites) != len(PUBLIC_SITE_REQUIREMENTS): + raise EvidenceError("six-public-sites receipt does not contain exactly six sites") + observed_ids: set[str] = set() + for requirement, site in zip(PUBLIC_SITE_REQUIREMENTS, sites): + if not isinstance(site, dict): + raise EvidenceError("six-public-sites receipt contains an invalid site row") + site_id = str(requirement["site_id"]) + if site.get("site_id") != site_id or site_id in observed_ids: + raise EvidenceError("six-public-sites site order/identity mismatch") + observed_ids.add(site_id) + base_url = str(requirement["base_url"]) + if site.get("base_url") != base_url: + raise EvidenceError(f"six-public-sites base URL mismatch: {site_id}") + routes = site.get("routes") + expected_routes = tuple(str(value) for value in requirement["routes"]) + if not isinstance(routes, list) or len(routes) != len(expected_routes): + raise EvidenceError(f"six-public-sites route coverage mismatch: {site_id}") + for expected_route, route in zip(expected_routes, routes): + if not isinstance(route, dict): + raise EvidenceError(f"invalid route observation: {site_id}") + expected_url = exact_public_site_url(base_url, expected_route) + expected_final = expected_site_final_url(requirement, expected_route) + size = route.get("bytes_observed") + digest = route.get("sha256_observed") + if ( + route.get("path") != expected_route + or route.get("url") != expected_url + or route.get("final_url") != expected_final + or route.get("http_status") != 200 + or route.get("content_type") != "text/html" + or not isinstance(size, int) + or isinstance(size, bool) + or size <= 0 + or size > site_route_max_bytes(requirement, expected_route) + or not isinstance(digest, str) + or re.fullmatch(r"[0-9a-f]{64}", digest) is None + ): + raise EvidenceError( + f"six-public-sites route observation mismatch: {site_id}{expected_route}" + ) + checks = site.get("semantic_checks") + expected_checks = required_site_checks(requirement) + if not isinstance(checks, dict) or set(checks) != set(expected_checks): + raise EvidenceError(f"six-public-sites semantic-check coverage mismatch: {site_id}") + failed = sorted(key for key, value in checks.items() if value is not True) + if failed: + raise EvidenceError( + f"six-public-sites semantic checks are not all true for {site_id}: " + + ", ".join(failed) + ) + + +def validate_capa_effectiveness_record_pointers( + current_status: dict[str, Any], + capa: dict[str, Any], +) -> None: + closure = capa.get("closure") + current_capa = current_status.get("capa") + if not isinstance(closure, dict) or not isinstance(current_capa, dict): + raise EvidenceError("current status/CAPA closure object missing") + expected = list(CAPA_EFFECTIVENESS_RECORDS) + if ( + closure.get("verification_record") != CURRENT_REDOWNLOAD_RECORD + or closure.get("verification_records") != expected + or current_capa.get("verification_records") != expected + or not all( + record in current_status.get("source_records", []) for record in expected + ) + ): + raise EvidenceError( + "CAPA closure must bind redownload and six-public-sites effectiveness records" + ) def verify_repository_metadata(phase: str = "POSTPUBLICATION") -> list[dict[str, Any]]: @@ -1030,6 +1383,10 @@ def verify_repository_metadata(phase: str = "POSTPUBLICATION") -> list[dict[str, if manifest.get("schema") != "project-shadow.publication-manifest.v2": raise EvidenceError("unsupported publication manifest schema") + if current_status.get("schema") != "project-shadow.current-release-status.v1": + raise EvidenceError("unsupported current release-status schema") + if capa.get("schema") != "project-shadow.capa.v1": + raise EvidenceError("unsupported CAPA schema") if manifest.get("repository") != "PauseBeforeHarmProtocol/Project-Shadow": raise EvidenceError("unexpected repository identity") if manifest.get("publication_phase") != phase: @@ -1107,20 +1464,28 @@ def verify_repository_metadata(phase: str = "POSTPUBLICATION") -> list[dict[str, raise EvidenceError(f"release {tag!r} has invalid state for phase {phase}") note = RELEASE_NOTES[tag].read_text(encoding="utf-8") - for expected in (tag,) + identity_markers: + note_markers = identity_markers if tag == "generic-myth-v0.2.0" else (tag,) + identity_markers + for expected in note_markers: if expected not in note: raise EvidenceError(f"release notes for {tag} omit {expected!r}") - if not note.lstrip().startswith("**"): + if tag != "generic-myth-v0.2.0" and not note.lstrip().startswith("**"): raise EvidenceError(f"release notes for {tag} lack a bold first-line warning") if tag == "myth-v0.3.5": source_warning_present = ( "automatically generated source archive" in note and "repackaged copy" in note ) - else: + elif tag == "r1.0.1-2026-08-17": + source_warning_present = ( + "automatically generated source ZIP and TAR archives" in note + and "repository snapshots" in note + ) + elif tag != "generic-myth-v0.2.0": source_warning_present = ( "automatically generated" in note and "Source code (zip)" in note ) + else: + source_warning_present = True if not source_warning_present: raise EvidenceError(f"release notes for {tag} omit the source-archive warning") @@ -1528,19 +1893,41 @@ def verify_repository_metadata(phase: str = "POSTPUBLICATION") -> list[dict[str, "pending-effectiveness postpublication state is inconsistent" ) elif capa_status == "CLOSED_EFFECTIVE": + closed_at = closure.get("closed_at") if ( manifest.get("postpublication_state") != "PUBLISHED_EFFECTIVENESS_VERIFIED" or current_capa.get("effectiveness_verified") is not True or closure.get("effectiveness_verified") is not True - or closure.get("verification_record") - != "governance/R1_0_1_PUBLIC_REDOWNLOAD_VERIFICATION_2026-08-17.json" ): raise EvidenceError("current status/CAPA postpublication closure mismatch") - validate_current_redownload( - require_object(load_json(CURRENT_REDOWNLOAD), CURRENT_REDOWNLOAD.name), + closed_time = parse_rfc3339_utc("CAPA closure timestamp", closed_at) + validate_capa_effectiveness_record_pointers(current_status, capa) + redownload_record = require_object( + load_json(CURRENT_REDOWNLOAD), + CURRENT_REDOWNLOAD.name, + ) + six_site_record = require_object( + load_json(SIX_SITE_EFFECTIVENESS), + SIX_SITE_EFFECTIVENESS.name, + ) + validate_current_redownload(redownload_record, by_tag) + validate_six_public_sites_effectiveness( + six_site_record, by_tag, ) + evidence_time = max( + parse_rfc3339_utc( + "R1.0.1 redownload timestamp", + redownload_record.get("recorded_at"), + ), + parse_rfc3339_utc( + "six-public-sites effectiveness timestamp", + six_site_record.get("verified_at"), + ), + ) + if closed_time < evidence_time: + raise EvidenceError("CAPA closure predates retained effectiveness evidence") else: raise EvidenceError(f"unsupported postpublication CAPA state: {capa_status!r}") @@ -1598,7 +1985,10 @@ def request(url: str) -> Any: parsed = urllib.parse.urlparse(url) if parsed.scheme != "https" or not parsed.netloc: raise EvidenceError(f"refusing non-HTTPS URL: {url}") - headers = {"User-Agent": USER_AGENT} + headers = { + "Accept-Encoding": "identity", + "User-Agent": USER_AGENT, + } if parsed.netloc == "api.github.com": headers["Accept"] = "application/vnd.github+json" token = os.environ.get("GITHUB_TOKEN") @@ -1620,32 +2010,87 @@ def request(url: str) -> Any: return response +def public_download_provider(url: str) -> str | None: + hostname = (urllib.parse.urlparse(url).hostname or "").lower() + if hostname == "github.com": + return "GITHUB" + if hostname == "huggingface.co": + return "HUGGING_FACE" + return None + + +def validate_public_download_final_host( + provider: str, + final_url: str, +) -> str: + parsed = urllib.parse.urlparse(final_url) + hostname = (parsed.hostname or "").lower() + allowed = hostname in PUBLIC_DOWNLOAD_FINAL_HOSTS[provider] + if provider == "HUGGING_FACE" and hostname.endswith(".aws.cdn.hf.co"): + allowed = True + if parsed.scheme != "https" or not allowed: + raise EvidenceError( + f"unexpected {provider} public-download final host: {final_url}" + ) + return hostname + + def download_and_hash(asset: dict[str, Any], destination: Path) -> Path: destination.mkdir(parents=True, exist_ok=True) output = destination / asset["filename"] + if output.exists() or output.is_symlink(): + raise EvidenceError(f"refusing to overwrite release download: {output}") digest = hashlib.sha256() count = 0 - with request(asset["download_url"]) as response, output.open("wb") as handle: - header = response.headers.get("Content-Length") - if header is not None: - try: - advertised = int(header) - except ValueError as exc: - raise EvidenceError(f"invalid Content-Length for {asset['filename']}") from exc - if advertised != asset["bytes"]: - raise EvidenceError( - f"Content-Length mismatch for {asset['filename']}: " - f"expected {asset['bytes']}; found {advertised}" - ) - while True: - block = response.read(1024 * 1024) - if not block: - break - count += len(block) - if count > asset["bytes"]: - raise EvidenceError(f"oversized release download: {asset['filename']}") - digest.update(block) - handle.write(block) + temporary: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="wb", + dir=destination, + prefix=f".{asset['filename']}.", + delete=False, + ) as handle: + temporary = Path(handle.name) + with request(asset["download_url"]) as response: + if getattr(response, "status", None) != 200: + raise EvidenceError( + f"unexpected public-download status for {asset['filename']}" + ) + provider = public_download_provider(str(asset["download_url"])) + if provider is not None: + validate_public_download_final_host(provider, response.geturl()) + header = response.headers.get("Content-Length") + if header is not None: + try: + advertised = int(header) + except ValueError as exc: + raise EvidenceError( + f"invalid Content-Length for {asset['filename']}" + ) from exc + if advertised != asset["bytes"]: + raise EvidenceError( + f"Content-Length mismatch for {asset['filename']}: " + f"expected {asset['bytes']}; found {advertised}" + ) + while True: + block = response.read(1024 * 1024) + if not block: + break + count += len(block) + if count > asset["bytes"]: + raise EvidenceError( + f"oversized release download: {asset['filename']}" + ) + digest.update(block) + handle.write(block) + handle.flush() + os.fsync(handle.fileno()) + os.link(temporary, output) + temporary.unlink() + temporary = None + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) if count != asset["bytes"] or digest.hexdigest() != asset["sha256"]: raise EvidenceError( f"release download identity mismatch for {asset['filename']}: " @@ -1654,6 +2099,17 @@ def download_and_hash(asset: dict[str, Any], destination: Path) -> Path: return output +def download_and_hash_at_url( + asset: dict[str, Any], + download_url: str, + destination: Path, +) -> Path: + """Download one exact asset identity from an explicitly selected mirror.""" + mirrored = dict(asset) + mirrored["download_url"] = download_url + return download_and_hash(mirrored, destination) + + def check_link(url: str) -> None: with request(url) as response: status = getattr(response, "status", None) @@ -1679,7 +2135,8 @@ def fetch_public_json(url: str) -> dict[str, Any]: def normalize_release_body(value: str) -> str: - return value.replace("\r\n", "\n").replace("\r", "\n").rstrip() + normalized = value.replace("\r\n", "\n").replace("\r", "\n") + return "\n".join(line.rstrip() for line in normalized.splitlines()).rstrip() def verify_live_release_metadata( @@ -1730,36 +2187,264 @@ def verify_live_release_metadata( ) -def fetch_public_page(url: str) -> str: +def fetch_public_site_route( + requirement: dict[str, Any], + route: str, +) -> tuple[str, dict[str, Any]]: + url = exact_public_site_url(str(requirement["base_url"]), route) + expected_final = expected_site_final_url(requirement, route) + limit = site_route_max_bytes(requirement, route) with request(url) as response: status = getattr(response, "status", None) - if status is not None and not 200 <= status < 400: + final_url = response.geturl() + content_type = response.headers.get_content_type() + requested = urllib.parse.urlparse(url) + final = urllib.parse.urlparse(final_url) + if status != 200: raise EvidenceError(f"unexpected HTTP {status} for {url}") - body = response.read(MAX_LINK_RESPONSE_BYTES + 1) - if len(body) > MAX_LINK_RESPONSE_BYTES: - raise EvidenceError(f"public page exceeds bounded verification size: {url}") + if ( + final.scheme != requested.scheme + or final.netloc != requested.netloc + or final_url != expected_final + ): + raise EvidenceError(f"public page final URL mismatch: {url} -> {final_url}") + body = bytearray() + digest = hashlib.sha256() + while True: + block = response.read(min(1024 * 1024, limit + 1 - len(body))) + if not block: + break + body.extend(block) + digest.update(block) + if len(body) > limit: + raise EvidenceError(f"public page exceeds bounded size {limit}: {url}") + if content_type != "text/html": + raise EvidenceError(f"public page content type is not text/html: {url}") try: - return body.decode("utf-8") + source = bytes(body).decode("utf-8") except UnicodeDecodeError as exc: raise EvidenceError(f"public page is not UTF-8: {url}") from exc + return source, { + "path": route, + "url": url, + "final_url": final_url, + "http_status": status, + "bytes_observed": len(body), + "sha256_observed": digest.hexdigest(), + "content_type": content_type, + } -def verify_public_site_release_links(rows: list[dict[str, Any]]) -> None: +class PublicHTMLView(HTMLParser): + """Extract visible text and anchors while excluding executable/hidden payloads.""" + + HIDDEN_TAGS = frozenset({"script", "style", "template", "noscript"}) + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.hidden_depth = 0 + self.text_chunks: list[str] = [] + self.hrefs: list[str] = [] + + def handle_starttag( + self, + tag: str, + attrs: list[tuple[str, str | None]], + ) -> None: + lowered = tag.lower() + if lowered in self.HIDDEN_TAGS: + self.hidden_depth += 1 + return + if self.hidden_depth == 0 and lowered == "a": + for name, value in attrs: + if name.lower() == "href" and isinstance(value, str): + self.hrefs.append(html.unescape(value)) + + def handle_endtag(self, tag: str) -> None: + if tag.lower() in self.HIDDEN_TAGS and self.hidden_depth: + self.hidden_depth -= 1 + + def handle_data(self, data: str) -> None: + if self.hidden_depth == 0: + self.text_chunks.append(data) + + +def public_html_view(source: str) -> tuple[str, tuple[str, ...]]: + parser = PublicHTMLView() + parser.feed(source) + parser.close() + text = re.sub(r"\s+", " ", " ".join(parser.text_chunks)).strip() + return text, tuple(parser.hrefs) + + +def normalized_public_text(source: str) -> str: + return public_html_view(source)[0] + + +def public_site_semantic_checks( + requirement: dict[str, Any], + pages: dict[str, str], +) -> dict[str, bool]: + """Evaluate the corrected boundary as meaning, not link presence alone.""" + missing_routes = [route for route in requirement["routes"] if route not in pages] + if missing_routes: + raise EvidenceError( + f"semantic input lacks routes for {requirement['site_id']}: " + + ", ".join(missing_routes) + ) + home_source = pages["/"] + home_text, home_links = public_html_view(home_source) + combined_source = "\n".join(pages[route] for route in requirement["routes"]) + combined_text = normalized_public_text(combined_source) canonical_release = "https://projectshadow.frylock117.chatgpt.site/release" - for index, url in enumerate(PUBLIC_SITE_URLS): - page = fetch_public_page(url) - expected = 'href="/release"' if index == 0 else canonical_release - if expected not in page: - raise EvidenceError(f"public site omits canonical release link: {url}") + canonical_capa = "https://projectshadow.frylock117.chatgpt.site/capa" + site_id = str(requirement["site_id"]) + release_link = "/release" if site_id == "project-shadow" else canonical_release + capa_link = "/capa" if site_id == "project-shadow" else canonical_capa + + checks: dict[str, bool] = { + "r1_0_1_current": re.search( + r"\b(?:project\s+shadow\s+)?(?:1\.0\.1|r1\.0\.1)\b", + home_text, + re.IGNORECASE, + ) is not None, + "contains_no_myth_package": re.search( + r"\b(?:contains?\s+no\s+myth\s+package|myth[- ]free\s+r1\.0\.1|" + r"no\s+myth\s+package\s+is\s+embedded|zero\s+embedded\s+myth\s+payload)\b", + home_text, + re.IGNORECASE, + ) is not None, + "generic_v0_2_0": re.search( + r"\bgeneric\s+myth\s+v0\.2\.0\b", home_text, re.IGNORECASE + ) is not None, + "full_canon_v0_3_5": re.search( + r"\bfull[- ]canon\s+myth\s+v0\.3\.5\b", home_text, re.IGNORECASE + ) is not None, + "sidecars_separate": re.search(r"\bseparate\b", home_text, re.IGNORECASE) + is not None, + "sidecars_optional": re.search(r"\boptional\b", home_text, re.IGNORECASE) + is not None, + "sidecars_default_off": re.search( + r"\b(?:default[- ]off|default\s+off|off\s+is\s+the\s+default)\b", + home_text, + re.IGNORECASE, + ) is not None, + "sidecars_nonauthorizing": re.search( + r"\b(?:non[- ]?authorizing|cannot\s+authorize|never\s+authorizes?|" + r"neither.{0,100}authorizes?)\b", + home_text, + re.IGNORECASE, + ) is not None, + "canonical_release_link": release_link in home_links, + "capa_link": capa_link in home_links, + } + if site_id != "project-shadow": + checks["no_direct_github_release_link"] = not any( + urllib.parse.urlparse(link).netloc.lower() == "github.com" + and "/PauseBeforeHarmProtocol/Project-Shadow/releases" in link + for link in home_links + ) - release_page = fetch_public_page(canonical_release) - for row in rows: + exact_values = { + "outer_exact_identity": ("5,731,663", OUTER_FINAL["sha256"]), + "inner_exact_identity": ("5,463,189", INNER_FINAL["sha256"]), + "generic_exact_identity": ("93,676", GENERIC_FINAL["sha256"]), + "full_canon_exact_identity": ( + "1,428,812", + "2b55867fe7c502a0defd8d6f2e9b53fbd1caaf1b0f225a438bd45b04a3e7bae2", + ), + "member_set_identity": (OPERATIONAL_MEMBER_SET_SHA256,), + } + for name, markers in exact_values.items(): + if name in requirement["specific_checks"]: + checks[name] = all(marker in combined_text for marker in markers) + if "capa_id" in requirement["specific_checks"]: + checks["capa_id"] = CAPA_ID in combined_text + if "capa_state" in requirement["specific_checks"]: + checks["capa_state"] = re.search( + r"\b(?:implemented\W+pending\W+effectiveness|closed\W+effective|" + r"closed\W+effectiveness\W+verified)\b", + combined_text, + re.IGNORECASE, + ) is not None + if "archived_manual_current_boundary" in requirement["specific_checks"]: + manual = normalized_public_text(pages["/manual.html"]) + checks["archived_manual_current_boundary"] = all( + re.search(pattern, manual, re.IGNORECASE) is not None + for pattern in ( + r"(?:1\.0\.1|r1\.0\.1)", + r"contains?\s+no\s+myth\s+package", + r"generic\s+myth\s+v0\.2\.0", + r"full[- ]canon\s+myth\s+v0\.3\.5", + ) + ) + if "national_trump_record" in requirement["specific_checks"]: + checks["national_trump_record"] = re.search( + r"\bnational\s+trump\s+record\b", + normalized_public_text(pages["/national.html"]), + re.IGNORECASE, + ) is not None + if "preserved_predecessor" in requirement["specific_checks"]: + checks["preserved_predecessor"] = ( + re.search(r"\bpreserved\s+predecessor\b", combined_text, re.IGNORECASE) + is not None + and "2f8fe1530b6a83294d15011df95853aaecf08fa4dba756f0c2e91dd089e1b1ec" + in combined_text + ) + return checks + + +def observe_public_sites(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + release_page = "" + observations: list[dict[str, Any]] = [] + for requirement in PUBLIC_SITE_REQUIREMENTS: + pages: dict[str, str] = {} + routes: list[dict[str, Any]] = [] + for route in requirement["routes"]: + source, observation = fetch_public_site_route(requirement, str(route)) + pages[str(route)] = source + routes.append(observation) + checks = public_site_semantic_checks(requirement, pages) + expected_checks = required_site_checks(requirement) + if set(checks) != set(expected_checks): + raise EvidenceError( + f"live public-site semantic coverage mismatch: {requirement['site_id']}" + ) + failed = sorted(key for key, value in checks.items() if value is not True) + if failed: + raise EvidenceError( + f"live public-site boundary mismatch for {requirement['site_id']}: " + + ", ".join(failed) + ) + observations.append( + { + "site_id": requirement["site_id"], + "base_url": requirement["base_url"], + "routes": routes, + "semantic_checks": checks, + } + ) + if requirement["site_id"] == "project-shadow": + release_page = pages["/release"] + if not release_page: + raise EvidenceError("canonical Project Shadow release page was not checked") + current_rows = [ + row for row in rows if row.get("publication_state") == "PUBLISHED" + ] + if len(current_rows) != 3: + raise EvidenceError("canonical release-page current-row coverage is ambiguous") + for row in current_rows: asset = row["asset"] for expected in (asset["download_url"], asset["sha256"]): if expected not in release_page: raise EvidenceError( f"canonical public release page omits {row['role']} identity: {expected}" ) + return observations + + +def verify_public_site_release_links(rows: list[dict[str, Any]]) -> None: + observe_public_sites(rows) def verify_online( @@ -1772,12 +2457,22 @@ def verify_online( phase = phase_name(phase) online_rows = published_rows(rows) if phase == "PREPUBLICATION" else rows downloaded: dict[str, Path] = {} + hf_downloaded: dict[str, Path] = {} repository = "PauseBeforeHarmProtocol/Project-Shadow" for row in online_rows: asset = row["asset"] downloaded[row["role"]] = download_and_hash(asset, destination) release_url = f"https://github.com/{repository}/releases/tag/{row['tag']}" check_link(release_url) + + by_tag = {str(row["tag"]): row for row in rows} + for tag in ("generic-myth-v0.2.0", "r1.0.1-2026-08-17"): + row = by_tag[tag] + hf_downloaded[tag] = download_and_hash_at_url( + row["asset"], + f"{HF_SPACE_RESOLVE_BASE}/{CURRENT_MIRROR_PATHS[tag]}", + destination / "hugging-face", + ) verify_live_release_metadata(online_rows, expected_latest_tag) current_status = require_object(load_json(CURRENT_STATUS), CURRENT_STATUS.name) if ( @@ -1810,6 +2505,60 @@ def verify_online( ) print(completed.stdout.rstrip()) + generic = downloaded.get("OPTIONAL_GENERIC_COMPANION") + hf_generic = hf_downloaded.get("generic-myth-v0.2.0") + if generic is None or hf_generic is None: + raise EvidenceError("downloaded assets lack both Generic v0.2.0 mirrors") + for host, archive in (("GITHUB", generic), ("HUGGING_FACE", hf_generic)): + generic_completed = subprocess.run( + [ + sys.executable, + "-I", + "-S", + "-B", + str(ROOT / "tools" / "verify_generic_myth_v0_2_0.py"), + str(archive), + ], + cwd=ROOT, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + if generic_completed.returncode != 0: + raise EvidenceError( + f"bounded verification of {host} Generic v0.2.0 failed:\n" + + generic_completed.stdout + ) + print(f"{host} Generic v0.2.0: {generic_completed.stdout.rstrip()}") + + current_r1 = downloaded.get("R1_REFERENCE_CORRECTED") + hf_current_r1 = hf_downloaded.get("r1.0.1-2026-08-17") + if current_r1 is None or hf_current_r1 is None: + raise EvidenceError("downloaded assets lack both corrected R1.0.1 mirrors") + for host, archive in (("GITHUB", current_r1), ("HUGGING_FACE", hf_current_r1)): + current_completed = subprocess.run( + [ + sys.executable, + "-I", + "-S", + "-B", + str(ROOT / "tools" / "verify_outer_release.py"), + str(archive), + ], + cwd=ROOT, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + if current_completed.returncode != 0: + raise EvidenceError( + f"positive recursive verification of {host} R1.0.1 failed:\n" + + current_completed.stdout + ) + print(f"{host} R1.0.1: {current_completed.stdout.rstrip()}") + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) @@ -1883,7 +2632,7 @@ def main() -> int: phase == "POSTPUBLICATION" and current_status.get("capa", {}).get("status") == "CLOSED_EFFECTIVE" ): - print("PASS: six public-site release links") + print("PASS: six public-site routes and corrected-boundary semantics") elif phase == "POSTPUBLICATION": print("INFO: six-site CAPA effectiveness verification remains pending") else: